diff --git a/.github/actions/coverage/action.yml b/.github/actions/coverage/action.yml index efadc06..a7894ca 100644 --- a/.github/actions/coverage/action.yml +++ b/.github/actions/coverage/action.yml @@ -59,8 +59,9 @@ runs: - name: Generate Coverage shell: bash + working-directory: offchain run: | - cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + cargo llvm-cov --all-features --workspace --lcov --output-path ../lcov.info env: CARGO_TERM_COLOR: always CACHE_ON_FAILURE: true diff --git a/.github/actions/gcp-auth-infra/action.yml b/.github/actions/gcp-auth-infra/action.yml new file mode 100644 index 0000000..9f47415 --- /dev/null +++ b/.github/actions/gcp-auth-infra/action.yml @@ -0,0 +1,37 @@ +name: Authenticate to GCP (infra project) +description: Authenticate via workload identity to the GCP project that owns GCR + shared images +inputs: + project-id: + description: 'GCP infra project ID' + required: true + project-number: + description: 'GCP infra project number' + required: true + provider-name: + description: 'Workload identity pool provider name (e.g. nexus-tools)' + required: true +outputs: + auth_token: + description: 'OAuth2 access token for GCR login' + value: ${{ steps.auth.outputs.auth_token }} +runs: + using: composite + steps: + - name: Authenticate + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ inputs.project-id }} + workload_identity_provider: "projects/${{ inputs.project-number }}/locations/global/workloadIdentityPools/${{ inputs.project-id }}/providers/${{ inputs.provider-name }}" + # No service_account / token_format — use Direct WIF. + # google-github-actions/auth exposes the federated token via + # `auth_token` output, which docker/login-action consumes below. + # `token_format: 'access_token'` would require a service_account + # to impersonate, which we don't have for this WIF setup. + + - name: Login to GCR + uses: docker/login-action@v3 + with: + registry: gcr.io + username: oauth2accesstoken + password: ${{ steps.auth.outputs.auth_token }} diff --git a/.github/actions/gcp-auth-protocol/action.yml b/.github/actions/gcp-auth-protocol/action.yml new file mode 100644 index 0000000..c161081 --- /dev/null +++ b/.github/actions/gcp-auth-protocol/action.yml @@ -0,0 +1,33 @@ +name: Authenticate to GCP (protocol project) +description: Authenticate via workload identity to the GCP project that owns GCS + Secret Manager +inputs: + project-id: + description: 'GCP project ID' + required: true + project-number: + description: 'GCP project number' + required: true + provider-name: + description: 'Workload identity pool provider name (e.g. nexus-tools)' + required: true +outputs: + credentials_file_path: + description: 'Path to the generated credentials file' + value: ${{ steps.auth.outputs.credentials_file_path }} +runs: + using: composite + steps: + - name: Authenticate + id: auth + uses: google-github-actions/auth@v2 + with: + project_id: ${{ inputs.project-id }} + workload_identity_provider: "projects/${{ inputs.project-number }}/locations/global/workloadIdentityPools/${{ inputs.project-id }}/providers/${{ inputs.provider-name }}" + + - name: Setup Python (gcloud needs it) + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup Cloud SDK + uses: google-github-actions/setup-gcloud@v2 diff --git a/.github/actions/install-nexus-cli/action.yml b/.github/actions/install-nexus-cli/action.yml new file mode 100644 index 0000000..bfe2227 --- /dev/null +++ b/.github/actions/install-nexus-cli/action.yml @@ -0,0 +1,29 @@ +name: Install Nexus CLI +description: Extract the nexus CLI from the nexus-sdk shell image +inputs: + nexus-tag: + description: 'Tag of gcr.io/.../nexus-sdk/shell to extract from' + required: true + infra-registry: + description: 'Infra GCR registry prefix (e.g. gcr.io/production-tf-talus-infra)' + required: true +runs: + using: composite + steps: + - name: Extract nexus binary + shell: bash + env: + NEXUS_TAG: ${{ inputs.nexus-tag }} + INFRA_REGISTRY: ${{ inputs.infra-registry }} + run: | + set -euo pipefail + mkdir -p "$HOME/.local/bin" + CONTAINER_ID=$(docker create "${INFRA_REGISTRY}/nexus-sdk/shell:${NEXUS_TAG}") + docker cp "${CONTAINER_ID}:/usr/local/bin/nexus" "$HOME/.local/bin/nexus" + docker rm "${CONTAINER_ID}" + chmod +x "$HOME/.local/bin/nexus" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Verify + shell: bash + run: nexus --version diff --git a/.github/actions/install-sui/action.yml b/.github/actions/install-sui/action.yml new file mode 100644 index 0000000..08cc1ca --- /dev/null +++ b/.github/actions/install-sui/action.yml @@ -0,0 +1,45 @@ +name: Install Sui CLI +description: Install the Sui CLI from suiup, with cache support +inputs: + sui-channel: + description: 'Sui release channel (e.g. testnet, mainnet)' + required: true + cache-version: + description: 'Bump to invalidate the cache' + required: false + default: 'v1' +runs: + using: composite + steps: + - name: Cache Sui binary + id: cache + uses: actions/cache@v4 + with: + path: ~/.local/bin/sui + key: sui-${{ runner.os }}-${{ inputs.sui-channel }}-${{ inputs.cache-version }} + + - name: Install suiup and Sui CLI + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + SUI_CHANNEL: ${{ inputs.sui-channel }} + run: | + set -euo pipefail + install_dir="$HOME/.local/bin" + temp_dir="$(mktemp -d)" + mkdir -p "$install_dir" + curl --retry 5 --retry-all-errors -sSfL \ + -o "$temp_dir/suiup.tar.gz" \ + https://github.com/MystenLabs/suiup/releases/latest/download/suiup-Linux-musl-x86_64.tar.gz + tar -xzf "$temp_dir/suiup.tar.gz" -C "$temp_dir" + install -m 0755 "$temp_dir/suiup" "$install_dir/suiup" + "$install_dir/suiup" install "sui@$SUI_CHANNEL" -y + + - name: Add Sui to PATH + shell: bash + run: echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Verify + shell: bash + run: sui --version diff --git a/.github/actions/pre-commit/dependencies/action.yml b/.github/actions/pre-commit/dependencies/action.yml index 469ce99..d862cc1 100644 --- a/.github/actions/pre-commit/dependencies/action.yml +++ b/.github/actions/pre-commit/dependencies/action.yml @@ -20,4 +20,4 @@ runs: - name: Install NPM stuff from helpers/npm-install-g.txt shell: bash - run: ./helpers/npm-install-g.sh + run: ./offchain/helpers/npm-install-g.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..53c7529 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,252 @@ +name: CI + +on: + workflow_dispatch: + inputs: + pr-number: + description: 'PR number to publish for (chain ops fire against the PR HEAD)' + required: false + type: string + target-env: + description: 'Deploy the dispatched ref directly against this env. For iteration on the deploy pipeline without merging to main. Leave empty if using pr-number.' + required: false + type: choice + default: '' + options: + - '' + - testnet + - mainnet + pull_request: + types: [opened, synchronize, reopened] + push: + branches: + - main + - 'iterate/testnet/**' + - 'iterate/mainnet/**' + # Tag-driven deploy. The tag prefix selects the env: + # testnet-* → full pipeline against testnet + # mainnet-* → full pipeline against mainnet + # Tags are immutable per the convention, so each release-style + # promotion lands on a specific commit and the deploy run is + # attributable to that exact ref. No merge-strategy gotchas, no + # ephemeral promote/* branches, no long-lived deploy branches to + # keep clean. + tags: + - 'testnet-*' + - 'mainnet-*' + +permissions: + contents: write + packages: write + id-token: write + pull-requests: write + checks: write + +# Serialize chain-touching runs per target network so two pushes can't race on +# wallet consolidation, register PTBs, or terraform apply locks. Different +# networks (testnet vs mainnet) run in parallel; PRs (dry-run only) are keyed +# per-PR; push-to-main (image builds only, no chain ops) is keyed per-ref. +# `cancel-in-progress: false` queues subsequent runs instead of cancelling — +# we never want to abandon a half-finished register/apply sequence. +concurrency: + group: >- + ${{ + (startsWith(github.ref_name, 'mainnet-') || startsWith(github.ref_name, 'iterate/mainnet/')) && 'chain-mainnet' || + (startsWith(github.ref_name, 'testnet-') || startsWith(github.ref_name, 'iterate/testnet/')) && 'chain-testnet' || + (github.event_name == 'workflow_dispatch' && inputs.target-env != '') && format('chain-{0}', inputs.target-env) || + github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || + format('ref-{0}', github.ref) + }} + cancel-in-progress: false + +jobs: + # ── Resolve target ref for workflow_dispatch with PR# ──────── + resolve-target: + if: github.event_name == 'workflow_dispatch' && inputs.pr-number != '' + runs-on: ubuntu-latest + outputs: + ref: ${{ steps.r.outputs.ref }} + head-sha: ${{ steps.r.outputs.head-sha }} + base-ref: ${{ steps.r.outputs.base-ref }} + steps: + - name: Resolve + id: r + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + BASE=$(gh api "repos/${{ github.repository }}/pulls/${{ inputs.pr-number }}" --jq '.base.ref') + HEAD_REF=$(gh api "repos/${{ github.repository }}/pulls/${{ inputs.pr-number }}" --jq '.head.ref') + HEAD_SHA=$(gh api "repos/${{ github.repository }}/pulls/${{ inputs.pr-number }}" --jq '.head.sha') + echo "base-ref=$BASE" >> "$GITHUB_OUTPUT" + echo "ref=$HEAD_REF" >> "$GITHUB_OUTPUT" + echo "head-sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + # ── Discover tools (always) ────────────────────────────────── + discover: + uses: ./.github/workflows/offchain-tools.discover.yml + with: + base-ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }} + + # target-ref passed to callables. The callables' `environment:` expression + # treats this string as the env hint: it resolves to 'mainnet' when this + # evaluates to 'mainnet', else 'testnet'. Precedence: + # 1. PR# dispatch → PR base branch (testnet/mainnet) + # 2. dispatch with target-env input → that env literal + # 3. push to iterate//... → that env literal + # 4. PR event → PR base + # 5. push to a deploy branch → branch name (main/testnet/mainnet) + + # ── Build / push ───────────────────────────────────────────── + # PR (any base): dry-run on changed matrix. + # Push main: push on changed matrix (no chain ops). + # Push iterate//** or tag testnet-*/mainnet-*: push on all matrix. + # workflow_dispatch with PR# or target-env: push on all matrix. + deploy: + needs: [discover, resolve-target] + if: always() && !cancelled() && needs.discover.result == 'success' + uses: ./.github/workflows/offchain-tools.deploy.yml + with: + target-ref: >- + ${{ + needs.resolve-target.outputs.base-ref || + inputs.target-env || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'mainnet-') && 'mainnet') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'testnet-') && 'testnet') || + (startsWith(github.ref_name, 'iterate/mainnet/') && 'mainnet') || + (startsWith(github.ref_name, 'iterate/testnet/') && 'testnet') || + (github.event_name == 'pull_request' && github.base_ref) || + github.ref_name + }} + matrix-json: >- + ${{ + (github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == 'main')) + && needs.discover.outputs.matrix-changed + || needs.discover.outputs.matrix-all + }} + dry-run: ${{ github.event_name == 'pull_request' }} + secrets: inherit + + # ── Prepare ────────────────────────────────────────────────── + # Fires on: + # - push to iterate//** branches + # - push of testnet-* / mainnet-* tags + # - workflow_dispatch with pr-number OR target-env + prepare: + needs: [discover, deploy, resolve-target] + if: >- + always() && !cancelled() && needs.deploy.result == 'success' && + ((github.event_name == 'workflow_dispatch' && (inputs.pr-number != '' || inputs.target-env != '')) || + (github.event_name == 'push' && github.ref_type == 'branch' && ( + startsWith(github.ref_name, 'iterate/testnet/') || + startsWith(github.ref_name, 'iterate/mainnet/') + )) || + (github.event_name == 'push' && github.ref_type == 'tag' && ( + startsWith(github.ref_name, 'testnet-') || + startsWith(github.ref_name, 'mainnet-') + ))) + uses: ./.github/workflows/offchain-tools.prepare.yml + with: + target-ref: >- + ${{ + needs.resolve-target.outputs.base-ref || + inputs.target-env || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'mainnet-') && 'mainnet') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'testnet-') && 'testnet') || + (startsWith(github.ref_name, 'iterate/mainnet/') && 'mainnet') || + (startsWith(github.ref_name, 'iterate/testnet/') && 'testnet') || + (github.event_name == 'pull_request' && github.base_ref) || + github.ref_name + }} + matrix-json: ${{ needs.discover.outputs.matrix-all }} + secrets: inherit + + # ── Register (after prepare) ──────────────────────────────── + register: + needs: [prepare, resolve-target] + if: always() && !cancelled() && needs.prepare.result == 'success' + uses: ./.github/workflows/offchain-tools.register.yml + with: + target-ref: >- + ${{ + needs.resolve-target.outputs.base-ref || + inputs.target-env || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'mainnet-') && 'mainnet') || + (github.ref_type == 'tag' && startsWith(github.ref_name, 'testnet-') && 'testnet') || + (startsWith(github.ref_name, 'iterate/mainnet/') && 'mainnet') || + (startsWith(github.ref_name, 'iterate/testnet/') && 'testnet') || + (github.event_name == 'pull_request' && github.base_ref) || + github.ref_name + }} + secrets: inherit + + # ── Readiness (PRs targeting testnet/mainnet only) ────────── + readiness: + needs: discover + if: >- + github.event_name == 'pull_request' && + (github.base_ref == 'testnet' || github.base_ref == 'mainnet') && + fromJson(needs.discover.outputs.matrix-changed).include[0] != null + uses: ./.github/workflows/offchain-tools.readiness.yml + with: + matrix-json: ${{ needs.discover.outputs.matrix-changed }} + secrets: inherit + + # ── Trigger tf-nexus-tools apply (reconcile Cloud Run from GCS state) ── + # After register has written the new tool's config blob + on-chain entry, + # kick the terraform stack so Cloud Run services / ALB / DNS records get + # materialized. We wait for the dispatched run to complete and inherit + # its conclusion, so register-then-apply is one atomic gate. Uses + # GH_ACCESS_TOKEN PAT because GITHUB_TOKEN can't dispatch in other repos. + trigger-tf-apply: + needs: [register] + if: always() && !cancelled() && needs.register.result == 'success' + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} + steps: + - name: Require GH_ACCESS_TOKEN + run: | + if [ -z "${GH_TOKEN:-}" ]; then + echo "::error::GH_ACCESS_TOKEN secret is not configured on this repo (or not scoped to it)." + exit 1 + fi + + # Polls actions:read endpoints only — works with fine-grained PATs. + # `gh run watch` would also fetch check annotations (checks:read), + # which fine-grained PATs cannot carry. ava-game uses the same + # action for the same reason. + - name: Dispatch tf-nexus-tools terraform apply and wait + uses: the-actions-org/workflow-dispatch@v4.0.0 + with: + repo: Talus-Network/tf-nexus-tools + ref: production + workflow: Terraform + token: ${{ secrets.GH_ACCESS_TOKEN }} + inputs: '{}' + + # ── ci-gate: single required status check for branch protection ── + ci-gate: + if: always() + needs: [discover, deploy, prepare, register, readiness, trigger-tf-apply] + runs-on: ubuntu-latest + steps: + - name: Check job results + run: | + set -euo pipefail + declare -A r=( + [discover]="${{ needs.discover.result }}" + [deploy]="${{ needs.deploy.result }}" + [prepare]="${{ needs.prepare.result }}" + [register]="${{ needs.register.result }}" + [readiness]="${{ needs.readiness.result }}" + [trigger-tf-apply]="${{ needs.trigger-tf-apply.result }}" + ) + for j in "${!r[@]}"; do + v="${r[$j]}" + case "$v" in + success|skipped) echo "✓ $j: $v" ;; + *) echo "✗ $j: $v"; FAIL=1 ;; + esac + done + [ -z "${FAIL:-}" ] || exit 1 diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 02162ca..9f37020 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -11,7 +11,9 @@ jobs: uses: ./.github/workflows/detect_changes.yml with: files: | - tools/** + offchain/tools/** + offchain/Cargo.toml + offchain/Cargo.lock update-baseline: runs-on: ubuntu-latest diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index ab7427b..b6a2307 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -9,7 +9,9 @@ jobs: uses: ./.github/workflows/detect_changes.yml with: files: | - tools/** + offchain/tools/** + offchain/Cargo.toml + offchain/Cargo.lock coverage: runs-on: ubuntu-latest needs: detect-changes diff --git a/.github/workflows/deploy-payments-stripe-mainnet.yml b/.github/workflows/deploy-payments-stripe-mainnet.yml deleted file mode 100644 index 3d90565..0000000 --- a/.github/workflows/deploy-payments-stripe-mainnet.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: deploy-payments-stripe-mainnet - -on: - push: - tags: - - "v-payments-stripe-*" - -permissions: - contents: read - id-token: write - -env: - PROJECT_ID: talus-tools-mainnet - REGION: us-central1 - REPO: nexus-tools - CRATE: payments-stripe - SERVICE: payments-stripe-mainnet - -jobs: - guard: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Require testnet success on this commit - uses: actions/github-script@v7 - with: - script: | - const { owner, repo } = context.repo; - const ref = context.sha; - const { data } = await github.rest.checks.listForRef({ - owner, repo, ref, check_name: 'build-deploy-register' - }); - const ok = data.check_runs.some(c => - c.name === 'build-deploy-register' && c.conclusion === 'success' - ); - if (!ok) { - core.setFailed('Testnet deploy must succeed on this commit before mainnet.'); - } - - build-deploy-register: - needs: guard - runs-on: ubuntu-latest - environment: mainnet - steps: - - uses: actions/checkout@v4 - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_MAINNET }} - service_account: github-actions@${{ env.PROJECT_ID }}.iam.gserviceaccount.com - - - uses: google-github-actions/setup-gcloud@v2 - - - name: Configure Docker - run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet - - - name: Build and push image - env: - SHA: ${{ github.sha }} - run: | - IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/${CRATE}:${SHA}" - docker build -f tools/${CRATE}/deploy/Dockerfile -t "$IMAGE" . - docker push "$IMAGE" - - - name: Deploy to Cloud Run - env: - SHA: ${{ github.sha }} - run: | - export REGISTRY="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}" - envsubst < tools/${CRATE}/deploy/cloud-run.mainnet.yaml > /tmp/svc.yaml - gcloud run services replace /tmp/svc.yaml --region "$REGION" --quiet - URL=$(gcloud run services describe "$SERVICE" --region "$REGION" --format='value(status.url)') - echo "URL=$URL" >> "$GITHUB_ENV" - - - name: Smoke test - run: | - for p in $(jq -r '.[]' tools/${CRATE}/paths.json); do - curl -fsS "${URL}${p}/health" - curl -fsS "${URL}${p}/meta" | jq -e .fqn >/dev/null - done - - - name: Install Nexus CLI - run: | - curl -fsSL https://raw.githubusercontent.com/Talus-Network/homebrew-tap/main/install.sh | bash - echo "$HOME/.nexus/bin" >> "$GITHUB_PATH" - - - name: Register / update tools on Nexus mainnet - run: bash tools/${CRATE}/deploy/register.sh "$URL" mainnet diff --git a/.github/workflows/deploy-payments-stripe-testnet.yml b/.github/workflows/deploy-payments-stripe-testnet.yml deleted file mode 100644 index 59f7a38..0000000 --- a/.github/workflows/deploy-payments-stripe-testnet.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: deploy-payments-stripe-testnet - -on: - push: - branches: [main] - paths: - - "tools/payments-stripe/**" - - ".github/workflows/deploy-payments-stripe-testnet.yml" - workflow_dispatch: - -permissions: - contents: read - id-token: write - -env: - PROJECT_ID: talus-tools-testnet - REGION: us-central1 - REPO: nexus-tools - CRATE: payments-stripe - SERVICE: payments-stripe-testnet - -jobs: - build-deploy-register: - runs-on: ubuntu-latest - environment: testnet - steps: - - uses: actions/checkout@v4 - - - id: auth - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_TESTNET }} - service_account: github-actions@${{ env.PROJECT_ID }}.iam.gserviceaccount.com - - - uses: google-github-actions/setup-gcloud@v2 - - - name: Configure Docker - run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet - - - name: Build and push image - env: - SHA: ${{ github.sha }} - run: | - IMAGE="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}/${CRATE}:${SHA}" - docker build -f tools/${CRATE}/deploy/Dockerfile -t "$IMAGE" . - docker push "$IMAGE" - - - name: Deploy to Cloud Run - env: - SHA: ${{ github.sha }} - run: | - export REGISTRY="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO}" - envsubst < tools/${CRATE}/deploy/cloud-run.testnet.yaml > /tmp/svc.yaml - gcloud run services replace /tmp/svc.yaml --region "$REGION" --quiet - URL=$(gcloud run services describe "$SERVICE" --region "$REGION" --format='value(status.url)') - echo "URL=$URL" >> "$GITHUB_ENV" - - - name: Smoke test - run: | - for p in $(jq -r '.[]' tools/${CRATE}/paths.json); do - curl -fsS "${URL}${p}/health" - curl -fsS "${URL}${p}/meta" | jq -e .fqn >/dev/null - done - - - name: Install Nexus CLI - run: | - curl -fsSL https://raw.githubusercontent.com/Talus-Network/homebrew-tap/main/install.sh | bash - echo "$HOME/.nexus/bin" >> "$GITHUB_PATH" - - - name: Register / update tools on Nexus testnet - run: bash tools/${CRATE}/deploy/register.sh "$URL" testnet diff --git a/.github/workflows/offchain-tools.deploy.yml b/.github/workflows/offchain-tools.deploy.yml new file mode 100644 index 0000000..c3aae37 --- /dev/null +++ b/.github/workflows/offchain-tools.deploy.yml @@ -0,0 +1,112 @@ +name: Offchain Tools Deploy + +on: + workflow_call: + inputs: + target-ref: + required: false + type: string + matrix-json: + required: true + type: string + dry-run: + required: false + type: boolean + default: false + +permissions: + contents: read + packages: write + id-token: write + +jobs: + build: + name: "Build ${{ matrix.tool }} v${{ matrix.version }}" + runs-on: ubuntu-latest + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + strategy: + matrix: ${{ fromJson(inputs.matrix-json) }} + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute short SHA + id: sha + run: echo "short=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Compute image tags + id: tags + run: | + set -euo pipefail + SHA=${{ steps.sha.outputs.short }} + GHCR="ghcr.io/${{ github.repository }}/${{ matrix.tool }}:sha-${SHA}" + { + echo "list<> "$GITHUB_OUTPUT" + + # Build first, no auth required. The Rust compile is the slow part — + # doing auth before this risks the GCP access token (1h lifetime) + # expiring before the push window. Same pattern as ava-game's + # protocol.deploy.yml. + - name: Build (no push) + uses: docker/build-push-action@v6 + with: + context: . + file: ./offchain/Dockerfile + platforms: linux/amd64 + push: false + build-args: | + BINARY=${{ matrix.command }} + TOOL_DIR=${{ matrix.dir }} + TOOL_FQN_VERSION=${{ matrix.version }} + cache-from: type=gha,scope=offchain-${{ matrix.tool }} + cache-to: type=gha,mode=max,scope=offchain-${{ matrix.tool }} + + - name: Login to GHCR + if: ${{ !inputs.dry-run }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Authenticate to GCP (infra) + if: ${{ !inputs.dry-run }} + id: auth-infra + uses: ./.github/actions/gcp-auth-infra + with: + project-id: ${{ vars.GCP_INFRA_PROJECT_ID }} + project-number: ${{ vars.GCP_INFRA_PROJECT_NUMBER }} + provider-name: nexus-tools + + # Push pulls the layers we just built from the gha cache and emits + # them under the resolved tags. Token freshly minted in the step + # above, so this completes well within the 1h lifetime. + - name: Push + if: ${{ !inputs.dry-run }} + uses: docker/build-push-action@v6 + with: + context: . + file: ./offchain/Dockerfile + platforms: linux/amd64 + push: true + build-args: | + BINARY=${{ matrix.command }} + TOOL_DIR=${{ matrix.dir }} + TOOL_FQN_VERSION=${{ matrix.version }} + tags: ${{ steps.tags.outputs.list }} + cache-from: type=gha,scope=offchain-${{ matrix.tool }} + cache-to: type=gha,mode=max,scope=offchain-${{ matrix.tool }} diff --git a/.github/workflows/offchain-tools.discover.yml b/.github/workflows/offchain-tools.discover.yml new file mode 100644 index 0000000..fa6bb10 --- /dev/null +++ b/.github/workflows/offchain-tools.discover.yml @@ -0,0 +1,151 @@ +name: Offchain Tools Discover + +on: + workflow_call: + inputs: + base-ref: + description: 'Base ref to compute changed-files diff against' + required: false + type: string + default: '' + outputs: + matrix-all: + description: 'JSON matrix of every discovered tool' + value: ${{ jobs.discover.outputs.matrix-all }} + matrix-changed: + description: 'JSON matrix of tools whose subtree changed since base-ref' + value: ${{ jobs.discover.outputs.matrix-changed }} + content-hash: + description: 'Aggregate hash of all tools.json + their subtree versions' + value: ${{ jobs.discover.outputs.content-hash }} + +jobs: + discover: + name: Discover offchain tools + runs-on: ubuntu-latest + outputs: + matrix-all: ${{ steps.build.outputs.matrix-all }} + matrix-changed: ${{ steps.build.outputs.matrix-changed }} + content-hash: ${{ steps.build.outputs.content-hash }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed files + id: changes + if: inputs.base-ref != '' + uses: tj-actions/changed-files@v46 + with: + base_sha: ${{ inputs.base-ref }} + files: | + offchain/tools/** + offchain/Cargo.toml + offchain/Cargo.lock + offchain/Dockerfile + offchain/rust-toolchain.toml + + - name: Build matrices + id: build + shell: bash + env: + CHANGED_FILES: ${{ steps.changes.outputs.all_changed_files }} + # Space-separated list of tool_name values to skip. Sourced + # from the repo-level `BLOCKED_TOOLS` variable so security + # blocks can be flipped via repo settings without a PR. + # Unset = no tools blocked. + BLOCKED_TOOLS: ${{ vars.BLOCKED_TOOLS }} + run: | + set -euo pipefail + ALL='[]' + CHANGED='[]' + HASH_INPUT="" + + # Shared files force every tool into the "changed" set. + SHARED_TOUCHED=false + if [ -n "${CHANGED_FILES:-}" ]; then + for f in $CHANGED_FILES; do + case "$f" in + offchain/Cargo.toml|offchain/Cargo.lock|offchain/Dockerfile|offchain/rust-toolchain.toml) + SHARED_TOUCHED=true + ;; + esac + done + fi + echo "Shared files touched: $SHARED_TOUCHED" + + # Tools blocked from deployment are sourced from the repo + # variable `BLOCKED_TOOLS` (space-separated tool_name list). + # Discover excludes them from both matrices, which prevents + # the entire downstream pipeline (deploy → prepare → register + # → tf-apply) from touching them. Flip the variable in repo + # settings → Variables to unblock (no PR needed). + # + # Current blocks (2026-05-21): + # http — no SSRF guard; would let a DAG reach the GCP + # metadata server and exfiltrate an SA OAuth token. + # Unregistered on chain; re-enable after the guard + # ships in offchain/tools/http/. + read -ra BLOCKED_TOOLS_ARR <<< "${BLOCKED_TOOLS:-}" + echo "Blocked tools: ${BLOCKED_TOOLS_ARR[*]:-}" + + for TOOLS_JSON in offchain/tools/*/tools.json; do + [ -f "$TOOLS_JSON" ] || continue + DIR=$(dirname "$TOOLS_JSON") + TOOL_NAME=$(jq -r '.tool_name' "$TOOLS_JSON") + COMMAND=$(jq -r '.command' "$TOOLS_JSON") + + SKIP=false + for blocked in "${BLOCKED_TOOLS_ARR[@]}"; do + if [ "$TOOL_NAME" = "$blocked" ]; then + echo "::warning::Skipping blocked tool: $TOOL_NAME (in BLOCKED_TOOLS repo variable)" + SKIP=true + break + fi + done + [ "$SKIP" = "true" ] && continue + + TREE_HASH=$(git rev-parse "HEAD:${DIR}/") + CONTENT_HASH=$(printf '%s' "$TREE_HASH" | sha256sum | cut -d' ' -f1) + VERSION=$(printf '%d' "0x${CONTENT_HASH:0:8}") + + ENTRY=$(jq -nc \ + --arg tool "$TOOL_NAME" \ + --arg dir "$DIR" \ + --arg cmd "$COMMAND" \ + --arg ver "$VERSION" \ + '{tool: $tool, dir: $dir, command: $cmd, version: $ver}') + + ALL=$(echo "$ALL" | jq -c ". + [${ENTRY}]") + + # Is this tool changed? + TOOL_CHANGED=false + if [ "$SHARED_TOUCHED" = "true" ] || [ -z "${CHANGED_FILES:-}" ]; then + TOOL_CHANGED=true + else + for f in $CHANGED_FILES; do + case "$f" in + "${DIR}/"*) TOOL_CHANGED=true ;; + esac + done + fi + if [ "$TOOL_CHANGED" = "true" ]; then + CHANGED=$(echo "$CHANGED" | jq -c ". + [${ENTRY}]") + fi + + HASH_INPUT="${HASH_INPUT}${TOOL_NAME}:${VERSION}\n" + echo " - ${TOOL_NAME} v${VERSION} (changed=${TOOL_CHANGED})" + done + + CONTENT_HASH=$(printf '%s' "$HASH_INPUT" | sha256sum | cut -c1-16) + echo "matrix-all={\"include\":${ALL}}" >> "$GITHUB_OUTPUT" + echo "matrix-changed={\"include\":${CHANGED}}" >> "$GITHUB_OUTPUT" + echo "content-hash=${CONTENT_HASH}" >> "$GITHUB_OUTPUT" + + echo "::group::matrix-all" + echo "$ALL" | jq . + echo "::endgroup::" + echo "::group::matrix-changed" + echo "$CHANGED" | jq . + echo "::endgroup::" diff --git a/.github/workflows/offchain-tools.prepare.yml b/.github/workflows/offchain-tools.prepare.yml new file mode 100644 index 0000000..ddd3af5 --- /dev/null +++ b/.github/workflows/offchain-tools.prepare.yml @@ -0,0 +1,262 @@ +name: Offchain Tools Prepare + +on: + workflow_call: + inputs: + target-ref: + required: false + type: string + matrix-json: + required: true + type: string + +permissions: + contents: read + packages: read + id-token: write + +jobs: + prepare: + name: "Prepare ${{ matrix.tool }} v${{ matrix.version }}" + runs-on: ubuntu-latest + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + strategy: + matrix: ${{ fromJson(inputs.matrix-json) }} + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute short SHA + versioned name + id: names + run: | + SHA=${GITHUB_SHA::7} + echo "sha=${SHA}" >> "$GITHUB_OUTPUT" + echo "versioned=${{ matrix.tool }}-v${{ matrix.version }}" >> "$GITHUB_OUTPUT" + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull image + run: | + IMAGE="ghcr.io/${{ github.repository }}/${{ matrix.tool }}:sha-${{ steps.names.outputs.sha }}" + IMAGE="${IMAGE,,}" + docker pull "$IMAGE" + echo "IMAGE=$IMAGE" >> "$GITHUB_ENV" + + - name: Extract --meta + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/prepare/meta" + docker run --rm --entrypoint /usr/local/bin/${{ matrix.command }} \ + "$IMAGE" --meta \ + > "${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json" + echo "::group::Meta for ${{ matrix.tool }}" + jq '.[].fqn' "${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json" + echo "::endgroup::" + + - name: Render Cloud Run tool config + id: render + run: | + set -euo pipefail + DIR="${RUNNER_TEMP}/prepare/gcs/tools" + mkdir -p "$DIR" + VERSIONED="${{ steps.names.outputs.versioned }}" + # matrix.dir is the directory path (e.g. offchain/tools/storage-walrus). + TOOLS_JSON="${{ matrix.dir }}/tools.json" + EXTRA_ENV=$(jq -c '.environment // {}' "$TOOLS_JSON") + RESOURCES=$(jq -c '.resources // {}' "$TOOLS_JSON") + # nexus_contracts_tag is captured at prepare time and persisted into + # the per-tool config blob, so the (tool, version) pair is pinned to + # whichever nexus version was current when it was first prepared. + # Future NEXUS_TAG bumps only affect NEW deployments; existing + # services keep their original allowed-leaders mount path. + jq -n \ + --arg name "$VERSIONED" \ + --arg network "${{ vars.SUI_NETWORK }}" \ + --arg image "nexus-tools/${{ matrix.tool }}" \ + --arg tag "sha-${{ steps.names.outputs.sha }}" \ + --arg cmd "${{ matrix.command }}" \ + --arg nexus_tag "${{ vars.NEXUS_TAG }}" \ + --argjson extra_env "$EXTRA_ENV" \ + --argjson resources "$RESOURCES" \ + '{ + name: $name, + network: $network, + image: $image, + tag: $tag, + nexus_contracts_tag: $nexus_tag, + replicas: 1, + environment: ({ + RUST_LOG: "info", + BIND_ADDR: "0.0.0.0:8080", + NEXUS_TOOLKIT_CONFIG_PATH: "/app/secrets/toolkit-config.json" + } + $extra_env), + command: $cmd, + ports: [{containerPort: 8080}], + signed_http: {enabled: true}, + resources: $resources + }' > "$DIR/${VERSIONED}.json" + echo "config=$DIR/${VERSIONED}.json" >> "$GITHUB_OUTPUT" + + - name: Authenticate to GCP (protocol) + id: auth-gcs + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Authenticate to GCP (infra) + id: auth-infra + uses: ./.github/actions/gcp-auth-infra + with: + project-id: ${{ vars.GCP_INFRA_PROJECT_ID }} + project-number: ${{ vars.GCP_INFRA_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Generate signed HTTP keys (idempotent) + run: | + set -euo pipefail + VERSIONED="${{ steps.names.outputs.versioned }}" + PROJECT="${{ vars.GCP_PROJECT_ID }}" + META="${RUNNER_TEMP}/prepare/meta/${{ matrix.tool }}.json" + + FQNS=$(jq -r '.[].fqn' "$META" | tr '\n' ' ' | xargs) + echo "FQNs for $VERSIONED: $FQNS" + + FORCE_FLAG="" + EXISTING=$(gcloud secrets versions access latest \ + --secret="nexus-tools-${VERSIONED}-signed-http-keys" \ + --project="$PROJECT" 2>/dev/null || echo "") + if [ -n "$EXISTING" ]; then + EXISTING_FQNS=$(echo "$EXISTING" | jq -r '[.keys_registry[].fqn] | sort | join(" ")') + WANTED_FQNS=$(echo "$FQNS" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs) + if [ "$EXISTING_FQNS" != "$WANTED_FQNS" ]; then + echo "FQNs changed: '$EXISTING_FQNS' -> '$WANTED_FQNS' — forcing new keys" + FORCE_FLAG="--force" + else + echo "FQNs unchanged — skipping keygen" + exit 0 + fi + fi + + docker run --rm \ + -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/creds.json \ + -v "${{ steps.auth-gcs.outputs.credentials_file_path }}:/tmp/creds.json:ro" \ + gcr.io/${{ vars.GCP_INFRA_PROJECT_ID }}/nexus-next/generate-signed-http-keys:latest \ + python /app/bin/generate_signed_http_keys.py "nexus-tools" "$VERSIONED" \ + --fqns $FQNS \ + --project "$PROJECT" $FORCE_FLAG + + - name: Upload tool config to GCS + run: | + set -euo pipefail + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + VERSIONED="${{ steps.names.outputs.versioned }}" + LOCAL="${{ steps.render.outputs.config }}" + REMOTE="gs://${BUCKET}/${NETWORK}/offchain/tools/${VERSIONED}.json" + + # Preserve existing image tag pinning (only NEW versions adopt build SHA). + EXISTING_TAG=$(gsutil cat "$REMOTE" 2>/dev/null | jq -r '.tag // empty' || true) + if [ -n "$EXISTING_TAG" ]; then + echo "Preserving existing image tag: $EXISTING_TAG" + jq --arg t "$EXISTING_TAG" '.tag = $t' "$LOCAL" > "${LOCAL}.tmp" + mv "${LOCAL}.tmp" "$LOCAL" + fi + + LOCAL_HASH=$(sha256sum "$LOCAL" | cut -c1-16) + REMOTE_HASH=$(gsutil cat "$REMOTE" 2>/dev/null | sha256sum | cut -c1-16 || echo "") + if [ "$LOCAL_HASH" = "$REMOTE_HASH" ]; then + echo "Tool config unchanged — skipping upload" + else + gsutil cp "$LOCAL" "$REMOTE" + echo "Uploaded $REMOTE" + fi + + - name: Upload per-tool artifact + uses: actions/upload-artifact@v4 + with: + name: prepare-${{ matrix.tool }} + retention-days: 1 + path: | + ${{ runner.temp }}/prepare/meta/${{ matrix.tool }}.json + ${{ runner.temp }}/prepare/gcs/tools/${{ steps.names.outputs.versioned }}.json + + aggregate: + name: Aggregate prepare artifacts + runs-on: ubuntu-latest + needs: prepare + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + steps: + # Required so the local composite action gcp-auth-protocol resolves. + - name: Checkout + uses: actions/checkout@v4 + + - name: Download all prepare artifacts + uses: actions/download-artifact@v4 + with: + pattern: prepare-* + merge-multiple: true + path: ${{ runner.temp }}/prepare + + - name: Build manifest + id: manifest + run: | + set -euo pipefail + PREPARE="${{ runner.temp }}/prepare" + MANIFEST="${PREPARE}/manifest.json" + echo '{}' > "$MANIFEST" + MATRIX='${{ inputs.matrix-json }}' + echo "$MATRIX" | jq -c '.include[]' | while read -r entry; do + TOOL=$(echo "$entry" | jq -r '.tool') + VERSION=$(echo "$entry" | jq -r '.version') + COMMAND=$(echo "$entry" | jq -r '.command') + jq --arg t "$TOOL" --arg v "$VERSION" --arg c "$COMMAND" \ + '.[$t] = {command: $c, version: $v}' \ + "$MANIFEST" > "${MANIFEST}.tmp" && mv "${MANIFEST}.tmp" "$MANIFEST" + done + jq . "$MANIFEST" + HASH=$(sha256sum "$MANIFEST" | cut -c1-16) + echo "hash=$HASH" >> "$GITHUB_OUTPUT" + + - name: Authenticate to GCP (protocol) + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Upload manifest to GCS (content-addressed) + run: | + set -euo pipefail + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + HASH="${{ steps.manifest.outputs.hash }}" + REMOTE="gs://${BUCKET}/${NETWORK}/offchain/manifest/${HASH}.json" + if gsutil -q stat "$REMOTE" 2>/dev/null; then + echo "Manifest ${HASH}.json already exists — skipping" + else + gsutil cp "${{ runner.temp }}/prepare/manifest.json" "$REMOTE" + echo "Uploaded $REMOTE" + fi + + - name: Upload bundled prepare artifact + uses: actions/upload-artifact@v4 + with: + name: offchain-tools-prepare + retention-days: 1 + path: ${{ runner.temp }}/prepare/ diff --git a/.github/workflows/offchain-tools.readiness.yml b/.github/workflows/offchain-tools.readiness.yml new file mode 100644 index 0000000..c805f24 --- /dev/null +++ b/.github/workflows/offchain-tools.readiness.yml @@ -0,0 +1,66 @@ +name: Offchain Tools Readiness + +on: + workflow_call: + inputs: + matrix-json: + required: true + type: string + +permissions: + contents: read + id-token: write + +jobs: + check: + name: "Readiness ${{ matrix.tool }} v${{ matrix.version }}" + runs-on: ubuntu-latest + environment: >- + ${{ + (github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + strategy: + matrix: ${{ fromJson(inputs.matrix-json) }} + fail-fast: false + steps: + # Required so the local composite action gcp-auth-protocol resolves. + - name: Checkout + uses: actions/checkout@v4 + + - name: Authenticate to GCP (protocol) + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Check GCS artifacts + run: | + set -euo pipefail + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + TOOL="${{ matrix.tool }}" + VERSION="${{ matrix.version }}" + VERSIONED="${TOOL}-v${VERSION}" + + MISSING=() + + CFG="gs://${BUCKET}/${NETWORK}/offchain/tools/${VERSIONED}.json" + gsutil -q stat "$CFG" 2>/dev/null || MISSING+=("Cloud Run config: $CFG") + + REG_DIR="gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/" + REG_COUNT=$(gsutil ls "$REG_DIR" 2>/dev/null | wc -l || echo 0) + if [ "$REG_COUNT" = "0" ]; then + MISSING+=("FQN registrations under $REG_DIR") + fi + + if [ ${#MISSING[@]} -gt 0 ]; then + echo "::error::Missing artifacts for ${VERSIONED}:" + for m in "${MISSING[@]}"; do echo " - $m"; done + echo "" + echo "Trigger the Offchain Tools publish workflow with this PR number to seed them." + exit 1 + fi + + echo "✓ All artifacts present for ${VERSIONED}" diff --git a/.github/workflows/offchain-tools.register.yml b/.github/workflows/offchain-tools.register.yml new file mode 100644 index 0000000..f59e8cd --- /dev/null +++ b/.github/workflows/offchain-tools.register.yml @@ -0,0 +1,409 @@ +name: Offchain Tools Register + +on: + workflow_call: + inputs: + target-ref: + required: false + type: string + +permissions: + contents: read + id-token: write + +jobs: + register: + name: Register offchain tools + runs-on: ubuntu-latest + environment: >- + ${{ + (inputs.target-ref || github.event_name == 'pull_request' && github.base_ref || github.ref_name) == 'mainnet' && 'mainnet' || + 'testnet' + }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Sui CLI + uses: ./.github/actions/install-sui + with: + sui-channel: ${{ vars.SUI_CHANNEL }} + cache-version: ${{ vars.SUI_CACHE_VERSION }} + + - name: Download prepare artifacts + uses: actions/download-artifact@v4 + with: + name: offchain-tools-prepare + path: ${{ runner.temp }}/prepare + + - name: Show artifacts + id: artifacts + run: | + PREPARE="${{ runner.temp }}/prepare" + echo "manifest=${PREPARE}/manifest.json" >> "$GITHUB_OUTPUT" + echo "meta_dir=${PREPARE}/meta" >> "$GITHUB_OUTPUT" + jq . "${PREPARE}/manifest.json" + + - name: Authenticate to GCP (protocol) + id: auth-gcs + uses: ./.github/actions/gcp-auth-protocol + with: + project-id: ${{ vars.GCP_PROJECT_ID }} + project-number: ${{ vars.GCP_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Authenticate to GCP (infra) + uses: ./.github/actions/gcp-auth-infra + with: + project-id: ${{ vars.GCP_INFRA_PROJECT_ID }} + project-number: ${{ vars.GCP_INFRA_PROJECT_NUMBER }} + provider-name: nexus-tools + + - name: Install Nexus CLI + uses: ./.github/actions/install-nexus-cli + with: + nexus-tag: ${{ vars.NEXUS_TAG }} + infra-registry: gcr.io/${{ vars.GCP_INFRA_PROJECT_ID }} + + - name: Configure wallet and Nexus + run: | + set -euo pipefail + yes "" | sui client new-env --alias tool-register --rpc "${{ vars.NEXT_PUBLIC_SUI_RPC_URL }}" || true + sui client switch --env tool-register + + # SUI_DEPLOYER_PK is a `suiprivkey...` bech32 string (sui keytool + # export format). Same secret will drive future onchain Move + # deploys (`sui client publish`). + IMPORT=$(sui keytool import "${{ secrets.SUI_DEPLOYER_PK }}" ed25519) + ADDR=$(echo "$IMPORT" | grep -oE '0x[0-9a-fA-F]{64}') + sui client switch --address "$ADDR" + echo "DEPLOYER_ADDR=$ADDR" >> "$GITHUB_ENV" + + KEY_INDEX=$(sui keytool list --json | jq -r --arg addr "$ADDR" 'to_entries[] | select(.value.suiAddress == $addr) | .key') + SUI_PK=$(jq -r --argjson idx "$KEY_INDEX" '.[$idx]' "$HOME/.sui/sui_config/sui.keystore") + + NETWORK="${{ vars.SUI_NETWORK }}" + wget -q -O "$RUNNER_TEMP/objects.toml" \ + "https://storage.googleapis.com/production-talus-sui-objects/${{ vars.NEXUS_TAG }}/objects.${NETWORK}.toml?t=$(date +%s)" + + mkdir -p "$HOME/.nexus" + cp "$RUNNER_TEMP/objects.toml" "$HOME/.nexus/objects.${NETWORK}.toml" + nexus conf set \ + --nexus.objects "$HOME/.nexus/objects.${NETWORK}.toml" \ + --sui.rpc-url "${{ vars.NEXT_PUBLIC_SUI_RPC_URL }}" \ + --sui.pk "$SUI_PK" + + - name: Consolidate wallet gas coins + shell: bash + run: | + set -euo pipefail + # Merge most coins into one big primary so the register loop + # has enough headroom for budgets up to ~1 SUI per call + # (static deploy needed 1e9 MIST for heavy twitter schemas). + # Leaves a tiny leftover so we still have a distinct + # collateral coin per call. + echo "::group::wallet before consolidation" + sui client gas --json | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + echo "::endgroup::" + + # Target 4 coins so we have headroom for several register + # calls in parallel and across drain. Merge the two smallest + # each iteration; this levels small fragments up rather than + # leaving 1 huge + 3 small (which can't cover the 1 SUI + # budget for heavy schemas). + MAX_MERGES=40 + i=0 + while [ "$(sui client gas --json | jq 'length')" -gt 4 ] && [ "$i" -lt "$MAX_MERGES" ]; do + INFO=$(sui client gas --json) + TO_MERGE=$(echo "$INFO" | jq -r 'sort_by(.mistBalance|tonumber) | .[0].gasCoinId') + PRIMARY=$(echo "$INFO" | jq -r 'sort_by(.mistBalance|tonumber) | .[1].gasCoinId') + echo "merge[$i]: $TO_MERGE -> $PRIMARY" + sui client merge-coin \ + --primary-coin "$PRIMARY" \ + --coin-to-merge "$TO_MERGE" \ + --gas-budget 5000000 >/dev/null + i=$((i+1)) + done + + echo "::group::wallet after consolidation" + sui client gas --json | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + echo "::endgroup::" + + - name: Register offchain tools + shell: bash + run: | + set -euo pipefail + + MANIFEST="${{ steps.artifacts.outputs.manifest }}" + META_DIR="${{ steps.artifacts.outputs.meta_dir }}" + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + + # Snapshot already-registered FQNs once up front. Each + # `nexus tool register` call costs ~1-2s even for an + # already-registered FQN (it submits a tx, the Move call + # aborts with EFqnAlreadyExists, we detect it and preserve). + # Skipping the CLI call entirely for known-registered FQNs + # shortens a typical re-run from minutes to seconds. + REGISTERED_FQNS="${RUNNER_TEMP}/registered_fqns.txt" + nexus tool list --json 2>/dev/null | jq -r '.[].fqn' > "$REGISTERED_FQNS" || true + echo "::group::Already-registered FQNs on chain ($(wc -l < "$REGISTERED_FQNS"))" + cat "$REGISTERED_FQNS" + echo "::endgroup::" + + for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do + META="${META_DIR}/${TOOL}.json" + if [ ! -f "$META" ]; then + echo "::error::Meta file not found: $META" + ls -la "$META_DIR" >&2 || true + exit 1 + fi + VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST") + TOOL_HOST="${NETWORK}-${TOOL}-v${VERSION}.tools.internal" + echo "=== TOOL=$TOOL VERSION=$VERSION HOST=$TOOL_HOST ===" + + # Each tool binary's --meta emits an array of ToolMeta (one per + # FQN). The CLI's --from-meta takes a single ToolMeta on stdin, + # so iterate and pipe each entry. Each FQN has its own URL + # *path* (e.g. /i64/add/) baked into the meta — we keep that + # path and only swap the host from the in-container BIND_ADDR + # to the external internal-DNS hostname. + jq -c '.[]' "$META" | while read -r TOOL_META; do + FQN=$(echo "$TOOL_META" | jq -r '.fqn') + REG_PATH="gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json" + + # Skip the CLI call entirely if the FQN is already + # registered on chain (snapshot taken above). We still + # verify the GCS file is present so downstream + # signing-key / toolkit-config steps don't run blind. + if grep -qxF "$FQN" "$REGISTERED_FQNS"; then + if ! gsutil -q stat "$REG_PATH" 2>/dev/null; then + echo "::error::$FQN registered on-chain but no caps in GCS ($REG_PATH)" + exit 1 + fi + echo " ✓ $FQN already registered — skipping CLI call" + continue + fi + + META_URL=$(echo "$TOOL_META" | jq -r '.url') + # Strip scheme + host:port, keep the path (defaulting to /). + URL_PATH=$(echo "$META_URL" | sed -E 's|^[a-z]+://[^/]*||') + [ -z "$URL_PATH" ] && URL_PATH="/" + # Leader appends `invoke` via RFC 3986 relative-URL join: + # /i64/add/ + invoke = /i64/add/invoke ✓ + # /i64/add + invoke = /invoke ✗ + # Force a trailing slash so the join produces the right + # endpoint, matching the legacy v0.8 static-deploy URLs. + [ "${URL_PATH: -1}" != "/" ] && URL_PATH="${URL_PATH}/" + TOOL_URL="http://${TOOL_HOST}${URL_PATH}" + + # The CLI's gas pool is a single coin picked at startup + # (sui.rs:294 `with_gas(vec![one_coin])`); reusing the same + # coin across many CLI invocations drains it below the + # 100M budget and yields InsufficientGas. Re-query the + # wallet per call and pin the largest coin as gas and + # smallest (distinct) coin as collateral — fresh object + # refs every call, no chance of overlap. + GAS_INFO=$(sui client gas --json) + echo "::group::wallet coins before $FQN" + echo "$GAS_INFO" | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + echo "$GAS_INFO" | jq 'length as $n | (map(.mistBalance|tonumber) | add) as $sum | {coin_count: $n, total_mist: $sum, total_sui: ($sum/1000000000)}' + echo "::endgroup::" + GAS_BUDGET=1000000000 + # Pick any coin with balance ≥ budget — preferring the + # largest, but rotating to a different one if the previous + # call drained the prior-largest below budget. Sui rejects + # at validation (not execution) when the gas coin can't + # cover the budget reservation, so this guard saves us + # from opaque RPC errors and from staying stuck on a + # drained coin while others in the wallet still have + # plenty. + GAS_COIN=$(echo "$GAS_INFO" | jq -r --argjson b "$GAS_BUDGET" \ + 'map(select((.mistBalance|tonumber) >= $b)) + | sort_by(.mistBalance|tonumber) | last.gasCoinId // empty') + if [ -z "$GAS_COIN" ]; then + echo "::error::No coin in wallet has balance ≥ $GAS_BUDGET MIST." + echo "$GAS_INFO" | jq 'map({gasCoinId, mistBalance}) | sort_by(.mistBalance|tonumber) | reverse' + exit 1 + fi + GAS_BAL=$(echo "$GAS_INFO" | jq -r --arg g "$GAS_COIN" '.[] | select(.gasCoinId == $g) | .mistBalance') + # Collateral can be any other coin (split is 1 MIST), so + # take the smallest one not chosen as gas. + COLL_COIN=$(echo "$GAS_INFO" | jq -r --arg g "$GAS_COIN" \ + 'map(select(.gasCoinId != $g)) | sort_by(.mistBalance|tonumber) | first.gasCoinId // empty') + if [ -z "$COLL_COIN" ]; then + echo "::error::Wallet has only one coin; need ≥2 (gas + collateral)" + exit 1 + fi + COLL_BAL=$(echo "$GAS_INFO" | jq -r --arg c "$COLL_COIN" '.[] | select(.gasCoinId == $c) | .mistBalance') + echo " selected gas=$GAS_COIN ($GAS_BAL MIST) collateral=$COLL_COIN ($COLL_BAL MIST) budget=$GAS_BUDGET" + + echo "::group::CLI invocation for $FQN" + echo " url: $TOOL_URL" + echo " tool_meta: $(echo "$TOOL_META" | jq -c '{fqn, url, timeout, description: (.description|.[0:40])}')" + echo "::endgroup::" + + STDERR_LOG="${RUNNER_TEMP}/nexus-register.stderr" + set +e + # Heavy schemas (twitter get-tweet, llm chat-completion) + # need ~1 SUI of gas to cover storage; the proven value + # from the prior static deploy + # (tf-talus-nexus-v2 social-twitter.json:gasBudgetAmount). + # Pre-consolidated wallet coins (above) cover this. + OUT=$(echo "$TOOL_META" \ + | nexus tool register offchain \ + --from-meta - \ + --url "$TOOL_URL" \ + --sui-gas-coin "$GAS_COIN" \ + --collateral-coin "$COLL_COIN" \ + --sui-gas-budget "$GAS_BUDGET" \ + --invocation-cost 0 \ + --no-save \ + --json 2>"$STDERR_LOG") + RC=$? + set -e + echo "::group::CLI stderr for $FQN (rc=$RC)" + cat "$STDERR_LOG" || true + echo "::endgroup::" + echo "::group::CLI stdout for $FQN" + echo "$OUT" + echo "::endgroup::" + if [ $RC -ne 0 ]; then + echo "::error::nexus CLI exited $RC for $FQN" + exit $RC + fi + + ENTRY=$(echo "$OUT" | jq -c '.[0]') + ALREADY=$(echo "$ENTRY" | jq -r '.already_registered // false') + ERR=$(echo "$ENTRY" | jq -r '.error // empty') + + # v1.0.0 CLI flags already-registered by string-matching the + # outer entry name, but the MoveAbort surfaces from the inner + # helper `register_tool_` at instruction 14 (the + # EFqnAlreadyExists assertion in tool_registry.move:594), so + # the CLI misses it. Treat that exact signature as + # already-registered. TODO: fix upstream in nexus-sdk. + if [ "$ALREADY" != "true" ] && [ -n "$ERR" ] \ + && echo "$ERR" | grep -q 'function_name: Some(Identifier("register_tool_"))' \ + && echo "$ERR" | grep -q 'instruction: 14'; then + ALREADY="true" + fi + + if [ "$ALREADY" = "true" ]; then + # Caps must already exist in GCS from a prior run; refusing + # to proceed when they're missing keeps downstream steps + # (signing keys, toolkit-config) from running blind. + if ! gsutil -q stat "$REG_PATH" 2>/dev/null; then + echo "::error::$FQN registered on-chain but no caps in GCS ($REG_PATH)" + exit 1 + fi + echo " ✓ $FQN already registered — caps preserved" + continue + fi + + TOOL_CAP=$(echo "$ENTRY" | jq -r '.owner_cap_over_tool_id // empty') + GAS_CAP=$(echo "$ENTRY" | jq -r '.owner_cap_over_gas_id // empty') + if [ -z "$TOOL_CAP" ] || [ -z "$GAS_CAP" ]; then + echo "::error::register output missing caps for $FQN: $ENTRY" + exit 1 + fi + + # Merge with any existing record so downstream-added fields + # (signing_key_hash, tool_kid) survive re-registration. + EXISTING=$(gsutil cat "$REG_PATH" 2>/dev/null || echo '{}') + echo "$EXISTING" | jq \ + --arg fqn "$FQN" --arg t "$TOOL_CAP" --arg g "$GAS_CAP" \ + '. + {fqn: $fqn, owner_cap_over_tool: $t, owner_cap_over_gas: $g}' \ + | gsutil cp - "$REG_PATH" + echo " + $FQN registered (tool_cap=$TOOL_CAP gas_cap=$GAS_CAP)" + done + done + + - name: Register signing keys + shell: bash + run: | + set -euo pipefail + MANIFEST="${{ steps.artifacts.outputs.manifest }}" + META_DIR="${{ steps.artifacts.outputs.meta_dir }}" + PROJECT="${{ vars.GCP_PROJECT_ID }}" + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + + for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do + VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST") + VERSIONED="${TOOL}-v${VERSION}" + + KEYS=$(gcloud secrets versions access latest \ + --secret="nexus-tools-${VERSIONED}-signed-http-keys" \ + --project="$PROJECT") + KEYS_HASH=$(echo "$KEYS" | jq -c '[.keys_registry[] | {fqn, public_key_hex}]' | sha256sum | cut -c1-16) + + for FQN in $(jq -r '.[].fqn' "${META_DIR}/${TOOL}.json"); do + REG=$(gsutil cat "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json") + OWNER_CAP=$(echo "$REG" | jq -r '.owner_cap_over_tool') + REGISTERED_HASH=$(echo "$REG" | jq -r '.signing_key_hash // ""') + if [ "$REGISTERED_HASH" = "$KEYS_HASH" ]; then + echo " ✓ $FQN signing key unchanged — skipping" + continue + fi + + PRIVATE_KEY=$(echo "$KEYS" | jq -re --arg f "$FQN" '.keys_registry[] | select(.fqn == $f) | .private_key_hex') + REG_OUT=$(nexus tool auth register-key \ + --json \ + --tool-fqn "$FQN" \ + --owner-cap "$OWNER_CAP" \ + --signing-key "$PRIVATE_KEY" \ + --description "ci-managed") + KID=$(echo "$REG_OUT" | jq -r '.tool_kid') + + echo "$REG" | jq --arg h "$KEYS_HASH" --argjson kid "$KID" \ + '. + {signing_key_hash: $h, tool_kid: $kid}' \ + | gsutil cp - "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json" + echo " + $FQN registered with tool_kid=$KID" + done + done + + - name: Reconcile toolkit-config secret + shell: bash + run: | + set -euo pipefail + MANIFEST="${{ steps.artifacts.outputs.manifest }}" + META_DIR="${{ steps.artifacts.outputs.meta_dir }}" + PROJECT="${{ vars.GCP_PROJECT_ID }}" + BUCKET="${{ vars.GCP_PROJECT_ID }}-nexus-tools" + NETWORK="${{ vars.SUI_NETWORK }}" + + for TOOL in $(jq -r 'keys[]' "$MANIFEST"); do + VERSION=$(jq -r --arg t "$TOOL" '.[$t].version' "$MANIFEST") + VERSIONED="${TOOL}-v${VERSION}" + SECRET="nexus-tools-${VERSIONED}-signed-http-toolkit-config" + + gcloud secrets describe "$SECRET" --project="$PROJECT" >/dev/null 2>&1 \ + || gcloud secrets create "$SECRET" --project="$PROJECT" --replication-policy=automatic + + SKELETON=$(gcloud secrets versions access latest \ + --secret="nexus-tools-${VERSIONED}-signed-http-keys" \ + --project="$PROJECT" \ + | jq -c '.toolkit_config') + + DESIRED="$SKELETON" + for FQN in $(jq -r '.[].fqn' "${META_DIR}/${TOOL}.json"); do + REG=$(gsutil cat "gs://${BUCKET}/${NETWORK}/offchain/registration/${TOOL}/${FQN}.json") + KID=$(echo "$REG" | jq -r '.tool_kid // "MISSING"') + if [ "$KID" = "MISSING" ] || [ "$KID" = "null" ]; then + echo "::error::No tool_kid for $FQN — refusing to write toolkit-config" + exit 1 + fi + DESIRED=$(echo "$DESIRED" | jq -c --arg fqn "$FQN" --argjson kid "$KID" \ + '.signed_http.tools[$fqn].tool_kid = $kid') + done + + CURRENT=$(gcloud secrets versions access latest \ + --secret="$SECRET" --project="$PROJECT" 2>/dev/null | jq -c . 2>/dev/null || echo "") + if [ "$DESIRED" = "$CURRENT" ]; then + echo " ✓ toolkit-config for $VERSIONED unchanged" + else + echo -n "$DESIRED" | gcloud secrets versions add "$SECRET" --project="$PROJECT" --data-file=- + echo " + Wrote new toolkit-config for $VERSIONED" + fi + done diff --git a/.github/workflows/sync_docs.yml b/.github/workflows/sync_docs.yml index cc4f4ef..2c95b4d 100644 --- a/.github/workflows/sync_docs.yml +++ b/.github/workflows/sync_docs.yml @@ -16,7 +16,7 @@ jobs: with: files: | docs/** - tools/*/README.md + offchain/tools/*/README.md sync-docs: runs-on: ubuntu-latest @@ -47,13 +47,11 @@ jobs: cp -r docs/* gitbook-docs/${{ env.REPO_NAME }}/ fi - # Sync README.md files from tools/*/ - for readme in tools/*/README.md; do - if [ -f "$readme" ]; then - tool_name=$(basename $(dirname "$readme")) - mkdir -p "gitbook-docs/tools/$tool_name" - cp "$readme" "gitbook-docs/tools/$tool_name/" - fi + # Sync README.md files from offchain/tools/*/ + for readme in offchain/tools/*/README.md; do + tool_name=$(basename "$(dirname "$readme")") + mkdir -p "gitbook-docs/tools/$tool_name" + cp "$readme" "gitbook-docs/tools/$tool_name/" done # Check if any changes were made diff --git a/.gitignore b/.gitignore index f857cae..f6e8d30 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ sdk/tests/move/onchain_tool_test/Move.lock .DS_Store **/Move.lock + +# Local-only planning artifacts (brainstorming specs, plans, notes) +docs/superpowers/ diff --git a/.pre-commit/.just b/.pre-commit/.just index ef4485d..7ce8a23 100644 --- a/.pre-commit/.just +++ b/.pre-commit/.just @@ -1,4 +1,8 @@ -import '../helpers/helpers.just' +import '../offchain/helpers/helpers.just' + +# Cargo lives under offchain/; default all recipes there. Recipes that +# need a different PWD (md-lint, typos, shebang checks) `cd` explicitly. +set working-directory := '../offchain' [private] _default: @@ -227,6 +231,50 @@ executable-scripts: exit $error +# Verify every offchain/tools// follows the tool author contract: +# 1. has tools.json +# 2. has build.rs (which validates [[bin]] == crate name at compile time) +# 3. no fqn!("...@") hardcoded — must use +# fqn!(concat!("...@", env!("TOOL_FQN_VERSION"))) +# A hardcoded @N would compile fine but ship the wrong content-version +# to chain (or collide with the legacy @1 registrations), so this is the +# main thing the build system can't catch on its own. +check-tool-conventions: + #!/usr/bin/env bash + + set -euo pipefail + + cd $(git rev-parse --show-toplevel) + + error=0 + + for d in offchain/tools/*/; do + tool=$(basename "$d") + if [[ ! -f "${d}tools.json" ]]; then + echo "❌ ${d}: missing tools.json" >&2 + error=1 + fi + if [[ ! -f "${d}build.rs" ]]; then + echo "❌ ${d}: missing build.rs (needed to thread TOOL_FQN_VERSION)" >&2 + error=1 + fi + done + + # Any fqn!("...@") with a literal numeric version is a bug: + # the version must come from env!("TOOL_FQN_VERSION") via concat!(). + bad=$(grep -rEn 'fqn!\("[^"]+@[0-9]+"' offchain/tools/*/src/ 2>/dev/null || true) + if [[ -n "$bad" ]]; then + echo "❌ Hardcoded fqn! version found — use concat!(\"...@\", env!(\"TOOL_FQN_VERSION\")):" >&2 + echo "$bad" >&2 + error=1 + fi + + if [[ $error -eq 0 ]]; then + echo "✅ Tool conventions OK" >&2 + fi + exit $error + + nightly-version: #!/usr/bin/env bash diff --git a/.pre-commit/14-check-tool-conventions b/.pre-commit/14-check-tool-conventions new file mode 100755 index 0000000..5f23846 --- /dev/null +++ b/.pre-commit/14-check-tool-conventions @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +exec just pre-commit::check-tool-conventions diff --git a/README.md b/README.md index 67c390c..9fbdf10 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,203 @@ [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://github.com/Talus-Network/nexus-tools/blob/main/LICENSE) [![Actions](https://img.shields.io/badge/GitHub_Actions-Active-brightgreen)](https://github.com/Talus-Network/nexus-tools/actions) -This is a collection of [tools] that the Talus core team maintains. +A collection of [tools] that the Talus core team maintains. ## Development -We use [just][just-repo], a straightforward command runner similar to `make`. +We use [just][just-repo] as a command runner. List tasks with `just --list`. -To explore the available tasks, run `just --list`. +## Layout + +~~~text +nexus-tools/ +├── offchain/ # Rust workspace + tools (cd here before cargo) +│ ├── Cargo.toml +│ ├── tools// +│ │ ├── Cargo.toml +│ │ ├── tools.json # per-tool deploy config +│ │ ├── build.rs # validates bin/tool name, threads TOOL_FQN_VERSION +│ │ └── src/... +│ └── Dockerfile # shared, parameterized by PACKAGE/BINARY/TOOL_FQN_VERSION +├── onchain/ # reserved for future Move tools +└── .github/ + ├── actions/ # composite actions (install-sui, install-nexus-cli, gcp-auth-*) + └── workflows/ # CI: discover → deploy → prepare → register → trigger-tf-apply +~~~ + +All cargo commands run from `offchain/`: + +~~~bash +cd offchain +cargo build --workspace +cargo test --workspace +~~~ + +## Adding a new tool + +A "tool" is a single Rust crate under `offchain/tools//` with a few +conventions the CI pipeline relies on: + +1. **`tools.json`** — declares the deploy shape. The crate's name is the + key under `offchain/tools/`. Minimal example: + + ~~~json + { + "environment": { "RUST_LOG": "info" }, + "resources": { "cpu": "1", "memory": "512Mi" }, + "signed_http": { "enabled": true } + } + ~~~ + +1. **`build.rs`** — must compile-time validate that the crate name + matches the binary and emit `TOOL_FQN_VERSION` from the Docker + build-arg (defaults to `"1"` for local builds). Copy from an + existing tool like `offchain/tools/math/build.rs`. + +1. **`[[bin]]` in `Cargo.toml`** — binary name must equal the crate + name (the build.rs assertion enforces this). + +1. **`fqn!()` source threading** — every FQN literal in your tool must + use `concat!()` + `env!("TOOL_FQN_VERSION")` so the content version + flows from the build args into the registered FQN: + + ~~~rust + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.math.i64.add@", + env!("TOOL_FQN_VERSION") + )) + } + ~~~ + +1. **Optional per-route URL path** — the toolkit binary's `--meta` + output reports a per-FQN URL like `http://localhost/i64/add`. The + register step preserves that path (forcing a trailing slash) when + composing the on-chain URL, so the leader can reach the right + endpoint. + +Once those four pieces are in place, the CI pipeline discovers the +tool automatically by globbing `offchain/tools/*/tools.json`. + +## Branch & tag model + +Two roles for branches, one for tags: + +- **`main`** — dev integration. Tip of `main` is the source of truth + for what code exists. Pushes to `main` build + push images to GCR but + never touch the deployer wallet and never register FQNs on chain. +- **`iterate/testnet/` / `iterate/mainnet/`** — short- + lived branches for iterating on the deploy pipeline itself. Push + fires the full chain. Delete the branch once the rollout settles. +- **`testnet-*` / `mainnet-*` tags** — the canonical deploy path. Tag + a commit on `main` (for testnet) or any prior testnet-deployed + commit (for mainnet), push the tag, and CI runs the full chain + against the corresponding env. Tags are immutable, so each + deployment is attributable to an exact ref — no merge-strategy + gotchas, no long-lived deploy branches to keep clean. + +| Trigger | Matrix | Build | Push images | Chain ops | TF apply | +| --- | --- | --- | --- | --- | --- | +| PR (any base) | changed | ✓ | dry-run | — | — | +| Push to `main` | changed | ✓ | ✓ | — | — | +| Push to `iterate/testnet/**` / `iterate/mainnet/**` | all | ✓ | ✓ | ✓ (env) | ✓ | +| Push tag `testnet-*` / `mainnet-*` | all | ✓ | ✓ | ✓ (tag prefix) | ✓ | +| `workflow_dispatch` `target-env=testnet\|mainnet` | all | ✓ | ✓ | ✓ (chosen) | ✓ | +| `workflow_dispatch` `pr-number=` | all | ✓ | ✓ | ✓ (PR's base) | ✓ | + +## Lifecycle: PR → main → deploy + +1. **Open a PR with base `main`.** CI runs the dry-run path: discover + the changed-tool matrix, build each image, but don't push to the + registry and don't touch the chain. Reviewers verify the build is + clean and tests pass. +1. **Merge to `main`.** Same as the PR run, but images now get pushed + to GCR (`gcr.io//nexus-tools/:sha-<7>`). The + image is "available" but no Cloud Run service points at it yet and + no FQN is registered. +1. **Promote to a deploy env.** Pick one of: + - **Tag deploy (canonical)** — tag a commit and push the tag: + + ~~~bash + git tag testnet-v0.1.0 + git push origin testnet-v0.1.0 + ~~~ + + Tag prefix selects the env (`testnet-*` → testnet, `mainnet-*` + → mainnet). The full chain runs against that exact commit, and + the tag is an immutable record of what was deployed. Best for + anything you'd want to roll back to or audit later. + - **Iterate branch** — push to `iterate/testnet/` (or + `iterate/mainnet/`). Fires the same pipeline. Best for + fast iteration on the deploy itself (debugging the register + step, tf-nexus-tools side, etc.) where you don't want a tag for + every attempt. + - **Manual one-off** — Actions → CI → "Run workflow". Pick + `target-env=testnet|mainnet`, or pass `pr-number=` to deploy a + specific PR's head against the PR's base env. Best for ad-hoc + deploys without cutting a branch or tag. + + Convention for mainnet promotions: tag should sit on a commit that + has already been deployed to testnet (i.e. is reachable from a + prior `testnet-*` tag). Not enforced in CI yet — keep it as a + review discipline. + +After the chosen trigger fires, the full pipeline runs: discover → +deploy → prepare → register → trigger-tf-apply. `ci-gate` only goes +green if the dispatched `tf-nexus-tools` terraform run also succeeded, +so a green pipeline means Cloud Run + ALB + DNS are reconciled and the +FQN URLs on chain point at the right endpoints. + +## What "full pipeline" does + +1. **discover** — globs `offchain/tools/*/tools.json` and computes a + per-tool content version from `git rev-parse HEAD:offchain/tools//` + (first 8 hex of sha256 → u32). +1. **deploy** — builds per-tool Docker images and pushes to + `gcr.io//nexus-tools/:sha-<7>`. Build and auth + are split so token expiry can't kill a slow build mid-push. +1. **prepare** — runs each container's `--meta` to extract the + ToolMeta JSON, renders a Cloud Run config blob to + `gs:////offchain/tools/-v.json`, + and generates a signed-HTTP keys secret in Secret Manager. The + `nexus_contracts_tag` is baked from `vars.NEXUS_TAG` into each blob + so it stays pinned to whichever Nexus version was current at + prepare time. +1. **register** — for each FQN: + - skips the CLI call if already on chain (snapshot from `nexus tool list --json`), + - otherwise pipes the ToolMeta to `nexus tool register offchain --from-meta -`, + - persists `owner_cap_over_tool` + `owner_cap_over_gas` to + `gs:////offchain/registration//.json`, + - registers signing keys (`nexus tool auth register-key`), + - reconciles the per-tool `toolkit-config` Secret Manager secret. + Pre-step: consolidates the deployer wallet's coins so a single coin + can cover the 1 SUI per-tx budget that heavy schemas require. +1. **trigger-tf-apply** — dispatches `Talus-Network/tf-nexus-tools`'s + `terraform.yml` (via `the-actions-org/workflow-dispatch`, which + works with fine-grained PATs). Terraform materializes the Cloud + Run services, internal ALB, and DNS records based on the per-tool + blobs in GCS. ci-gate inherits the dispatched run's conclusion, so + register-then-apply is one atomic gate. + +## Operational notes + +- **Deployer wallet** — `SUI_DEPLOYER_PK` (env-scoped repo secret) is a + `suiprivkey...` bech32 string. The register step imports it into the + sui keychain at the start of the job. The same secret will eventually + drive on-chain Move publishes for `onchain/` tools. +- **NEXUS_TAG** — env variable (testnet/mainnet). Each tool's deploy + pins to whichever value was current at prepare time; later bumps to + `NEXUS_TAG` only affect *new* deploys. +- **Stale Cloud Run services** — terraform iterates over whatever + config blobs are in `gs://...offchain/tools/`. If a stale blob from + a prior iteration is left over, terraform will try to create a + service for it. Either re-prepare to refresh the blob or + `gsutil rm` the stale one. +- **Costs** — internal ALB stack (URL map + proxy + per-region + forwarding rules) is independent from `tf-talus-nexus-v2`. Future + work may consolidate the two; for now they're kept separate so the + static legacy deploy and the dynamic pipeline can't influence each + other. diff --git a/codecov.yml b/codecov.yml index 92f6896..8557a58 100644 --- a/codecov.yml +++ b/codecov.yml @@ -23,14 +23,19 @@ coverage: threshold: 10% # Allow 10% drop for edge cases base: auto if_ci_failed: error - informational: false + # Report but don't block. Project-level check still gates the + # PR on absolute coverage drift; this one was noisy on patches + # that only touch trivial code (e.g. the fqn!() concat rewrite + # where every tool's fn fqn() body changed but is still a + # one-line constant return). + informational: true component_management: individual_components: - component_id: tools name: Tools paths: - - tools/** + - offchain/tools/** comment: layout: "header, diff, flags, components, files, footer" @@ -53,7 +58,7 @@ ignore: flags: unittests: paths: - - tools/ + - offchain/tools/ carryforward: true github_checks: diff --git a/justfile b/justfile index af0071c..51c983d 100644 --- a/justfile +++ b/justfile @@ -5,14 +5,17 @@ # # -# Commands concerning native Nexus Tools -mod tools 'tools/.just' +# Commands concerning native Nexus Tools (offchain workspace) +mod tools 'offchain/tools/.just' -# Pre-commit hooks +# Example invocations for the memory-memwal Tools bundle +mod memwal 'offchain/tools/memory-memwal/.just' + +# Pre-commit hooks (still at repo root — they wrap git commit) mod pre-commit '.pre-commit/.just' -# Helpers -mod helpers 'helpers/helpers.just' +# Helpers (lives under the workspace) +mod helpers 'offchain/helpers/helpers.just' [private] _default: diff --git a/Cargo.lock b/offchain/Cargo.lock similarity index 81% rename from Cargo.lock rename to offchain/Cargo.lock index 1ef5f97..0f6e73a 100644 --- a/Cargo.lock +++ b/offchain/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -25,6 +31,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -36,9 +63,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -57,9 +84,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -86,9 +113,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "assert-json-diff" @@ -100,6 +127,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-openai" version = "0.27.2" @@ -113,12 +152,12 @@ dependencies = [ "eventsource-stream", "futures", "rand 0.8.5", - "reqwest", + "reqwest 0.12.25", "reqwest-eventsource", "secrecy", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", @@ -148,6 +187,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backoff" version = "0.4.0" @@ -247,7 +308,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -259,6 +320,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bnum" version = "0.13.0" @@ -271,6 +341,27 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.5.1" @@ -345,9 +436,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -357,6 +448,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -372,12 +472,46 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation" version = "0.9.4" @@ -413,6 +547,24 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -435,6 +587,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -442,9 +603,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -509,7 +670,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -560,12 +721,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -577,6 +749,18 @@ dependencies = [ "syn", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -590,7 +774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ "der", - "digest", + "digest 0.10.7", "elliptic-curve", "rfc6979", "signature", @@ -616,7 +800,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -635,7 +819,7 @@ checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -679,9 +863,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "0.1.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -689,9 +873,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -735,12 +919,13 @@ dependencies = [ "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", + "toml", ] [[package]] @@ -754,6 +939,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -788,6 +984,16 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fluent-uri" version = "0.3.2" @@ -799,6 +1005,17 @@ dependencies = [ "serde", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fmt-cmp" version = "0.1.2" @@ -811,6 +1028,18 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -845,6 +1074,12 @@ dependencies = [ "num", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -982,11 +1217,24 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "glob" version = "0.3.3" @@ -1042,11 +1290,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "headers" @@ -1090,7 +1352,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1109,17 +1371,18 @@ name = "http" version = "1.0.0" dependencies = [ "backon", - "base64 0.21.7", - "jsonschema", + "base64 0.22.1", + "jsonschema 0.46.5", "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", + "toml", "url", ] @@ -1179,6 +1442,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.32" @@ -1291,7 +1563,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.1", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -1404,6 +1676,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -1438,7 +1716,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -1509,9 +1789,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "log", @@ -1522,11 +1802,60 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ "quote", "syn", ] @@ -1543,10 +1872,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1561,18 +1892,47 @@ dependencies = [ "base64 0.22.1", "bytecount", "email_address", - "fancy-regex", + "fancy-regex 0.14.0", "fraction", "idna", "itoa", "num-cmp", "once_cell", "percent-encoding", - "referencing", + "referencing 0.28.3", + "regex-syntax", + "reqwest 0.12.25", + "serde", + "serde_json", + "uuid-simd", +] + +[[package]] +name = "jsonschema" +version = "0.46.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a5fe5206f06e589caf25e79fc05ccdf91fca745685fe9fe1a13bbdfb479a631" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex 0.18.0", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing 0.46.5", + "regex", "regex-syntax", - "reqwest", + "reqwest 0.13.3", + "rustls", "serde", "serde_json", + "unicode-general-category", "uuid-simd", ] @@ -1585,7 +1945,7 @@ dependencies = [ "cfg-if", "ecdsa", "elliptic-curve", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -1637,17 +1997,23 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.178" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -1662,13 +2028,13 @@ dependencies = [ "anyhow", "async-openai", "chrono", - "jsonschema", + "jsonschema 0.46.5", "log", "mockito", "nexus-sdk", "nexus-toolkit", "portpicker", - "reqwest", + "reqwest 0.12.25", "rstest", "schemars", "serde", @@ -1677,6 +2043,7 @@ dependencies = [ "strum", "strum_macros", "tokio", + "toml", ] [[package]] @@ -1708,14 +2075,54 @@ dependencies = [ "nexus-toolkit", "schemars", "serde", + "serde_json", "tokio", + "toml", ] [[package]] name = "memchr" version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" + +[[package]] +name = "memory-memwal" +version = "1.0.0" +dependencies = [ + "anyhow", + "dotenvy", + "ed25519-dalek", + "env_logger", + "hex", + "log", + "mockito", + "nexus-sdk", + "nexus-toolkit", + "reqwest 0.12.25", + "schemars", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "toml", + "url", + "uuid", + "zeroize", +] + +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" [[package]] name = "mime" @@ -1735,10 +2142,11 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.13.0" +version = "2.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0adbe6e92a6ce0fd6c4aac593fdfd3e3950b0f61b1a63aa9731eb6fd85776fa3" +checksum = "2929e494b2280e1e18959bb2e121da03347ae896896fdfaceaab43c88a02803f" dependencies = [ + "memo-map", "serde", ] @@ -1748,11 +2156,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "log", @@ -1762,9 +2180,9 @@ dependencies = [ [[package]] name = "mockito" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0603425789b4a70fcc4ac4f5a46a566c116ee3e2a6b768dc623f7719c611de" +checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" dependencies = [ "assert-json-diff", "bytes", @@ -1805,17 +2223,17 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.2.1", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework", "security-framework-sys", "tempfile", ] @@ -1834,7 +2252,7 @@ dependencies = [ "futures", "futures-util", "hex", - "jsonschema", + "jsonschema 0.28.3", "lazy-regex", "lazy_static", "petgraph", @@ -1842,15 +2260,15 @@ dependencies = [ "prost-types", "rand 0.8.5", "regex", - "reqwest", + "reqwest 0.12.25", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sui-crypto", "sui-rpc", "sui-sdk-types", "sui-transaction-builder", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-util", "toml", @@ -1872,7 +2290,7 @@ dependencies = [ "log", "nexus-sdk", "notify", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", @@ -2018,7 +2436,7 @@ checksum = "498447ee9c67221e1a558d5d49f63594c6ccaa2cdd437abec55c92e8c61887bc" dependencies = [ "base64 0.22.1", "cfg-if", - "digest", + "digest 0.10.7", "fmt-cmp", "hmac", "oauth-credentials", @@ -2055,15 +2473,14 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags 2.10.0", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -2085,11 +2502,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -2112,7 +2535,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -2145,12 +2568,13 @@ dependencies = [ "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", + "toml", ] [[package]] @@ -2274,6 +2698,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "primeorder" version = "0.13.6" @@ -2336,7 +2770,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -2404,8 +2838,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.1", - "thiserror 2.0.17", + "socket2 0.6.3", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -2426,7 +2860,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -2441,7 +2875,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.1", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -2461,6 +2895,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -2556,17 +2996,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0dcb5ab28989ad7c91eb1b9531a37a1a137cc69a0499aee4117cae4a107c464" dependencies = [ "ahash", - "fluent-uri", + "fluent-uri 0.3.2", "once_cell", "percent-encoding", "serde_json", ] +[[package]] +name = "referencing" +version = "0.46.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4e17ef386c5383591d07623d3de49cbc601156e7582973e6db98d66a57de2" +dependencies = [ + "ahash", + "fluent-uri 0.4.1", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -2576,9 +3033,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -2647,6 +3104,45 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.12", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "reqwest-eventsource" version = "0.6.0" @@ -2659,7 +3155,7 @@ dependencies = [ "mime", "nom", "pin-project-lite", - "reqwest", + "reqwest 0.12.25", "thiserror 1.0.69", ] @@ -2699,21 +3195,20 @@ dependencies = [ [[package]] name = "rstest" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", "rstest_macros", - "rustc_version", ] [[package]] name = "rstest_macros" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", @@ -2744,9 +3239,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags 2.10.0", "errno", @@ -2761,6 +3256,7 @@ version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -2776,10 +3272,10 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ - "openssl-probe", + "openssl-probe 0.1.6", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -2792,12 +3288,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -2844,9 +3368,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -2857,9 +3381,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" dependencies = [ "proc-macro2", "quote", @@ -2909,19 +3433,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.10.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -2994,15 +3505,15 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -3053,11 +3564,12 @@ dependencies = [ [[package]] name = "serial_test" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b258109f244e1d6891bf1053a55d63a5cd4f8f4c30cf9a1280989f80e7a1fa9" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" dependencies = [ - "futures", + "futures-executor", + "futures-util", "log", "once_cell", "parking_lot", @@ -3067,9 +3579,9 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "3.2.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" dependencies = [ "proc-macro2", "quote", @@ -3083,8 +3595,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] @@ -3094,8 +3606,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3119,10 +3642,32 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "similar" version = "2.7.0" @@ -3151,12 +3696,13 @@ dependencies = [ "nexus-sdk", "nexus-toolkit", "oauth1-request", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", + "toml", ] [[package]] @@ -3171,12 +3717,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3209,15 +3755,15 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", @@ -3304,7 +3850,7 @@ dependencies = [ "serde_json", "serde_with", "sui-sdk-types", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -3367,12 +3913,12 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -3389,6 +3935,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "toml", ] [[package]] @@ -3402,11 +3949,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -3422,9 +3969,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -3477,9 +4024,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3487,16 +4034,16 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3695,13 +4242,18 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ + "async-compression", "bitflags 2.10.0", "bytes", + "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", + "http-body-util", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -3778,9 +4330,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicase" @@ -3788,12 +4340,24 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -3802,9 +4366,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -3832,10 +4396,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.19.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ + "getrandom 0.4.2", "js-sys", "wasm-bindgen", ] @@ -3884,16 +4449,17 @@ name = "walrus" version = "1.0.0" dependencies = [ "anyhow", - "jsonschema", + "jsonschema 0.46.5", "mockito", "nexus-sdk", "nexus-toolkit", - "reqwest", + "reqwest 0.12.25", "schemars", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", + "toml", ] [[package]] @@ -3946,14 +4512,23 @@ version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.46.0", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -3964,22 +4539,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3987,9 +4559,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -4000,13 +4572,35 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -4020,11 +4614,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" dependencies = [ "js-sys", "wasm-bindgen", @@ -4040,6 +4646,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.4" @@ -4308,6 +4923,94 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.2" @@ -4431,6 +5134,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/offchain/Cargo.toml similarity index 59% rename from Cargo.toml rename to offchain/Cargo.toml index 7f6c092..576d6f0 100644 --- a/Cargo.toml +++ b/offchain/Cargo.toml @@ -14,25 +14,34 @@ keywords = ["nexus", "talus", "tools", "walrus"] categories = [] [workspace.dependencies] -anyhow = "1.0.97" -base64 = "0.21" -chrono = { version = "0.4", features = ["serde"] } -jsonschema = "0.28.3" -log = "^0.4.26" -minijinja = "2.8.0" -mockito = "1.7.0" +anyhow = "1.0.102" +base64 = "0.22.1" +bytes = "1.11.1" +dotenvy = "0.15.7" +ed25519-dalek = "2.2.0" +env_logger = "0.11.10" +hex = "0.4.3" +sha2 = "0.11.0" +chrono = { version = "0.4.44", features = ["serde"] } +jsonschema = "0.46.5" +log = "0.4.29" +minijinja = "2.20.0" +mockito = "1.7.2" portpicker = "0.1.1" -reqwest = { version = "0.12.25", features = ["json", "stream"] } -rstest = "0.25.0" -schemars = "1.0.0" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -serial_test = "3.2.0" -strum = "0.27" -strum_macros = "0.27" -thiserror = "2.0.12" -tokio = { version = "1.49.0", features = ["full"] } -warp = "0.3.7" +reqwest = { version = "0.12.25", features = ["json", "stream", "gzip", "brotli"] } +rstest = "0.26.1" +schemars = "1.2.1" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.149" +serial_test = "3.4.0" +strum = "0.28.0" +strum_macros = "0.28.0" +thiserror = "2.0.18" +tokio = { version = "1.52.3", features = ["full"] } +url = "2.5.8" +uuid = { version = "1.23.1", features = ["v4"] } +warp = "0.4.3" +zeroize = { version = "1", features = ["zeroize_derive"] } # === Nexus deps === [workspace.dependencies.nexus-sdk] diff --git a/offchain/Dockerfile b/offchain/Dockerfile new file mode 100644 index 0000000..499da87 --- /dev/null +++ b/offchain/Dockerfile @@ -0,0 +1,77 @@ +# syntax=docker/dockerfile:1 +# Shared Dockerfile for all offchain Nexus tools. +# +# Builds a single tool binary identified by BINARY and TOOL_DIR build args +# (TOOL_DIR is the tool's path relative to the build context, e.g. +# "offchain/tools/http"), embeds TOOL_FQN_VERSION (computed by CI from the +# tool's subtree hash) so the tool's FQN strings carry the content-addressed +# version. +# +# Building via --manifest-path scopes cargo to the workspace member's +# Cargo.toml, sidestepping ambiguity when a tool crate's name collides with +# a transitive crates.io dep of the same name (e.g. our `http` tool vs the +# `http` crate pulled in by reqwest). + +FROM rust:1.93.1-slim-bookworm AS builder + +ARG BINARY +ARG TOOL_DIR +ARG TOOL_FQN_VERSION=1 + +# Surfaced as a cargo env var via build.rs -> env!("TOOL_FQN_VERSION"). +ENV TOOL_FQN_VERSION=${TOOL_FQN_VERSION} + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Copy the entire offchain workspace. Cargo's incremental build + the +# buildkit cache mounts keep this fast. +COPY offchain/ ./offchain/ + +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/app/offchain/target,sharing=locked \ + set -e && \ + cargo build --locked --profile release \ + --manifest-path "${TOOL_DIR}/Cargo.toml" \ + --bin "${BINARY}" && \ + cp "offchain/target/release/${BINARY}" "/app/${BINARY}" + +# ── Collect runtime libs for the distroless stage ───────────────────── +FROM debian:bookworm-slim AS lib-collector + +ARG BINARY +ARG TARGETARCH + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libssl3 ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/${BINARY} /tmp/binary + +RUN set -e; \ + case "${TARGETARCH}" in \ + "amd64") LIB_DIR="/lib/x86_64-linux-gnu"; LINKER="/lib64/ld-linux-x86-64.so.2" ;; \ + "arm64") LIB_DIR="/lib/aarch64-linux-gnu"; LINKER="/lib/ld-linux-aarch64.so.1" ;; \ + *) echo "Unsupported TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + mkdir -p "/collected${LIB_DIR}" /collected/lib64 && \ + ldd /tmp/binary | awk '/=>/ { print $3 }' | while read -r lib; do \ + if [ -n "$lib" ] && [ -f "$lib" ]; then \ + cp -L "$lib" "/collected${LIB_DIR}/"; \ + fi \ + done && \ + cp -L "${LINKER}" "/collected${LINKER}" + +# ── Final distroless image ─────────────────────────────────────────── +FROM gcr.io/distroless/cc-debian12 + +ARG BINARY + +COPY --from=builder /app/${BINARY} /usr/local/bin/${BINARY} +COPY --from=lib-collector /collected/ / + +ENV PORT=8080 +EXPOSE 8080 diff --git a/deny.toml b/offchain/deny.toml similarity index 100% rename from deny.toml rename to offchain/deny.toml diff --git a/helpers/helpers.just b/offchain/helpers/helpers.just similarity index 98% rename from helpers/helpers.just rename to offchain/helpers/helpers.just index a81e61e..bd5ae07 100644 --- a/helpers/helpers.just +++ b/offchain/helpers/helpers.just @@ -119,4 +119,4 @@ get-nightly-version: #!/usr/bin/env bash # This file should contain the version of the nightly toolchain to use. - cat ../.nightly-version + cat ../../.nightly-version diff --git a/helpers/npm-install-g.sh b/offchain/helpers/npm-install-g.sh similarity index 55% rename from helpers/npm-install-g.sh rename to offchain/helpers/npm-install-g.sh index 5bc0f27..db8c74b 100755 --- a/helpers/npm-install-g.sh +++ b/offchain/helpers/npm-install-g.sh @@ -2,10 +2,11 @@ set -euo pipefail -# Go to the root of the git repository -cd $(git rev-parse --show-toplevel) +# Resolve the directory containing this script so the npm-install-g.txt +# lookup works regardless of the caller's PWD. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Install NPM stuff from helpers/npm-install-g.txt +# Install NPM stuff from the sibling helpers/npm-install-g.txt while read -r tool_at_version; do # Split the tool name and version tool=$(echo "$tool_at_version" | cut -d'@' -f1) @@ -16,4 +17,4 @@ while read -r tool_at_version; do else echo "Tool already installed: $tool_at_version" fi -done < helpers/npm-install-g.txt +done < "$SCRIPT_DIR/npm-install-g.txt" diff --git a/helpers/npm-install-g.txt b/offchain/helpers/npm-install-g.txt similarity index 100% rename from helpers/npm-install-g.txt rename to offchain/helpers/npm-install-g.txt diff --git a/rust-toolchain.toml b/offchain/rust-toolchain.toml similarity index 100% rename from rust-toolchain.toml rename to offchain/rust-toolchain.toml diff --git a/rustfmt.toml b/offchain/rustfmt.toml similarity index 100% rename from rustfmt.toml rename to offchain/rustfmt.toml diff --git a/tools/.just b/offchain/tools/.just similarity index 100% rename from tools/.just rename to offchain/tools/.just diff --git a/tools/exchanges-coinbase/Cargo.toml b/offchain/tools/exchanges-coinbase/Cargo.toml similarity index 84% rename from tools/exchanges-coinbase/Cargo.toml rename to offchain/tools/exchanges-coinbase/Cargo.toml index ab93616..5915d89 100644 --- a/tools/exchanges-coinbase/Cargo.toml +++ b/offchain/tools/exchanges-coinbase/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "exchanges-coinbase" +path = "src/main.rs" + [dependencies] chrono.workspace = true thiserror.workspace = true @@ -27,3 +31,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/exchanges-coinbase/README.md b/offchain/tools/exchanges-coinbase/README.md similarity index 100% rename from tools/exchanges-coinbase/README.md rename to offchain/tools/exchanges-coinbase/README.md diff --git a/offchain/tools/exchanges-coinbase/build.rs b/offchain/tools/exchanges-coinbase/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/exchanges-coinbase/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/exchanges-coinbase/src/coinbase_client.rs b/offchain/tools/exchanges-coinbase/src/coinbase_client.rs similarity index 100% rename from tools/exchanges-coinbase/src/coinbase_client.rs rename to offchain/tools/exchanges-coinbase/src/coinbase_client.rs diff --git a/tools/exchanges-coinbase/src/error.rs b/offchain/tools/exchanges-coinbase/src/error.rs similarity index 100% rename from tools/exchanges-coinbase/src/error.rs rename to offchain/tools/exchanges-coinbase/src/error.rs diff --git a/tools/exchanges-coinbase/src/main.rs b/offchain/tools/exchanges-coinbase/src/main.rs similarity index 100% rename from tools/exchanges-coinbase/src/main.rs rename to offchain/tools/exchanges-coinbase/src/main.rs diff --git a/tools/exchanges-coinbase/src/tools/get_product_stats.rs b/offchain/tools/exchanges-coinbase/src/tools/get_product_stats.rs similarity index 99% rename from tools/exchanges-coinbase/src/tools/get_product_stats.rs rename to offchain/tools/exchanges-coinbase/src/tools/get_product_stats.rs index 903cfcf..a8f6558 100644 --- a/tools/exchanges-coinbase/src/tools/get_product_stats.rs +++ b/offchain/tools/exchanges-coinbase/src/tools/get_product_stats.rs @@ -81,7 +81,10 @@ impl NexusTool for GetProductStats { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.exchanges.coinbase.get-product-stats@1") + fqn!(concat!( + "xyz.taluslabs.exchanges.coinbase.get-product-stats@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/exchanges-coinbase/src/tools/get_product_ticker.rs b/offchain/tools/exchanges-coinbase/src/tools/get_product_ticker.rs similarity index 99% rename from tools/exchanges-coinbase/src/tools/get_product_ticker.rs rename to offchain/tools/exchanges-coinbase/src/tools/get_product_ticker.rs index 54b1729..b018dfc 100644 --- a/tools/exchanges-coinbase/src/tools/get_product_ticker.rs +++ b/offchain/tools/exchanges-coinbase/src/tools/get_product_ticker.rs @@ -75,7 +75,10 @@ impl NexusTool for GetProductTicker { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.exchanges.coinbase.get-product-ticker@1") + fqn!(concat!( + "xyz.taluslabs.exchanges.coinbase.get-product-ticker@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/exchanges-coinbase/src/tools/get_spot_price.rs b/offchain/tools/exchanges-coinbase/src/tools/get_spot_price.rs similarity index 99% rename from tools/exchanges-coinbase/src/tools/get_spot_price.rs rename to offchain/tools/exchanges-coinbase/src/tools/get_spot_price.rs index f5d2383..796236e 100644 --- a/tools/exchanges-coinbase/src/tools/get_spot_price.rs +++ b/offchain/tools/exchanges-coinbase/src/tools/get_spot_price.rs @@ -83,7 +83,10 @@ impl NexusTool for GetSpotPrice { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.exchanges.coinbase.get-spot-price@1") + fqn!(concat!( + "xyz.taluslabs.exchanges.coinbase.get-spot-price@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/exchanges-coinbase/src/tools/mod.rs b/offchain/tools/exchanges-coinbase/src/tools/mod.rs similarity index 100% rename from tools/exchanges-coinbase/src/tools/mod.rs rename to offchain/tools/exchanges-coinbase/src/tools/mod.rs diff --git a/tools/exchanges-coinbase/src/tools/models.rs b/offchain/tools/exchanges-coinbase/src/tools/models.rs similarity index 100% rename from tools/exchanges-coinbase/src/tools/models.rs rename to offchain/tools/exchanges-coinbase/src/tools/models.rs diff --git a/offchain/tools/exchanges-coinbase/tools.json b/offchain/tools/exchanges-coinbase/tools.json new file mode 100644 index 0000000..845029b --- /dev/null +++ b/offchain/tools/exchanges-coinbase/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "exchanges-coinbase", + "command": "exchanges-coinbase", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/http/Cargo.toml b/offchain/tools/http/Cargo.toml similarity index 86% rename from tools/http/Cargo.toml rename to offchain/tools/http/Cargo.toml index 62a4645..b4154f7 100644 --- a/tools/http/Cargo.toml +++ b/offchain/tools/http/Cargo.toml @@ -10,6 +10,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "http" +path = "src/main.rs" + [dependencies] backon = "0.4" base64.workspace = true @@ -28,3 +32,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/http/README.md b/offchain/tools/http/README.md similarity index 100% rename from tools/http/README.md rename to offchain/tools/http/README.md diff --git a/offchain/tools/http/build.rs b/offchain/tools/http/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/http/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/http/src/errors.rs b/offchain/tools/http/src/errors.rs similarity index 100% rename from tools/http/src/errors.rs rename to offchain/tools/http/src/errors.rs diff --git a/tools/http/src/http.rs b/offchain/tools/http/src/http.rs similarity index 99% rename from tools/http/src/http.rs rename to offchain/tools/http/src/http.rs index a7037a9..3fb60a3 100644 --- a/tools/http/src/http.rs +++ b/offchain/tools/http/src/http.rs @@ -202,7 +202,10 @@ impl NexusTool for Http { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.http.generic@1") + fqn!(concat!( + "xyz.taluslabs.http.generic@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/http/src/http_client.rs b/offchain/tools/http/src/http_client.rs similarity index 100% rename from tools/http/src/http_client.rs rename to offchain/tools/http/src/http_client.rs diff --git a/tools/http/src/main.rs b/offchain/tools/http/src/main.rs similarity index 100% rename from tools/http/src/main.rs rename to offchain/tools/http/src/main.rs diff --git a/tools/http/src/models.rs b/offchain/tools/http/src/models.rs similarity index 100% rename from tools/http/src/models.rs rename to offchain/tools/http/src/models.rs diff --git a/tools/http/src/utils.rs b/offchain/tools/http/src/utils.rs similarity index 100% rename from tools/http/src/utils.rs rename to offchain/tools/http/src/utils.rs diff --git a/offchain/tools/http/tools.json b/offchain/tools/http/tools.json new file mode 100644 index 0000000..9be46f2 --- /dev/null +++ b/offchain/tools/http/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "http", + "command": "http", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/llm-openai-chat-completion/Cargo.toml b/offchain/tools/llm-openai-chat-completion/Cargo.toml similarity index 86% rename from tools/llm-openai-chat-completion/Cargo.toml rename to offchain/tools/llm-openai-chat-completion/Cargo.toml index 8e1ae5d..68718f0 100644 --- a/tools/llm-openai-chat-completion/Cargo.toml +++ b/offchain/tools/llm-openai-chat-completion/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "llm-openai-chat-completion" +path = "src/main.rs" + [dependencies] anyhow.workspace = true async-openai = "0.27" @@ -34,3 +38,7 @@ mockito.workspace = true portpicker.workspace = true rstest.workspace = true serial_test.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/llm-openai-chat-completion/README.md b/offchain/tools/llm-openai-chat-completion/README.md similarity index 100% rename from tools/llm-openai-chat-completion/README.md rename to offchain/tools/llm-openai-chat-completion/README.md diff --git a/offchain/tools/llm-openai-chat-completion/build.rs b/offchain/tools/llm-openai-chat-completion/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/llm-openai-chat-completion/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/llm-openai-chat-completion/src/main.rs b/offchain/tools/llm-openai-chat-completion/src/main.rs similarity index 99% rename from tools/llm-openai-chat-completion/src/main.rs rename to offchain/tools/llm-openai-chat-completion/src/main.rs index ebaefb9..5e40c4c 100644 --- a/tools/llm-openai-chat-completion/src/main.rs +++ b/offchain/tools/llm-openai-chat-completion/src/main.rs @@ -265,7 +265,10 @@ impl NexusTool for OpenaiChatCompletion { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.llm.openai.chat-completion@1") + fqn!(concat!( + "xyz.taluslabs.llm.openai.chat-completion@", + env!("TOOL_FQN_VERSION") + )) } fn timeout() -> std::time::Duration { diff --git a/tools/llm-openai-chat-completion/src/status.rs b/offchain/tools/llm-openai-chat-completion/src/status.rs similarity index 100% rename from tools/llm-openai-chat-completion/src/status.rs rename to offchain/tools/llm-openai-chat-completion/src/status.rs diff --git a/offchain/tools/llm-openai-chat-completion/tools.json b/offchain/tools/llm-openai-chat-completion/tools.json new file mode 100644 index 0000000..0252247 --- /dev/null +++ b/offchain/tools/llm-openai-chat-completion/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "llm-openai-chat-completion", + "command": "llm-openai-chat-completion", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/math/Cargo.toml b/offchain/tools/math/Cargo.toml similarity index 81% rename from tools/math/Cargo.toml rename to offchain/tools/math/Cargo.toml index 22890c6..69e5cfd 100644 --- a/tools/math/Cargo.toml +++ b/offchain/tools/math/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "math" +path = "src/main.rs" + [dependencies] schemars.workspace = true serde.workspace = true @@ -20,3 +24,7 @@ tokio.workspace = true # === Nexus deps === nexus-toolkit.workspace = true nexus-sdk.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/math/README.md b/offchain/tools/math/README.md similarity index 100% rename from tools/math/README.md rename to offchain/tools/math/README.md diff --git a/offchain/tools/math/build.rs b/offchain/tools/math/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/math/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/math/src/i64/add.rs b/offchain/tools/math/src/i64/add.rs similarity index 94% rename from tools/math/src/i64/add.rs rename to offchain/tools/math/src/i64/add.rs index 0bbd5b2..840e361 100644 --- a/tools/math/src/i64/add.rs +++ b/offchain/tools/math/src/i64/add.rs @@ -34,7 +34,10 @@ impl NexusTool for I64Add { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.add@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.add@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/i64/cmp.rs b/offchain/tools/math/src/i64/cmp.rs similarity index 94% rename from tools/math/src/i64/cmp.rs rename to offchain/tools/math/src/i64/cmp.rs index 71de02a..2521ced 100644 --- a/tools/math/src/i64/cmp.rs +++ b/offchain/tools/math/src/i64/cmp.rs @@ -36,7 +36,10 @@ impl NexusTool for I64Cmp { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.cmp@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.cmp@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/i64/mod.rs b/offchain/tools/math/src/i64/mod.rs similarity index 100% rename from tools/math/src/i64/mod.rs rename to offchain/tools/math/src/i64/mod.rs diff --git a/tools/math/src/i64/mul.rs b/offchain/tools/math/src/i64/mul.rs similarity index 94% rename from tools/math/src/i64/mul.rs rename to offchain/tools/math/src/i64/mul.rs index 5ccd528..fc75292 100644 --- a/tools/math/src/i64/mul.rs +++ b/offchain/tools/math/src/i64/mul.rs @@ -34,7 +34,10 @@ impl NexusTool for I64Mul { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.mul@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.mul@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/i64/sum.rs b/offchain/tools/math/src/i64/sum.rs similarity index 97% rename from tools/math/src/i64/sum.rs rename to offchain/tools/math/src/i64/sum.rs index 8d0408a..650e2ec 100644 --- a/tools/math/src/i64/sum.rs +++ b/offchain/tools/math/src/i64/sum.rs @@ -34,7 +34,10 @@ impl NexusTool for I64Sum { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.math.i64.sum@1") + fqn!(concat!( + "xyz.taluslabs.math.i64.sum@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/math/src/main.rs b/offchain/tools/math/src/main.rs similarity index 100% rename from tools/math/src/main.rs rename to offchain/tools/math/src/main.rs diff --git a/offchain/tools/math/tools.json b/offchain/tools/math/tools.json new file mode 100644 index 0000000..1f533b7 --- /dev/null +++ b/offchain/tools/math/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "math", + "command": "math", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/offchain/tools/memory-memwal/.env.example b/offchain/tools/memory-memwal/.env.example new file mode 100644 index 0000000..8fcc2fa --- /dev/null +++ b/offchain/tools/memory-memwal/.env.example @@ -0,0 +1,23 @@ +# memory-memwal — environment template. Copy to `.env` (gitignored) or +# export in the deployment environment. + +# Required. Hex-encoded 32-byte Ed25519 delegate private key (64 hex chars). +# Unset → boot continues, every signed call fails. Malformed → FATAL exit. +MEMWAL_DELEGATE_PRIVATE_KEY= + +# Recommended. MemWal account object ID (`0x…`, 64 hex chars). Sent as +# `x-account-id` and signed into the canonical message. Unset → relayer +# falls back to an on-chain registry scan by public key (slower, flakier). +MEMWAL_ACCOUNT_ID= + +# Optional. Default `https://relayer.staging.memwal.ai` (testnet). +# Use `https://relayer.memwal.ai` for mainnet. Validated at startup; +# only `https://` accepted unless MEMWAL_ALLOW_INSECURE=1. +MEMWAL_SERVER_URL= + +# Dev/test only. Accept `http://` URLs (e.g. mockito). Leave unset in prod. +# MEMWAL_ALLOW_INSECURE=1 + +# Optional. Bind address for the bundled HTTP server. Default `127.0.0.1:8080`. +# Use `0.0.0.0:8080` to accept external connections. +BIND_ADDR= diff --git a/offchain/tools/memory-memwal/.just b/offchain/tools/memory-memwal/.just new file mode 100644 index 0000000..c29aaa1 --- /dev/null +++ b/offchain/tools/memory-memwal/.just @@ -0,0 +1,329 @@ +# Example invocations for the memory-memwal bundle. Recipes target the +# bundle's local HTTP server and pretty-print responses via `jq`. Requires +# `curl`, `jq`, `cargo` on PATH and `MEMWAL_DELEGATE_PRIVATE_KEY` exported +# in the binary's environment. Default relayer is testnet; set +# `MEMWAL_SERVER_URL=https://relayer.memwal.ai` for mainnet. + +# Host:port the recipes target. Override on the CLI: +# just memwal --set host 0.0.0.0:9000 health +host := "127.0.0.1:8080" + +[private] +_default: + @just --list memwal + +# --- Process lifecycle --- +# The server runs detached; PID at target/memwal-server.pid, logs at +# target/memwal-server.log. server-start blocks until /tools answers. + +_pid_subpath := "target/memwal-server.pid" +_log_subpath := "target/memwal-server.log" + +# Print an array of tokens as a copy-paste-ready shell command. Quote-only- +# when-needed: single-quote any token that contains characters outside +# `[A-Za-z0-9_/.@:=,+-]`. Single source of truth for command tracing. +_print_cmd_fn := ''' +_print_cmd() { + local a + for a in "$@"; do + if [[ -z "$a" || "$a" =~ [^A-Za-z0-9_/.@:=,+-] ]]; then + printf "'%s' " "${a//\'/\'\\\'\'}" + else + printf '%s ' "$a" + fi + done + echo +} +''' + +# `cargo locate-project --workspace` is the authoritative anchor. Defined as +# a private recipe (not a backticked variable) so the cargo invocation only +# runs for recipes that need the path, not for every `just --list`. +[private] +_workspace-root: + @dirname "$(cargo locate-project --workspace --message-format plain)" + +# Override bind: just memwal server-start 0.0.0.0:9000 +# (also set host: just memwal --set host 0.0.0.0:9000 server-start) +# Loads tools/memory-memwal/.env before spawning; existing shell exports win. +# Build the release binary, spawn it detached, wait until /tools answers. +server-start bind=host: + #!/usr/bin/env bash + set -euo pipefail + workspace_root="$(just memwal::_workspace-root)" + pid_file="$workspace_root/{{_pid_subpath}}" + log_file="$workspace_root/{{_log_subpath}}" + env_file="$workspace_root/tools/memory-memwal/.env" + cd "$workspace_root" + + # Load .env defaults without clobbering vars already exported in the + # calling shell: snapshot current exports, source .env, then replay the + # snapshot so original values override any reassignments. Variables that + # only exist in .env survive the replay (they aren't in the snapshot). + if [[ -f "$env_file" ]]; then + echo "Loading env defaults from $env_file" + env_snapshot=$(declare -px | grep '^declare -x ' || true) + set -a + # shellcheck disable=SC1090 + source "$env_file" + set +a + eval "$env_snapshot" + fi + + if [[ -f "$pid_file" ]]; then + existing=$(cat "$pid_file") + if kill -0 "$existing" 2>/dev/null; then + echo "memwal server already running (PID $existing)"; exit 1 + fi + echo "Stale PID file; removing." + rm -f "$pid_file" + fi + echo "Building memory-memwal (release)..." + cargo +stable build --package memory-memwal --release --quiet + mkdir -p "$(dirname "$pid_file")" + BIND_ADDR={{bind}} nohup ./target/release/memory-memwal \ + > "$log_file" 2>&1 & + pid=$! + echo "$pid" > "$pid_file" + # The bind probe address may differ from {{bind}} (e.g. 0.0.0.0 is not + # routable from a client); always probe via {{host}}. + # If the binary exits before answering (e.g. FATAL on malformed + # MEMWAL_DELEGATE_PRIVATE_KEY), break early and dump the log. + for i in $(seq 1 30); do + if ! kill -0 "$pid" 2>/dev/null; then + echo "FAIL: server PID $pid exited during startup. Last log lines:" >&2 + tail -n 30 "$log_file" >&2 || true + rm -f "$pid_file" + exit 1 + fi + if curl -sSf "http://{{host}}/tools" > /dev/null 2>&1; then + echo "Ready at http://{{host}} (bind={{bind}}, PID $pid, log $log_file)" + exit 0 + fi + sleep 1 + done + echo "FAIL: server did not answer on http://{{host}}/tools within 30s." >&2 + echo "Last log lines:" >&2 + tail -n 30 "$log_file" >&2 || true + exit 1 + +# Stop the detached server: SIGTERM, wait 5 s, escalate to SIGKILL. +server-stop: + #!/usr/bin/env bash + set -euo pipefail + pid_file="$(just memwal::_workspace-root)/{{_pid_subpath}}" + if [[ ! -f "$pid_file" ]]; then + echo "No PID file at $pid_file; nothing to stop."; exit 0 + fi + pid=$(cat "$pid_file") + if ! kill -0 "$pid" 2>/dev/null; then + echo "PID $pid is not alive; clearing stale PID file." + rm -f "$pid_file"; exit 0 + fi + kill -TERM "$pid" + for i in $(seq 1 5); do + if ! kill -0 "$pid" 2>/dev/null; then + rm -f "$pid_file" + echo "Stopped PID $pid." + exit 0 + fi + sleep 1 + done + echo "PID $pid did not exit after SIGTERM; sending SIGKILL." >&2 + kill -KILL "$pid" || true + rm -f "$pid_file" + +# PID + process liveness + /tools reachability. Exit 0 if fully up. +server-status: + #!/usr/bin/env bash + set -euo pipefail + pid_file="$(just memwal::_workspace-root)/{{_pid_subpath}}" + if [[ ! -f "$pid_file" ]]; then + echo "stopped (no PID file)"; exit 1 + fi + pid=$(cat "$pid_file") + if ! kill -0 "$pid" 2>/dev/null; then + echo "stopped (PID $pid is not alive; PID file is stale)"; exit 1 + fi + if curl -sSf "http://{{host}}/tools" > /dev/null 2>&1; then + echo "running (PID $pid, http://{{host}})" + else + echo "process alive (PID $pid) but http://{{host}}/tools not answering" + exit 1 + fi + +# Tail the detached server's log (`tail -n 50 -F`). +server-logs: + tail -n 50 -F "$(just memwal::_workspace-root)/{{_log_subpath}}" + +# --- Read-only inspection (no auth, no relayer round-trip) --- + +# GET /tools — list registered tool FQNs and JSON schemas. No auth. +tools: + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + cmd=(curl --fail-with-body -sS "http://{{host}}/tools") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# --- Liveness (auth required; hits the relayer) --- + +# GET /health — checks delegate key + relayer reachability + version match. +health: + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + cmd=(curl --fail-with-body -sS "http://{{host}}/health") + _print_cmd "${cmd[@]}" + "${cmd[@]}" + +# --- Tool invocations --- + +# Store a memory; blocks until written. Returns blob_id. +remember text namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg text {{quote(text)}} --arg ns {{quote(namespace)}} \ + '{text: $text, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/remember/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Semantic search over stored memories; returns ranked matches. +recall query limit="5" namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc \ + --arg q {{quote(query)}} \ + --argjson limit {{quote(limit)}} \ + --arg ns {{quote(namespace)}} \ + '{query: $q, limit: $limit, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/recall/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Memory-augmented Q&A; returns LLM answer + source memories. +ask question limit="3" namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc \ + --arg q {{quote(question)}} \ + --argjson limit {{quote(limit)}} \ + --arg ns {{quote(namespace)}} \ + '{question: $q, limit: $limit, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/ask/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Extract facts and enqueue one memory-write per fact. Fire-and-forget. +analyze text namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg text {{quote(text)}} --arg ns {{quote(namespace)}} \ + '{text: $text, namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/analyze/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Delete every memory in a namespace; returns the count removed (owner-scoped). +forget namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg ns {{quote(namespace)}} '{namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/forget/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# Memory count + storage bytes for a namespace (quota guard; 1 GB per account). +stats namespace="default": + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --arg ns {{quote(namespace)}} '{namespace: $ns}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/stats/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# items_json: JSON array of `{"text": "...", "namespace": "..."}` (namespace optional per item). +# Store up to 20 memories in one batched call. +remember-bulk items_json: + #!/usr/bin/env bash + set -euo pipefail + {{_print_cmd_fn}} + body=$(jq -nc --argjson items {{quote(items_json)}} '{items: $items}') + cmd=(curl --fail-with-body -sS -X POST + "http://{{host}}/memory/remember_bulk/invoke" + -H 'Content-Type: application/json' + -d "$body") + _print_cmd "${cmd[@]}" + "${cmd[@]}" | jq + +# --- End-to-end smoke test --- + +# remember_bulk is omitted (weight 10 would push the demo over the 30/min cap). +# Total signed-request weight ~18 — within the cap, but back-to-back re-runs +# may 429 because the sliding window hasn't cleared yet. +# End-to-end demo: auto-manages the server, runs one of each tool. +demo: + #!/usr/bin/env bash + set -euo pipefail + + # Auto-manage the server: start it iff it isn't already running, and + # tear it down on exit only in that case so a developer-started server + # is left alone. + if just memwal::server-status > /dev/null 2>&1; then + echo "Using already-running server." + else + echo "Server not running — starting it now." + just memwal::server-start + # `set -e` ensures any failure below still hits this trap. + trap 'echo; echo "Demo done — stopping the server we started."; just memwal::server-stop' EXIT + fi + echo + + # Each wrapper recipe prints the actual curl command it's about to run + # (single source of truth via a bash array), so we just call them + # directly — no need for a wrapper echo helper. + ns="demo-$(date +%s)" + echo "== namespace: $ns ==" + echo + just memwal::tools + echo + just memwal::health + echo + echo + just memwal::remember "The Eiffel Tower is in Paris, France." "$ns" + echo + just memwal::recall "famous landmarks in France" 3 "$ns" + echo + just memwal::ask "Where is the Eiffel Tower?" 3 "$ns" + echo + just memwal::analyze "Carol leads design. Dave handles ops." "$ns" + echo + just memwal::stats "$ns" + echo + just memwal::forget "$ns" diff --git a/offchain/tools/memory-memwal/Cargo.toml b/offchain/tools/memory-memwal/Cargo.toml new file mode 100644 index 0000000..9999f1f --- /dev/null +++ b/offchain/tools/memory-memwal/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "memory-memwal" +description = "Nexus Tools for MemWal persistent memory" + +edition.workspace = true +version.workspace = true +repository.workspace = true +homepage.workspace = true +license.workspace = true +readme.workspace = true +authors.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +anyhow.workspace = true +dotenvy.workspace = true +ed25519-dalek.workspace = true +env_logger.workspace = true +hex.workspace = true +log.workspace = true +nexus-sdk.workspace = true +nexus-toolkit.workspace = true +reqwest.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true +tokio.workspace = true +url.workspace = true +uuid.workspace = true +zeroize.workspace = true + +[dev-dependencies] +mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" + +[[bin]] +name = "memory-memwal" +path = "src/main.rs" diff --git a/offchain/tools/memory-memwal/README.md b/offchain/tools/memory-memwal/README.md new file mode 100644 index 0000000..d612827 --- /dev/null +++ b/offchain/tools/memory-memwal/README.md @@ -0,0 +1,373 @@ +# Memory Tools (MemWal) + +A Nexus Tool bundle exposing seven memory operations backed by the +[MemWal](https://memwal.ai) relayer: + +- `remember` / `remember_bulk` — store one or up to 20 memories +- `recall` — semantic search over stored memories +- `ask` — memory-augmented Q&A (LLM with retrieved context) +- `analyze` — extract facts from text and store each as a memory +- `forget` — delete every memory in a namespace +- `stats` — report memory count and storage bytes for a namespace + +## Pinned MemWal release + +Wire format pinned to +[`@mysten-incubation/memwal@0.0.4`](https://github.com/MystenLabs/MemWal/tree/%40mysten-incubation%2Fmemwal%400.0.4/services/server) +(server `Cargo.toml` v`0.1.0`). `health_check()` enforces the version at +runtime against `GET /health`. On a new relayer tag with a Cargo bump or +any change to `auth.rs` / `types.rs` / `routes.rs` / `rate_limit.rs`, +update `MEMWAL_API_VERSION` in `src/client.rs` and re-audit those four files. + +## Build & Run + +```sh +# Build (release) +cargo build --package memory-memwal --release + +# Run locally — binds to 127.0.0.1:8080 by default +cargo run --package memory-memwal + +# Override the bind address +BIND_ADDR=0.0.0.0:9000 cargo run --package memory-memwal +``` + +## Environment Variables + +`.env` (cwd-only) is loaded at startup via [`dotenvy`](https://crates.io/crates/dotenvy); +existing exports win. + +| Variable | Required | Default | Description | +| --- | --- | --- | --- | +| `MEMWAL_DELEGATE_PRIVATE_KEY` | **yes** | — | Hex-encoded 32-byte Ed25519 (Elliptic Curve Digital Signature Algorithm) delegate private key | +| `MEMWAL_ACCOUNT_ID` | recommended | — | MemWal account object ID (`0x…`). Sent as `x-account-id` and signed into the canonical message. Unset → relayer falls back to a slower on-chain registry scan | +| `MEMWAL_SERVER_URL` | no | `https://relayer.staging.memwal.ai` (testnet) | Relayer base URL. Validated as `https://` (no path/query/fragment/userinfo). Mainnet: `https://relayer.memwal.ai` | +| `MEMWAL_ALLOW_INSECURE` | no | unset | Set to `1` to accept `http://` relayer URLs (dev/mockito only) | + +## Server Endpoints + +```sh +# List registered tool paths +GET /tools + +# Liveness check (validates key + reaches the relayer) +GET /health + +# Invoke a tool (POST, JSON body) +POST //invoke +``` + +## Tools + +All seven tools return `Err { reason: String }` on failure (`reason` is a +short, terse description; full upstream bodies are not inlined). Per-tool +sections below cover only the `Ok` variant unless a failure mode is +tool-specific. + +--- + +# `xyz.taluslabs.memory.memwal.remember@1` + +Store a single piece of text as a persistent memory. The call blocks until the +memory is durably written to Walrus and returns the resulting blob ID — the +next vertex in a Nexus DAG (Directed Acyclic Graph) will not activate until +the write is confirmed. + +## Input + +**`text`: `String`** *(required)* + +The text to store as a memory. + +**`namespace`: `String`** *(optional)* + +Namespace used to scope this memory. Defaults to `"default"` on the server +when omitted. + +## Output Variants & Ports + +**`ok`** + +The memory was durably stored. + +- **`ok.blob_id`: `String`** — Walrus blob identifier for the stored memory. + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/remember/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "Paris is the capital of France"}' +# {"ok":{"blob_id":""}} + +# With an explicit namespace +curl -s -X POST http://127.0.0.1:8080/memory/remember/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "Alice works at ACME", "namespace": "people"}' +# {"ok":{"blob_id":""}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.recall@1` + +Semantic search over stored memories. Returns the closest matches ranked by +cosine distance — lower distance means more relevant. + +## Input + +**`query`: `String`** *(required)* + +Natural-language query to search for relevant memories. + +**`limit`: `u32`** *(optional)* + +Maximum number of results to return. Server default applies when omitted. + +**`namespace`: `String`** *(optional)* + +Namespace to search within. Searches the `"default"` namespace when omitted. + +## Output Variants & Ports + +**`ok`** + +Search completed. The result list may be empty if nothing matched. + +- **`ok.results`: `Array`** — Memories ranked by relevance, each containing: + - **`text`: `String`** — The stored text of the memory. + - **`blob_id`: `String`** — Walrus blob identifier. + - **`distance`: `f64`** — Cosine distance from the query vector (lower = more relevant). +- **`ok.namespace`: `String`** — The namespace that was searched (one query = one namespace). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/recall/invoke \ + -H 'Content-Type: application/json' \ + -d '{"query": "capital cities in Europe", "limit": 3}' +# {"ok":{"results":[{"text":"Paris is the capital of France","blob_id":"...","distance":0.12,"namespace":"default"}]}} + +# Empty result when nothing matches +curl -s -X POST http://127.0.0.1:8080/memory/recall/invoke \ + -H 'Content-Type: application/json' \ + -d '{"query": "something completely unknown"}' +# {"ok":{"results":[]}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.ask@1` + +Memory-augmented question answering. The relayer retrieves the most relevant +stored memories for the question, injects them as context into an LLM (Large +Language Model) prompt, and returns the generated answer together with the +source memories that informed it. + +## Input + +**`question`: `String`** *(required)* + +The question to answer using stored memories as context. + +**`namespace`: `String`** *(optional)* + +Namespace to retrieve memories from. Uses the `"default"` namespace when +omitted. + +**`limit`: `u32`** *(optional)* + +Maximum number of source memories to inject as context. Server default applies +when omitted. + +## Output Variants & Ports + +**`ok`** + +Question answered successfully. + +- **`ok.answer`: `String`** — LLM-generated answer. +- **`ok.sources`: `Array`** — Memories used as context, each containing: + - **`blob_id`: `String`** — Walrus blob identifier. + - **`text`: `String`** — The stored text of the memory. + - **`distance`: `f64`** — Cosine distance from the question vector. + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/ask/invoke \ + -H 'Content-Type: application/json' \ + -d '{"question": "What is the capital of France?"}' +# {"ok":{"answer":"Paris","sources":[{"blob_id":"...","text":"Paris is the capital of France","distance":0.05}]}} + +# No relevant memories → answer with empty sources +curl -s -X POST http://127.0.0.1:8080/memory/ask/invoke \ + -H 'Content-Type: application/json' \ + -d '{"question": "What is the speed of light?"}' +# {"ok":{"answer":"I don't know","sources":[]}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.analyze@1` + +Extract discrete facts from a text document and store each fact as an +individual memory. The relayer runs an LLM fact-extraction pass and enqueues +one memory-write job per fact found. + +This tool returns immediately after the jobs are enqueued — it does **not** +block on individual Walrus writes. Use `job_count` for downstream monitoring; +use `remember` if you need a confirmed blob ID. + +## Input + +**`text`: `String`** *(required)* + +The text from which to extract and store facts. + +**`namespace`: `String`** *(optional)* + +Namespace to store the extracted facts in. Uses the `"default"` namespace when +omitted. + +## Output Variants & Ports + +**`ok`** + +Facts were submitted for storage. + +- **`ok.job_count`: `u32`** — Number of individual memory-write jobs enqueued. + Zero means no facts were extracted from the text. + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/analyze/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "Alice lives in Paris. Bob works at ACME. Paris is in France."}' +# {"ok":{"job_count":3}} + +# No facts extracted → job_count is 0 +curl -s -X POST http://127.0.0.1:8080/memory/analyze/invoke \ + -H 'Content-Type: application/json' \ + -d '{"text": "..."}' +# {"ok":{"job_count":0}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.remember_bulk@1` + +Store up to 20 memories in a single batched call. The relayer rate-limits +`/api/remember` at weight 5 per call but `/api/remember/bulk` at weight 10 for +up to 20 items — a 10× efficiency gain when batching is feasible. The call +blocks until every item is durably written; a single failed item fails the +whole batch. + +## Input + +**`items`: `Array`** *(required, 1–20 entries)* + +Each entry has: + +- **`text`: `String`** *(required)* — the text to store. +- **`namespace`: `String`** *(optional)* — namespace for this item. A single + bulk call can write to multiple namespaces. + +## Output Variants & Ports + +**`ok`** + +Every item was durably stored. + +- **`ok.blob_ids`: `Array`** — Walrus blob identifiers, in the same + order as the input `items`. + +**`err`** — additionally: the whole batch fails if any individual item fails (no partial success). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/remember_bulk/invoke \ + -H 'Content-Type: application/json' \ + -d '{ + "items": [ + {"text": "Paris is the capital of France"}, + {"text": "Tokyo is the capital of Japan"}, + {"text": "Madrid is the capital of Spain"} + ] + }' +# {"ok":{"blob_ids":["","",""]}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.forget@1` + +Delete every memory in a namespace. Owner-scoped — only memories belonging to +the authenticated account are removed. Lifecycle complement to `remember`: +useful for clearing scratch namespaces at the end of a DAG run, or for +recovering quota before the 1 GB per-account cap is hit. + +## Input + +**`namespace`: `String`** *(optional)* + +The namespace to clear. Defaults to `"default"` on the server when omitted. + +## Output Variants & Ports + +**`ok`** + +Deletion completed. + +- **`ok.deleted`: `u64`** — number of memories removed. Zero is a valid + success (the namespace was empty or did not exist). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/forget/invoke \ + -H 'Content-Type: application/json' \ + -d '{"namespace": "scratch-pad"}' +# {"ok":{"deleted":12}} + +# Clearing an empty namespace is fine +curl -s -X POST http://127.0.0.1:8080/memory/forget/invoke \ + -H 'Content-Type: application/json' \ + -d '{"namespace": "never-used"}' +# {"ok":{"deleted":0}} +``` + +--- + +# `xyz.taluslabs.memory.memwal.stats@1` + +Report the number of memories and total encrypted byte size stored in a +namespace for the authenticated account. Useful as a quota guard before +heavy writes — the relayer enforces 1 GB per-account storage and starts +returning HTTP 402 once that's exceeded. + +## Input + +**`namespace`: `String`** *(optional)* + +Defaults to `"default"` on the server when omitted. + +## Output Variants & Ports + +**`ok`** + +- **`ok.memory_count`: `i64`** — number of memories in the namespace. +- **`ok.storage_bytes`: `i64`** — total encrypted byte size on Walrus. +- **`ok.namespace`: `String`** — the resolved namespace (mirrors what the + server interpreted; `"default"` when the input was omitted). + +## Example + +```sh +curl -s -X POST http://127.0.0.1:8080/memory/stats/invoke \ + -H 'Content-Type: application/json' \ + -d '{"namespace": "people"}' +# {"ok":{"memory_count":17,"storage_bytes":5242880,"namespace":"people"}} +``` diff --git a/offchain/tools/memory-memwal/build.rs b/offchain/tools/memory-memwal/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/memory-memwal/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/offchain/tools/memory-memwal/src/analyze.rs b/offchain/tools/memory-memwal/src/analyze.rs new file mode 100644 index 0000000..d81cc98 --- /dev/null +++ b/offchain/tools/memory-memwal/src/analyze.rs @@ -0,0 +1,205 @@ +//! # `xyz.taluslabs.memory.memwal.analyze@1` +//! +//! Extract facts from a document via an LLM pass and enqueue one memory-write +//! job per fact. Returns the count of jobs submitted (fire-and-forget — no +//! per-job polling). + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// The text from which to extract and store facts. + text: String, + /// Namespace to store the extracted facts in. Uses the default namespace + /// when omitted. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Facts were submitted for storage. `job_count` is the number of + /// individual memory-write jobs enqueued on the server. + Ok { + job_count: u32, + }, + Err { + reason: String, + }, +} + +pub(crate) struct AnalyzeAndRemember { + client: MemWalClient, +} + +impl NexusTool for AnalyzeAndRemember { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.analyze@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/analyze" + } + + fn description() -> &'static str { + "Extract facts from a text document and store each fact as a separate memory." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self + .client + .analyze(&input.text, input.namespace.as_deref()) + .await + { + Ok(count) => Output::Ok { + job_count: count as u32, + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> AnalyzeAndRemember { + AnalyzeAndRemember { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns `Ok { job_count }` matching the number of job_ids returned + /// by the server. + /// Failure mode caught: job_ids list not counted, job_count is always 0 or wrong. + #[tokio::test] + async fn invoke_returns_job_count() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/analyze") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_ids": ["j1", "j2", "j3"]}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "Alice lives in Paris. Bob works at ACME. Paris is in France.".into(), + namespace: None, + }) + .await; + + assert!( + matches!(output, Output::Ok { job_count: 3 }), + "expected Ok{{job_count: 3}}" + ); + } + + /// `invoke` returns `Ok { job_count: 0 }` when the server returns no jobs + /// (e.g., no facts extracted from the text). + /// Failure mode caught: empty job_ids list treated as an error. + #[tokio::test] + async fn invoke_returns_zero_for_empty_job_ids() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/analyze") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_ids": []}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "...".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Ok { job_count: 0 }), + "empty job_ids must produce Ok{{job_count: 0}}" + ); + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP error from analyze endpoint swallowed. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/analyze") + .with_status(500) + .with_body("error") + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "any text".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } +} diff --git a/offchain/tools/memory-memwal/src/ask.rs b/offchain/tools/memory-memwal/src/ask.rs new file mode 100644 index 0000000..f0968a1 --- /dev/null +++ b/offchain/tools/memory-memwal/src/ask.rs @@ -0,0 +1,249 @@ +//! # `xyz.taluslabs.memory.memwal.ask@1` +//! +//! Nexus Tool for memory-augmented Q&A. The relayer retrieves the most +//! relevant stored memories for the question, injects them into an LLM prompt, +//! and returns both the generated answer and the source memories that informed +//! it. + +use { + crate::client::{AskSource, MemWalClient}, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// The question to answer using stored memories as context. + question: String, + /// Namespace to retrieve memories from. Uses the default namespace when omitted. + namespace: Option, + /// Maximum number of source memories to inject as context. + limit: Option, +} + +/// A memory that contributed to the answer. +#[derive(Serialize, JsonSchema)] +pub(crate) struct AnswerSource { + /// Walrus blob identifier for the encrypted memory. + blob_id: String, + /// The stored text of the memory. + text: String, + /// Cosine distance from the question vector — lower means more relevant. + distance: f64, +} + +impl From for AnswerSource { + fn from(s: AskSource) -> Self { + Self { + blob_id: s.blob_id, + text: s.text, + distance: s.distance, + } + } +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// LLM-generated answer and the memories used as context. + Ok { + answer: String, + sources: Vec, + }, + Err { + reason: String, + }, +} + +pub(crate) struct AskMemory { + client: MemWalClient, +} + +impl NexusTool for AskMemory { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.ask@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/ask" + } + + fn description() -> &'static str { + "Answer a question by retrieving relevant memories and generating a response with an LLM." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self + .client + .ask(&input.question, input.namespace.as_deref(), input.limit) + .await + { + Ok(resp) => Output::Ok { + answer: resp.answer, + sources: resp.memories.into_iter().map(AnswerSource::from).collect(), + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> AskMemory { + AskMemory { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns the LLM answer and its sources on a successful call. + /// Failure mode caught: successful ask response incorrectly mapped to `Err`. + #[tokio::test] + async fn invoke_returns_answer_and_sources() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/ask") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "answer": "Paris", + "memories_used": 1, + "memories": [ + { + "blob_id": "blob-1", + "text": "Paris is the capital of France", + "distance": 0.05 + } + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + question: "What is the capital of France?".into(), + namespace: None, + limit: None, + }) + .await; + match output { + Output::Ok { answer, sources } => { + assert_eq!(answer, "Paris"); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].text, "Paris is the capital of France"); + assert_eq!(sources[0].blob_id, "blob-1"); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok` with an empty sources list when the server returns none. + /// Failure mode caught: empty sources array causes a parse error or Err variant. + #[tokio::test] + async fn invoke_handles_empty_sources() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/ask") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"answer": "I don't know", "memories_used": 0, "memories": []}).to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + question: "unknown topic".into(), + namespace: None, + limit: None, + }) + .await; + assert!( + matches!(output, Output::Ok { sources, .. } if sources.is_empty()), + "empty sources must produce Ok with empty vec" + ); + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP 500 swallowed, returning a garbage answer. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/ask") + .with_status(503) + .with_body("service unavailable") + .create_async() + .await; + + let output = tool + .invoke(Input { + question: "anything".into(), + namespace: None, + limit: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 503 must produce Err variant" + ); + } +} diff --git a/offchain/tools/memory-memwal/src/auth.rs b/offchain/tools/memory-memwal/src/auth.rs new file mode 100644 index 0000000..190396e --- /dev/null +++ b/offchain/tools/memory-memwal/src/auth.rs @@ -0,0 +1,240 @@ +//! Ed25519 request signing for the MemWal relayer. +//! +//! ```text +//! x-public-key : hex(public_key) +//! x-timestamp : unix_seconds +//! x-nonce : UUID v4 (fresh per request; server checks against a 600 s +//! Redis replay cache, missing header → HTTP 426) +//! x-signature : hex(Ed25519_sign(key, +//! "{ts}.{METHOD}.{path}.{sha256hex(body)}.{nonce}.{account_id}")) +//! ``` +//! +//! `{account_id}` is `""` when `x-account-id` is not sent. Server rejects +//! timestamps outside ±5 min — host clock must be reasonably accurate. + +use { + crate::error::AuthError, + ed25519_dalek::{Signer, SigningKey}, + sha2::{Digest, Sha256}, + std::time::{SystemTime, UNIX_EPOCH}, + zeroize::{Zeroize, Zeroizing}, +}; + +/// Headers produced by [`sign_request`]. +pub(crate) struct AuthHeaders { + pub(crate) public_key: String, + pub(crate) signature: String, + pub(crate) timestamp: String, + pub(crate) nonce: String, +} + +/// Parse a hex 32-byte Ed25519 secret into a `SigningKey` plus its public +/// key in hex. Called once at startup so signing skips per-request hex +/// decode, SHA-512 derivation, and Curve25519 scalar mult. +pub(crate) fn parse_signing_key(private_key_hex: &str) -> Result<(SigningKey, String), AuthError> { + if private_key_hex.is_empty() { + return Err(AuthError::MissingKey); + } + // Zeroize the heap buffer + stack copy; SigningKey is ZeroizeOnDrop but + // these intermediates aren't — a core dump in between would recover the key. + let raw: Zeroizing> = Zeroizing::new(hex::decode(private_key_hex)?); + if raw.len() != 32 { + return Err(AuthError::InvalidKeyLength(raw.len())); + } + let mut key_bytes: [u8; 32] = [0u8; 32]; + key_bytes.copy_from_slice(&raw); + let signing_key = SigningKey::from_bytes(&key_bytes); + key_bytes.zeroize(); + let public_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + Ok((signing_key, public_key_hex)) +} + +/// Canonical signed message per the relayer's `services/server/src/auth.rs` +/// at tag `@mysten-incubation/memwal@0.0.4`. Pure so the format can be +/// locked by a unit test independent of the crypto. +fn canonical_message( + ts: &str, + method: &str, + path: &str, + body_hash: &str, + nonce: &str, + account_id: &str, +) -> String { + format!("{ts}.{method}.{path}.{body_hash}.{nonce}.{account_id}") +} + +/// Build the auth headers for one request. Mints a fresh UUID v4 nonce per +/// call. `method` uppercase, `path` includes the leading `/`, `body` is the +/// exact bytes that go on the wire (empty slice for GET), `account_id` must +/// match what's sent as `x-account-id` (or `""` if the header is omitted). +pub(crate) fn sign_request( + signing_key: &SigningKey, + public_key_hex: &str, + method: &str, + path: &str, + body: &[u8], + account_id: &str, +) -> Result { + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| AuthError::Clock(e.to_string()))? + .as_secs() + .to_string(); + + let nonce = uuid::Uuid::new_v4().to_string(); + let body_hash = hex::encode(Sha256::digest(body)); + + let message = canonical_message(&ts, method, path, &body_hash, &nonce, account_id); + let signature = signing_key.sign(message.as_bytes()); + + Ok(AuthHeaders { + public_key: public_key_hex.to_string(), + signature: hex::encode(signature.to_bytes()), + timestamp: ts, + nonce, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_key() -> (SigningKey, String) { + // Deterministic test key: 32 bytes of 0x42. + parse_signing_key(&hex::encode([0x42u8; 32])).expect("test key must parse") + } + + /// `canonical_message` matches the relayer's expected 6-segment dot-separated + /// wire format byte-for-byte. + /// Failure mode caught: any drift in the canonical-message format (separator + /// swap, segment reorder, missing field) would pass every existing shape + /// test and only fail with HTTP 401 against the live relayer. + #[test] + fn canonical_message_format_locked() { + let m = canonical_message( + "1700000000", + "POST", + "/api/remember", + "abc123", + "11111111-2222-3333-4444-555555555555", + "0xacct", + ); + assert_eq!( + m, + "1700000000.POST./api/remember.abc123.11111111-2222-3333-4444-555555555555.0xacct" + ); + } + + /// `parse_signing_key` rejects an empty hex string with `MissingKey`. + /// Failure mode caught: missing key is silently ignored, producing + /// unauthenticated requests. + #[test] + fn parse_signing_key_rejects_empty() { + assert!(matches!(parse_signing_key(""), Err(AuthError::MissingKey))); + } + + /// `parse_signing_key` rejects non-hex with `InvalidHex`. + /// Failure mode caught: bad key bypasses validation and produces garbage + /// headers. + #[test] + fn parse_signing_key_rejects_non_hex() { + assert!(matches!( + parse_signing_key("not-hex!!"), + Err(AuthError::InvalidHex(_)) + )); + } + + /// `parse_signing_key` rejects an under-length key with `InvalidKeyLength`. + /// Failure mode caught: a 16-byte key would silently produce a 32-byte + /// signing key by zero-extension if the byte-length check were removed. + #[test] + fn parse_signing_key_rejects_short_key() { + let short = hex::encode([0u8; 16]); + assert!(matches!( + parse_signing_key(&short), + Err(AuthError::InvalidKeyLength(16)) + )); + } + + /// `parse_signing_key` returns a public key whose hex form is 64 chars. + /// Failure mode caught: public-key derivation drops bytes or returns an + /// empty string, so all subsequent x-public-key headers are malformed. + #[test] + fn parse_signing_key_derives_public_key_hex() { + let (_sk, pk_hex) = test_key(); + assert_eq!(pk_hex.len(), 64); + } + + /// `sign_request` produces non-empty headers with the right lengths for a + /// valid key and request. + /// Failure mode caught: auth module silently returns empty headers. + #[test] + fn sign_request_returns_headers() { + let (sk, pk_hex) = test_key(); + let h = sign_request( + &sk, + &pk_hex, + "POST", + "/api/remember", + b"{\"text\":\"hello\"}", + "", + ) + .expect("should sign successfully"); + + assert_eq!(h.public_key, pk_hex); + assert!(!h.signature.is_empty()); + assert!(!h.timestamp.is_empty()); + assert!(!h.nonce.is_empty()); + assert_eq!(h.public_key.len(), 64); + assert_eq!(h.signature.len(), 128); + } + + /// `sign_request` mints a fresh UUID-formatted nonce on every call. + /// Failure mode caught: a static or empty nonce would trigger replay + /// rejection (or HTTP 426) on the second invocation against the relayer. + #[test] + fn sign_request_nonce_is_unique_uuid_per_call() { + let (sk, pk_hex) = test_key(); + let h1 = sign_request(&sk, &pk_hex, "POST", "/api/remember", b"{}", "").unwrap(); + let h2 = sign_request(&sk, &pk_hex, "POST", "/api/remember", b"{}", "").unwrap(); + assert_ne!(h1.nonce, h2.nonce); + assert!(uuid::Uuid::parse_str(&h1.nonce).is_ok()); + assert!(uuid::Uuid::parse_str(&h2.nonce).is_ok()); + } + + /// Repeated signing with the same key returns the same public key. + /// Failure mode caught: per-call public-key derivation produces a different + /// public key, which would break attribution server-side. + #[test] + fn sign_request_same_key_same_pubkey() { + let (sk, pk_hex) = test_key(); + let h1 = sign_request( + &sk, + &pk_hex, + "POST", + "/api/recall", + b"{\"query\":\"foo\"}", + "", + ) + .unwrap(); + let h2 = sign_request( + &sk, + &pk_hex, + "POST", + "/api/recall", + b"{\"query\":\"foo\"}", + "", + ) + .unwrap(); + assert_eq!(h1.public_key, h2.public_key); + } + + /// `sign_request` with an empty body does not panic. + /// Failure mode caught: empty-body path panics or errors for GET requests. + #[test] + fn sign_request_handles_empty_body() { + let (sk, pk_hex) = test_key(); + let result = sign_request(&sk, &pk_hex, "GET", "/api/remember/some-job-id", b"", ""); + assert!(result.is_ok()); + } +} diff --git a/offchain/tools/memory-memwal/src/client.rs b/offchain/tools/memory-memwal/src/client.rs new file mode 100644 index 0000000..7053920 --- /dev/null +++ b/offchain/tools/memory-memwal/src/client.rs @@ -0,0 +1,1455 @@ +//! MemWal HTTP client. +//! +//! Credentials (`MEMWAL_DELEGATE_PRIVATE_KEY`, optional `MEMWAL_ACCOUNT_ID`) +//! and the relayer URL (`MEMWAL_SERVER_URL`) come from env at startup — never +//! from tool inputs, which flow through the Nexus DAG as on-chain data. + +use { + crate::{ + auth::{parse_signing_key, sign_request}, + error::{AuthError, MemWalError}, + }, + ed25519_dalek::SigningKey, + serde::{Deserialize, Serialize}, + std::{ + collections::HashMap, + sync::{Arc, Once, OnceLock}, + time::Duration, + }, + tokio::time::{sleep, Instant}, + url::Url, + zeroize::Zeroizing, +}; + +/// One-shot guard so the "missing delegate key" warning fires once even +/// though `from_env` runs once per registered tool. Keep the closure +/// trivially fast — `Once::call_once` blocks every other caller until it +/// returns. +static WARN_MISSING_KEY: Once = Once::new(); + +/// Load `/.env` if present. Cwd-only (no parent walk) so a planted +/// `.env` above the binary's cwd cannot influence the process. Existing +/// exports always win. Must be called before the tokio runtime is built — +/// `set_var` is unsound from a multi-threaded process. +pub(crate) fn load_dotenv_if_present() { + let candidate = match std::env::current_dir() { + Ok(d) => d.join(".env"), + Err(e) => { + log::warn!("could not read cwd while looking for .env: {e}"); + return; + } + }; + if !candidate.is_file() { + return; + } + match dotenvy::from_path(&candidate) { + Ok(()) => log::info!("loaded env vars from {}", candidate.display()), + Err(e) => log::warn!("failed to load {}: {e}", candidate.display()), + } +} + +/// Classification of the delegate key env var. Hand-written `Debug` redacts +/// the secret in `Ok` — never `derive` Debug on this type. +enum KeyValidation { + /// Valid: 32-byte Ed25519 scalar, hex-encoded. Wrapped in `Zeroizing` so + /// the heap buffer is wiped on drop. + Ok(Zeroizing), + /// Unset or empty. Boot continues; signed calls fail with MissingKey. + Missing, + /// Set but malformed. Carries an operator-facing reason. + Invalid(String), +} + +impl std::fmt::Debug for KeyValidation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Ok(_) => write!(f, "Ok()"), + Self::Missing => write!(f, "Missing"), + Self::Invalid(reason) => f.debug_tuple("Invalid").field(reason).finish(), + } + } +} + +impl PartialEq for KeyValidation { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Ok(a), Self::Ok(b)) => a.as_str() == b.as_str(), + (Self::Missing, Self::Missing) => true, + (Self::Invalid(a), Self::Invalid(b)) => a == b, + _ => false, + } + } +} +impl Eq for KeyValidation {} + +/// Pure classifier — no I/O, no exits, deterministic. Side effects live in +/// [`read_validated_private_key`]. +fn classify_key(value: Option<&str>) -> KeyValidation { + let raw = match value { + Some(s) if !s.is_empty() => s, + _ => return KeyValidation::Missing, + }; + let bytes = match hex::decode(raw) { + Ok(b) => b, + Err(e) => { + return KeyValidation::Invalid(format!( + "is set but is not valid hex ({e}). Expected 64 hex chars \ + encoding a 32-byte Ed25519 scalar." + )); + } + }; + if bytes.len() != 32 { + return KeyValidation::Invalid(format!( + "decoded to {} bytes, expected 32 (Ed25519 scalar).", + bytes.len() + )); + } + KeyValidation::Ok(Zeroizing::new(raw.to_string())) +} + +/// Eager startup-time key validation. Without this, a malformed key would +/// only surface on the first `/invoke` (well after `server-start` reports +/// "Ready") because `bootstrap!` constructs `NexusTool` instances lazily. +/// Called from `main`, which owns the only `process::exit` site. +pub(crate) fn validate_credentials_at_startup() -> Result<(), String> { + read_validated_private_key().map(|_| ()) +} + +/// `Ok(Some(hex))` valid; `Ok(None)` unset (warn-and-continue); `Err(reason)` +/// malformed. `main` exits on `Err`; `from_env` downgrades it to a log so a +/// mid-process key rotation doesn't kill in-flight requests. +fn read_validated_private_key() -> Result>, String> { + // `env::var` returns a fresh `String` — wrap it immediately so that + // intermediate copy is also zeroed on drop. + let raw = std::env::var(ENV_PRIVATE_KEY).ok().map(Zeroizing::new); + match classify_key(raw.as_deref().map(|z| z.as_str())) { + KeyValidation::Ok(k) => Ok(Some(k)), + KeyValidation::Missing => { + WARN_MISSING_KEY.call_once(|| { + log::warn!( + "{ENV_PRIVATE_KEY} is not set — every signed call and \ + per-tool health check will fail with MissingKey until \ + this env var is exported in the process environment." + ); + }); + Ok(None) + } + KeyValidation::Invalid(reason) => Err(reason), + } +} + +/// MemWal relayer version this crate is pinned to. +/// +/// Source of truth: tag [`@mysten-incubation/memwal@0.0.4`][tag], +/// `services/server/Cargo.toml`. Every wire-format invariant in this crate +/// (canonical signed message, header names, endpoint paths, request/ +/// response shapes) was derived from +/// `services/server/src/{auth,types,routes,rate_limit}.rs` at that tag. +/// +/// **Maintenance contract:** on every new relayer tag with a Cargo version +/// bump *or* any change to those four files, re-audit and update this +/// constant. [`MemWalClient::health_check`] enforces the version match at +/// runtime against `GET /health`'s `version` field. +/// +/// [tag]: https://github.com/MystenLabs/MemWal/tree/%40mysten-incubation%2Fmemwal%400.0.4/services/server +pub(crate) const MEMWAL_API_VERSION: &str = "0.1.0"; + +#[allow(dead_code)] +pub(crate) const RELAYER_URL_MAINNET: &str = "https://relayer.memwal.ai"; + +/// Walrus Foundation's pre-production deployment is wired to Sui testnet — +/// the `staging` hostname is not a typo. +pub(crate) const RELAYER_URL_TESTNET: &str = "https://relayer.staging.memwal.ai"; + +/// Default points at testnet so the beta API can be exercised without +/// real SUI; mainnet requires opting in via `MEMWAL_SERVER_URL`. +pub(crate) const DEFAULT_SERVER_URL: &str = RELAYER_URL_TESTNET; + +pub(crate) const ENV_SERVER_URL: &str = "MEMWAL_SERVER_URL"; +pub(crate) const ENV_PRIVATE_KEY: &str = "MEMWAL_DELEGATE_PRIVATE_KEY"; +pub(crate) const ENV_ACCOUNT_ID: &str = "MEMWAL_ACCOUNT_ID"; + +/// First poll fires after a short delay so fast jobs resolve fast. +const POLL_INITIAL: Duration = Duration::from_millis(100); +/// Cap on inter-poll delay — bounds the rate-limit cost of a slow job. +const POLL_MAX: Duration = Duration::from_secs(4); +/// Wall-clock budget per `poll_job` / `poll_bulk_jobs`. Sized for Walrus +/// erasure-coded tail latency (30–60 s under load). +const POLL_BUDGET: Duration = Duration::from_secs(60); + +fn next_backoff(current: Duration) -> Duration { + current.saturating_mul(2).min(POLL_MAX) +} + +/// Add 0..25% positive jitter to `delay` so concurrent polls of the same +/// job (e.g. two clients waiting on a shared job_id) don't lock-step on the +/// exact 100/200/400 ms ticks. Entropy comes from the system clock's +/// sub-second nanoseconds — adequate for spreading-out, not for security. +fn jittered(delay: Duration) -> Duration { + let jitter_ms = (delay.as_millis() as u64 / 4).max(1); + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos() as u64) + .unwrap_or(0); + delay + Duration::from_millis(nanos % jitter_ms) +} + +fn check_text_len(field: &str, value: &str) -> Result<(), MemWalError> { + if value.len() > MAX_TEXT_BYTES { + return Err(MemWalError::Config(format!( + "{field} exceeds maximum allowed size of {MAX_TEXT_BYTES} bytes (got {})", + value.len() + ))); + } + Ok(()) +} + +/// Mirrors the relayer's `MAX_BULK_ITEMS` so a 21-item batch fails at the +/// tool boundary with a clear reason instead of an opaque HTTP 400. +pub(crate) const MAX_BULK_ITEMS: usize = 20; + +/// Defensive cap on text-carrying tool inputs. Oversized inputs fail before +/// burning a signed-request slot. +pub(crate) const MAX_TEXT_BYTES: usize = 1 << 20; // 1 MiB + +/// Map non-2xx into `MemWalError`: 429 → `RateLimited` (parses +/// `Retry-After`); everything else → a terse `Server(_)` that does NOT +/// inline the upstream body, since `/invoke` callers are unauthenticated. +/// +/// The full body (capped 256 chars) is logged under `target=memwal::upstream`. +/// It may quote stored memory text or other relayer internals — operators +/// who treat logs as lower-trust than the binary should filter via +/// `RUST_LOG=memwal::upstream=off`. +async fn map_error_response(resp: reqwest::Response) -> MemWalError { + let status = resp.status(); + if status.as_u16() == 429 { + let retry_after_secs = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.trim().parse::().ok()); + let body = resp.text().await.unwrap_or_default(); + log::warn!( + target: "memwal::upstream", + "HTTP 429 (retry-after={retry_after_secs:?}): {}", + body.chars().take(256).collect::() + ); + return MemWalError::RateLimited { retry_after_secs }; + } + let body = resp.text().await.unwrap_or_default(); + log::warn!( + target: "memwal::upstream", + "HTTP {}: {}", + status, + body.chars().take(256).collect::() + ); + let terse = match status.as_u16() { + 401 | 403 => "upstream auth failure".to_string(), + 408 | 504 => "upstream timeout".to_string(), + s if (400..500).contains(&s) => format!("upstream rejected the request (HTTP {s})"), + s => format!("upstream unavailable (HTTP {s})"), + }; + MemWalError::Server(terse) +} + +// --------------------------------------------------------------------------- +// API response shapes +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +pub(crate) struct RememberResponse { + pub(crate) job_id: String, +} + +/// Job state from the relayer. `Unknown` (via `#[serde(other)]`) captures +/// future statuses — polling loops surface it as an error rather than +/// looping on something they don't recognize. +#[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum JobStatus { + Pending, + Running, + Uploaded, + Done, + Failed, + #[serde(other)] + Unknown, +} + +impl JobStatus { + fn is_in_progress(self) -> bool { + matches!( + self, + JobStatus::Pending | JobStatus::Running | JobStatus::Uploaded + ) + } +} + +#[derive(Deserialize)] +pub(crate) struct JobStatusResponse { + pub(crate) status: JobStatus, + pub(crate) blob_id: Option, +} + +#[derive(Deserialize)] +pub(crate) struct AnalyzeResponse { + pub(crate) job_ids: Vec, +} + +/// One result item from `POST /api/recall` (`RecallResult` in server types). +/// The server does not include `namespace` in per-result items. +#[derive(Deserialize, Clone)] +pub(crate) struct MemoryResult { + pub(crate) text: String, + pub(crate) blob_id: String, + pub(crate) distance: f64, +} + +#[derive(Deserialize)] +pub(crate) struct RecallResponse { + pub(crate) results: Vec, +} + +/// One source memory from `POST /api/ask` (`RecallResult` in server types). +#[derive(Deserialize, Clone)] +pub(crate) struct AskSource { + pub(crate) blob_id: String, + pub(crate) text: String, + pub(crate) distance: f64, +} + +/// Response from `POST /api/ask`. The server field is `memories`, not `sources`. +#[derive(Deserialize)] +pub(crate) struct AskResponse { + pub(crate) answer: String, + pub(crate) memories: Vec, +} + +/// Response from `POST /api/forget` — count of memories removed. +#[derive(Deserialize)] +pub(crate) struct ForgetResponse { + pub(crate) deleted: u64, +} + +/// Response from `POST /api/stats` — per-namespace usage summary. +#[derive(Deserialize)] +pub(crate) struct StatsResponse { + pub(crate) memory_count: i64, + pub(crate) storage_bytes: i64, + pub(crate) namespace: String, +} + +/// Response from `POST /api/remember/bulk` — 202 Accepted with one job_id per item. +#[derive(Deserialize)] +pub(crate) struct RememberBulkResponse { + pub(crate) job_ids: Vec, +} + +/// One result entry from `POST /api/remember/bulk/status`. +#[derive(Deserialize, Clone)] +pub(crate) struct BulkStatusItem { + pub(crate) job_id: String, + pub(crate) status: JobStatus, + pub(crate) blob_id: Option, +} + +/// Response from `POST /api/remember/bulk/status` — one entry per requested job_id. +#[derive(Deserialize)] +pub(crate) struct BulkStatusResponse { + pub(crate) results: Vec, +} + +/// `GET /health` body. `version` is required (enforces the maintenance-pin +/// contract on [`MEMWAL_API_VERSION`]); other fields are allowed and ignored. +#[derive(Deserialize)] +struct HealthResponse { + version: Option, +} + +// --------------------------------------------------------------------------- +// Client +// --------------------------------------------------------------------------- + +/// Process-wide HTTP client shared across every `MemWalClient` so the +/// connection pool, TLS session cache, and HTTP/2 multiplexing survive +/// across `invoke` calls. `reqwest::Client` clone is a cheap `Arc` bump. +static SHARED_HTTP: OnceLock = OnceLock::new(); + +fn shared_http() -> reqwest::Client { + SHARED_HTTP + .get_or_init(|| { + reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(10)) + .pool_idle_timeout(Duration::from_secs(90)) + .pool_max_idle_per_host(8) + .tcp_keepalive(Duration::from_secs(30)) + .build() + .expect("reqwest client builder must succeed with these options") + }) + .clone() +} + +/// `MEMWAL_ALLOW_INSECURE=1` opts in to non-HTTPS relayer URLs (dev/test only). +fn allow_insecure() -> bool { + std::env::var("MEMWAL_ALLOW_INSECURE") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) +} + +/// Validate a relayer base URL. Rejects: non-https (unless `allow_insecure`), +/// path beyond root, query/fragment, userinfo. `allow_insecure` is a +/// parameter (not read from env) so the policy is pure-testable. +fn parse_relayer_url(raw: &str, allow_insecure: bool) -> Result { + // Don't echo the raw URL in parse errors: an embedded + // `user:secret@host` plus a typo would leak the secret into stderr, + // log files, and (because each tool's `new()` panics on Config) into + // the panic message itself. + let url = + Url::parse(raw).map_err(|e| MemWalError::Config(format!("invalid relayer URL: {e}")))?; + let scheme = url.scheme(); + let scheme_ok = scheme == "https" || (allow_insecure && scheme == "http"); + if !scheme_ok { + return Err(MemWalError::Config(format!( + "relayer URL must use https (got scheme `{scheme}`)" + ))); + } + let path = url.path(); + if !path.is_empty() && path != "/" { + return Err(MemWalError::Config(format!( + "relayer URL must not carry a path (got `{path}`)" + ))); + } + if url.query().is_some() || url.fragment().is_some() { + return Err(MemWalError::Config( + "relayer URL must not carry a query or fragment".into(), + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(MemWalError::Config( + "relayer URL must not embed credentials (userinfo)".into(), + )); + } + Ok(url) +} + +/// Per-tool HTTP client. All non-trivial state is behind `Arc`/`reqwest::Client` +/// so cloning is cheap — every `NexusTool` holds one and reuses it across +/// every `invoke`. The signing key is parsed at construction so the per-call +/// path never re-decodes hex or rebuilds the public key. +#[derive(Clone)] +pub(crate) struct MemWalClient { + http: reqwest::Client, + api_base: Url, + /// `None` when the env key was missing or malformed; signed calls then + /// return `AuthError::MissingKey` instead of taking down the process. + signing: Option>, + /// Empty when unconfigured (relayer falls back to a registry scan). + /// Signed into the canonical message; sent as `x-account-id` only when + /// non-empty (mirrors the JS SDK). + account_id: Arc, +} + +struct SigningMaterial { + signing_key: SigningKey, + public_key_hex: String, +} + +impl MemWalClient { + /// Production constructor: validates `MEMWAL_SERVER_URL` (`Err(Config)` + /// on parse failure) and parses the delegate key. A missing/malformed + /// key logs but doesn't error here — `main`'s startup validation is the + /// fail-fast site. + pub(crate) fn from_env() -> Result { + let api_base_raw = std::env::var(ENV_SERVER_URL) + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| DEFAULT_SERVER_URL.to_string()); + let api_base = parse_relayer_url(&api_base_raw, allow_insecure())?; + + let signing = match read_validated_private_key() { + Ok(Some(hex)) => match parse_signing_key(&hex) { + Ok((signing_key, public_key_hex)) => Some(Arc::new(SigningMaterial { + signing_key, + public_key_hex, + })), + Err(e) => { + log::error!("{ENV_PRIVATE_KEY} could not be parsed: {e}"); + None + } + }, + Ok(None) => None, + Err(reason) => { + log::error!("{ENV_PRIVATE_KEY} {reason}"); + None + } + }; + + let account_id: Arc = std::env::var(ENV_ACCOUNT_ID).unwrap_or_default().into(); + + Ok(Self { + http: shared_http(), + api_base, + signing, + account_id, + }) + } + + /// Test-only constructor. Accepts an unparsed URL (HTTP loopback for + /// mockito) and a hex key; panics on parse failure since tests should + /// not be exercising the error path. + #[cfg(test)] + pub(crate) fn with_test_config( + api_base: &str, + private_key_hex: &str, + account_id: &str, + ) -> Self { + let url = Url::parse(api_base).expect("test URL must parse"); + let (signing_key, public_key_hex) = + parse_signing_key(private_key_hex).expect("test key must parse"); + Self { + http: reqwest::Client::new(), + api_base: url, + signing: Some(Arc::new(SigningMaterial { + signing_key, + public_key_hex, + })), + account_id: account_id.into(), + } + } + + /// Sign and dispatch a request, returning the `reqwest::Response` so + /// callers can `.json()` it. + fn signed_headers( + &self, + method: &str, + path: &str, + body: &[u8], + ) -> Result { + let signing = self + .signing + .as_ref() + .ok_or(MemWalError::Auth(AuthError::MissingKey))?; + Ok(sign_request( + &signing.signing_key, + &signing.public_key_hex, + method, + path, + body, + &self.account_id, + )?) + } + + fn join_path(&self, path: &str) -> Result { + self.api_base + .join(path) + .map_err(|e| MemWalError::Config(format!("invalid path `{path}`: {e}"))) + } + + // ----------------------------------------------------------------------- + // Low-level request helpers + // ----------------------------------------------------------------------- + + async fn post( + &self, + path: &str, + body: &B, + ) -> Result { + let body_bytes = serde_json::to_vec(body).expect("serializable body"); + let headers = self.signed_headers("POST", path, &body_bytes)?; + let url = self.join_path(path)?; + + let mut req = self + .http + .post(url) + .header("x-public-key", &headers.public_key) + .header("x-signature", &headers.signature) + .header("x-timestamp", &headers.timestamp) + .header("x-nonce", &headers.nonce) + .header("content-type", "application/json"); + if !self.account_id.is_empty() { + req = req.header("x-account-id", self.account_id.as_ref()); + } + let resp = req.body(body_bytes).send().await?; + + if !resp.status().is_success() { + return Err(map_error_response(resp).await); + } + + Ok(resp) + } + + async fn get(&self, path: &str) -> Result { + let headers = self.signed_headers("GET", path, b"")?; + let url = self.join_path(path)?; + + let mut req = self + .http + .get(url) + .header("x-public-key", &headers.public_key) + .header("x-signature", &headers.signature) + .header("x-timestamp", &headers.timestamp) + .header("x-nonce", &headers.nonce); + if !self.account_id.is_empty() { + req = req.header("x-account-id", self.account_id.as_ref()); + } + let resp = req.send().await?; + + if !resp.status().is_success() { + return Err(map_error_response(resp).await); + } + + Ok(resp) + } + + // ----------------------------------------------------------------------- + // Domain-level API calls + // ----------------------------------------------------------------------- + + /// `POST /api/remember` — enqueue a single memory. + pub(crate) async fn remember( + &self, + text: &str, + namespace: Option<&str>, + ) -> Result { + check_text_len("text", text)?; + #[derive(Serialize)] + struct Req<'a> { + text: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: RememberResponse = self + .post("/api/remember", &Req { text, namespace }) + .await? + .json() + .await?; + Ok(resp.job_id) + } + + /// Poll `GET /api/remember/:job_id` to terminal state. Exponential + /// backoff (POLL_INITIAL → POLL_MAX) within POLL_BUDGET; an unrecognized + /// status from the server is a hard error, not an in-progress signal. + /// + /// **Cancel-safety:** the loop is cancel-safe locally, but the + /// server-side write is **not cancellable** — dropping the future after + /// the initial 202 leaks one Walrus blob. + pub(crate) async fn poll_job(&self, job_id: &str) -> Result { + let path = format!("/api/remember/{job_id}"); + let deadline = Instant::now() + POLL_BUDGET; + let mut delay = POLL_INITIAL; + + loop { + sleep(jittered(delay)).await; + let status: JobStatusResponse = self.get(&path).await?.json().await?; + match status.status { + JobStatus::Done => { + return status.blob_id.ok_or_else(|| { + MemWalError::Server(format!( + "job {job_id} reached terminal status `done` but \ + blob_id is missing from the response" + )) + }); + } + JobStatus::Failed => return Err(MemWalError::JobFailed(job_id.to_string())), + s if s.is_in_progress() => {} + JobStatus::Unknown => { + return Err(MemWalError::Server(format!( + "job {job_id} returned an unrecognized status" + ))); + } + _ => unreachable!("is_in_progress covers Pending/Running/Uploaded"), + } + if Instant::now() >= deadline { + return Err(MemWalError::Timeout(job_id.to_string())); + } + delay = next_backoff(delay); + } + } + + /// `POST /api/recall` — semantic search over stored memories. + pub(crate) async fn recall( + &self, + query: &str, + limit: Option, + namespace: Option<&str>, + ) -> Result, MemWalError> { + check_text_len("query", query)?; + #[derive(Serialize)] + struct Req<'a> { + query: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: RecallResponse = self + .post( + "/api/recall", + &Req { + query, + limit, + namespace, + }, + ) + .await? + .json() + .await?; + Ok(resp.results) + } + + /// `POST /api/ask` — memory-augmented Q&A. + pub(crate) async fn ask( + &self, + question: &str, + namespace: Option<&str>, + limit: Option, + ) -> Result { + check_text_len("question", question)?; + #[derive(Serialize)] + struct Req<'a> { + question: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + limit: Option, + } + // The server field is `memories`, not `sources` — AskResponse reflects this. + let resp: AskResponse = self + .post( + "/api/ask", + &Req { + question, + namespace, + limit, + }, + ) + .await? + .json() + .await?; + Ok(resp) + } + + /// `POST /api/analyze` — extract facts from text and enqueue each as a memory. + /// Returns the number of jobs submitted. + pub(crate) async fn analyze( + &self, + text: &str, + namespace: Option<&str>, + ) -> Result { + check_text_len("text", text)?; + #[derive(Serialize)] + struct Req<'a> { + text: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: AnalyzeResponse = self + .post("/api/analyze", &Req { text, namespace }) + .await? + .json() + .await?; + Ok(resp.job_ids.len()) + } + + /// `POST /api/forget` — delete every memory in the given namespace. + /// Owner-scoped: only memories whose owner matches the authenticated + /// account can be deleted, regardless of the namespace string sent. + pub(crate) async fn forget(&self, namespace: Option<&str>) -> Result { + #[derive(Serialize)] + struct Req<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + let resp: ForgetResponse = self + .post("/api/forget", &Req { namespace }) + .await? + .json() + .await?; + Ok(resp.deleted) + } + + /// `POST /api/stats` — memory count + stored byte total for a namespace. + pub(crate) async fn stats( + &self, + namespace: Option<&str>, + ) -> Result { + #[derive(Serialize)] + struct Req<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + self.post("/api/stats", &Req { namespace }) + .await? + .json() + .await + .map_err(MemWalError::from) + } + + /// Submit up to `MAX_BULK_ITEMS` texts in one 202-Accepted call. Callers + /// must enforce the `1..=MAX_BULK_ITEMS` cap before this — oversized + /// batches get an opaque HTTP 400 from the relayer. Pair with + /// [`poll_bulk_jobs`] for the batched status endpoint. + pub(crate) async fn remember_bulk( + &self, + items: &[(&str, Option<&str>)], + ) -> Result, MemWalError> { + for (text, _) in items { + check_text_len("text", text)?; + } + #[derive(Serialize)] + struct Item<'a> { + text: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + namespace: Option<&'a str>, + } + #[derive(Serialize)] + struct Req<'a> { + items: Vec>, + } + let req = Req { + items: items + .iter() + .map(|(text, ns)| Item { + text, + namespace: *ns, + }) + .collect(), + }; + let resp: RememberBulkResponse = + self.post("/api/remember/bulk", &req).await?.json().await?; + Ok(resp.job_ids) + } + + /// Poll batched jobs to terminal state, returning blob_ids in input + /// order. First `failed` surfaces as `Err(JobFailed)`. Each poll queries + /// only the still-pending subset (cached terminal jobs in a HashMap), + /// so rate-limit cost shrinks with completion. Same cancel-safety + /// caveat as [`MemWalClient::poll_job`], scaled to MAX_BULK_ITEMS. + pub(crate) async fn poll_bulk_jobs( + &self, + job_ids: &[String], + ) -> Result, MemWalError> { + if job_ids.is_empty() { + return Ok(Vec::new()); + } + let deadline = Instant::now() + POLL_BUDGET; + let mut delay = POLL_INITIAL; + let mut terminal: HashMap = HashMap::with_capacity(job_ids.len()); + + loop { + let pending: Vec = job_ids + .iter() + .filter(|id| !terminal.contains_key(*id)) + .cloned() + .collect(); + if pending.is_empty() { + break; + } + + sleep(jittered(delay)).await; + let statuses = self.poll_bulk_status_once(&pending).await?; + let by_id: HashMap<&str, &BulkStatusItem> = + statuses.iter().map(|s| (s.job_id.as_str(), s)).collect(); + + for id in &pending { + let Some(item) = by_id.get(id.as_str()) else { + return Err(MemWalError::Server(format!( + "bulk status response missing job {id}" + ))); + }; + match item.status { + JobStatus::Done | JobStatus::Failed => { + terminal.insert(id.clone(), (*item).clone()); + } + s if s.is_in_progress() => {} + JobStatus::Unknown => { + return Err(MemWalError::Server(format!( + "job {id} returned an unrecognized status" + ))); + } + _ => unreachable!("is_in_progress covers Pending/Running/Uploaded"), + } + } + + if Instant::now() >= deadline { + return Err(MemWalError::Timeout(format!( + "{} bulk jobs did not finish in time", + job_ids.len() - terminal.len() + ))); + } + delay = next_backoff(delay); + } + + // All jobs terminal — assemble in input order, surface first failure. + job_ids + .iter() + .map(|id| { + let item = terminal + .get(id) + .expect("loop only exits when terminal covers job_ids"); + match item.status { + JobStatus::Done => item.blob_id.clone().ok_or_else(|| { + MemWalError::Server(format!( + "job {id} reached terminal status `done` but \ + blob_id is missing" + )) + }), + JobStatus::Failed => Err(MemWalError::JobFailed(id.clone())), + _ => unreachable!("terminal cache only stores Done/Failed"), + } + }) + .collect() + } + + /// Single shot of `POST /api/remember/bulk/status`. Returns whatever the + /// server says without polling. Exposed for callers that want to inspect + /// per-job state directly (e.g. tolerate partial failures). + pub(crate) async fn poll_bulk_status_once( + &self, + job_ids: &[String], + ) -> Result, MemWalError> { + #[derive(Serialize)] + struct Req<'a> { + job_ids: &'a [String], + } + let resp: BulkStatusResponse = self + .post("/api/remember/bulk/status", &Req { job_ids }) + .await? + .json() + .await?; + Ok(resp.results) + } + + /// `GET /health` — reachability + API-version match. Public (no auth). + /// Required: JSON body with `version` matching [`MEMWAL_API_VERSION`]. + /// Missing field / wrong type / non-JSON / mismatch all return `Err`; + /// "best-effort" version check would void the maintenance-pin contract. + pub(crate) async fn health_check(&self) -> Result<(), MemWalError> { + let url = self.join_path("/health")?; + let resp = self.http.get(url).send().await?; + let status = resp.status(); + if !status.is_success() { + return Err(MemWalError::Server(format!( + "relayer health returned HTTP {}", + status.as_u16() + ))); + } + let body: HealthResponse = resp.json().await.map_err(|e| { + MemWalError::Server(format!("relayer /health did not return JSON: {e}")) + })?; + let server_ver = body.version.ok_or_else(|| { + MemWalError::Server( + "relayer /health did not return a `version` field — pre-pinned-tag relayer?".into(), + ) + })?; + if server_ver != MEMWAL_API_VERSION { + return Err(MemWalError::Server(format!( + "relayer version mismatch: tools expect {MEMWAL_API_VERSION}, \ + server reports {server_ver} — update the tools or pin the relayer" + ))); + } + Ok(()) + } + + /// Returns `Err(MissingKey)` when the delegate key was missing or + /// malformed at construction time. Validity follows from the type: + /// `signing` only carries `Some` once `parse_signing_key` succeeded, + /// so there is no per-call hex decode or length check to repeat. + pub(crate) fn validate_key(&self) -> Result<(), AuthError> { + if self.signing.is_none() { + return Err(AuthError::MissingKey); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `classify_key(None)` reports `Missing`. + /// Failure mode caught: an unset env var is silently treated as a valid + /// empty key, masking the misconfiguration behind apparent boot success. + #[test] + fn classify_key_missing_when_unset() { + assert_eq!(classify_key(None), KeyValidation::Missing); + } + + /// `classify_key(Some(""))` reports `Missing`, matching shells that export + /// the variable with an empty value. + /// Failure mode caught: an empty-string env var is treated as a valid key + /// and produces empty hex headers on every signed request. + #[test] + fn classify_key_missing_when_empty() { + assert_eq!(classify_key(Some("")), KeyValidation::Missing); + } + + /// A canonical 32-byte hex key is classified as `Ok` and the original + /// hex string is preserved verbatim (the signer uses the hex form, not + /// re-encoded bytes). + /// Failure mode caught: a valid key is misclassified as Invalid, which + /// would abort the binary on the happy path. + #[test] + fn classify_key_ok_on_valid_32_byte_hex() { + let key = hex::encode([0x42u8; 32]); + match classify_key(Some(&key)) { + KeyValidation::Ok(returned) => assert_eq!(returned.as_str(), key.as_str()), + other => panic!("expected Ok(\"{key}\"), got {other:?}"), + } + } + + /// Non-hex input is classified as `Invalid` with a reason mentioning hex. + /// Failure mode caught: malformed input passes validation, exits would be + /// missing, and the binary boots with garbage credentials that produce + /// confusing relayer errors at request time. + #[test] + fn classify_key_invalid_on_non_hex() { + match classify_key(Some("not-hex!!")) { + KeyValidation::Invalid(reason) => assert!( + reason.contains("hex"), + "reason must mention hex format; got: {reason}" + ), + other => panic!("expected Invalid, got {other:?}"), + } + } + + /// Hex that decodes to fewer than 32 bytes is classified as `Invalid` + /// with a byte-count reason. + /// Failure mode caught: a truncated key is accepted, leading to silent + /// signature verification failures the operator cannot diagnose. + #[test] + fn classify_key_invalid_on_too_few_bytes() { + let short = hex::encode([0u8; 16]); // 16 bytes + match classify_key(Some(&short)) { + KeyValidation::Invalid(reason) => { + assert!( + reason.contains("16") && reason.contains("32"), + "reason must mention actual (16) and expected (32) byte counts; got: {reason}" + ); + } + other => panic!("expected Invalid, got {other:?}"), + } + } + + /// Hex that decodes to more than 32 bytes is also classified as `Invalid`. + /// Failure mode caught: an over-long key (e.g. a Sui-formatted key + /// pasted with extra flag bytes) is silently truncated or accepted. + #[test] + fn classify_key_invalid_on_too_many_bytes() { + let long = hex::encode([0u8; 64]); // 64 bytes + match classify_key(Some(&long)) { + KeyValidation::Invalid(reason) => { + assert!( + reason.contains("64") && reason.contains("32"), + "reason must mention actual (64) and expected (32) byte counts; got: {reason}" + ); + } + other => panic!("expected Invalid, got {other:?}"), + } + } + + /// All-`0xff` 32-byte input is `Ok` — Ed25519 has no scalar-range + /// rejection in dalek's `from_bytes`, which SHA-512-derives the scalar. + /// Failure mode caught: an over-strict client-side filter rejects a key + /// the relayer would actually accept, blocking legitimate deployments. + #[test] + fn classify_key_ok_on_all_ones_32_bytes() { + let key = hex::encode([0xffu8; 32]); + assert_eq!( + classify_key(Some(&key)), + KeyValidation::Ok(Zeroizing::new(key)) + ); + } + + /// `Debug` impl on `KeyValidation::Ok` redacts the hex secret. + /// Failure mode caught: a future refactor switches the manual `Debug` impl + /// back to a derive, or someone accidentally prints `{:?}` on a + /// KeyValidation — either way, the delegate key would land in a log line + /// or panic message. This test asserts the redaction is structural. + #[test] + fn key_validation_debug_redacts_secret() { + let key_hex = hex::encode([0x42u8; 32]); + let v = KeyValidation::Ok(Zeroizing::new(key_hex.clone())); + let debug_output = format!("{v:?}"); + assert!( + !debug_output.contains(&key_hex), + "Debug output must not contain the hex secret. Got: {debug_output}" + ); + assert_eq!(debug_output, "Ok()"); + } + + /// `Debug` impl on the other two variants still emits informative text + /// (no redaction needed — they don't carry secrets). + /// Failure mode caught: an over-zealous redaction sweep silently strips + /// the `Invalid` reason, hiding the actual misconfiguration from the + /// startup log line. + #[test] + fn key_validation_debug_preserves_non_secret_variants() { + assert_eq!(format!("{:?}", KeyValidation::Missing), "Missing"); + let reason = "is set but is not valid hex (...)".to_string(); + let dbg = format!("{:?}", KeyValidation::Invalid(reason.clone())); + assert!(dbg.contains(&reason), "Invalid must show its reason: {dbg}"); + } + + /// `PartialEq` distinguishes every cross-variant pair. + /// Failure mode caught: a hand-written `eq` that accidentally returns + /// `true` on cross-variant comparisons would let miscategorized inputs + /// slip through equality checks. + #[test] + fn key_validation_partial_eq_cross_variants_differ() { + let v_ok = KeyValidation::Ok(Zeroizing::new("x".into())); + let v_missing = KeyValidation::Missing; + let v_invalid = KeyValidation::Invalid("r".into()); + assert_ne!(v_ok, v_missing); + assert_ne!(v_ok, v_invalid); + assert_ne!(v_missing, v_invalid); + } + + /// `jittered(delay)` returns a value in `[delay, delay + ceil(delay/4))`. + /// Failure mode caught: a regression that overshoots the 25% jitter cap + /// would silently extend the polling deadline, or a missing-jitter + /// regression (returning exactly `delay`) would re-introduce lock-step + /// polling between concurrent clients on the same job. + #[test] + fn jittered_bounded() { + for d_ms in [100u64, 250, 500, 1000, 4000] { + let d = Duration::from_millis(d_ms); + // The function isn't deterministic, but a small batch will + // exercise enough nanosecond values to expose an out-of-range + // result. + for _ in 0..50 { + let got = jittered(d); + assert!( + got >= d, + "jittered({d:?}) returned {got:?}, less than the base delay" + ); + let max_jitter = Duration::from_millis((d_ms / 4).max(1)); + assert!( + got < d + max_jitter, + "jittered({d:?}) returned {got:?}, exceeded base + 25%" + ); + } + } + } + + /// `check_text_len` accepts an input exactly at the cap, rejects one + /// byte over, and the error message names the cap and the actual size. + /// Failure mode caught: an off-by-one on the `>` vs `>=` comparison, + /// or an error message that doesn't help the operator diagnose what + /// they sent. + #[test] + fn check_text_len_boundary() { + let at_limit = "x".repeat(MAX_TEXT_BYTES); + assert!(check_text_len("text", &at_limit).is_ok()); + + let over_limit = "x".repeat(MAX_TEXT_BYTES + 1); + match check_text_len("text", &over_limit) { + Err(MemWalError::Config(reason)) => { + assert!( + reason.contains(&MAX_TEXT_BYTES.to_string()), + "reason must mention the cap; got: {reason}" + ); + assert!( + reason.contains(&(MAX_TEXT_BYTES + 1).to_string()), + "reason must mention the actual size; got: {reason}" + ); + } + other => panic!("expected Config(...), got {other:?}"), + } + } + + // ----------------------------------------------------------------------- + // parse_relayer_url policy tests + // + // The flag is passed explicitly (rather than read from env) so each test + // is hermetic — no global state, no test ordering, no MEMWAL_ALLOW_INSECURE + // bleed-through across the suite. + // ----------------------------------------------------------------------- + + /// Accepts a canonical https URL with no trailing slash. + /// Failure mode caught: an over-strict validator that rejects the most + /// common form would block the default deployment. + #[test] + fn parse_relayer_url_accepts_canonical_https() { + assert!(parse_relayer_url("https://relayer.memwal.ai", false).is_ok()); + } + + /// Accepts a trailing slash — url::Url normalises the path to "/". + /// Failure mode caught: a regression where the path check rejects "/" + /// would break operators who copy-paste from a browser bar. + #[test] + fn parse_relayer_url_accepts_trailing_slash() { + assert!(parse_relayer_url("https://relayer.memwal.ai/", false).is_ok()); + } + + /// Accepts an explicit port. + /// Failure mode caught: a host:port URL is misparsed or rejected. + #[test] + fn parse_relayer_url_accepts_port() { + assert!(parse_relayer_url("https://relayer.memwal.ai:8443", false).is_ok()); + } + + /// Rejects http:// when allow_insecure is false. + /// Failure mode caught: a regression that defaults to permissive + /// scheme handling would let an operator (or a tool input that hadn't + /// been removed) downgrade signing material to cleartext. + #[test] + fn parse_relayer_url_rejects_http_when_insecure_disallowed() { + let err = + parse_relayer_url("http://relayer.memwal.ai", false).expect_err("must reject http"); + assert!(err.to_string().contains("https")); + } + + /// Accepts http:// when allow_insecure is true (for local dev / mockito). + /// Failure mode caught: the escape hatch is broken; mockito tests can't + /// construct a real `MemWalClient::from_env` path. + #[test] + fn parse_relayer_url_accepts_http_when_insecure_allowed() { + assert!(parse_relayer_url("http://127.0.0.1:1234", true).is_ok()); + } + + /// Rejects exotic schemes regardless of allow_insecure. + /// Failure mode caught: a `ws://` or `file://` URL bypasses scheme + /// checks and reaches reqwest, which would either error obscurely or + /// open an unintended transport. + #[test] + fn parse_relayer_url_rejects_non_http_schemes() { + for s in ["ws://x", "file:///etc/passwd", "data:text/plain,foo"] { + assert!( + parse_relayer_url(s, true).is_err(), + "expected rejection of `{s}` even with allow_insecure" + ); + } + } + + /// Rejects URLs that carry a path beyond the root. + /// Failure mode caught: `Url::join("/api/remember")` against a base + /// like `https://host/v1` would yield `https://host/api/remember` + /// (Url::join treats absolute paths as replacements), silently + /// rewriting what the operator configured. + #[test] + fn parse_relayer_url_rejects_path() { + let err = parse_relayer_url("https://relayer.memwal.ai/v1", false) + .expect_err("path must be rejected"); + assert!(err.to_string().contains("path")); + } + + /// Rejects URLs that carry a query string. + /// Failure mode caught: `?leak=/api/recall` smuggled in the base URL + /// would either be silently dropped by Url::join or merged into the + /// signed-path semantics, depending on the relayer's parsing. + #[test] + fn parse_relayer_url_rejects_query() { + let err = parse_relayer_url("https://relayer.memwal.ai/?leak=1", false) + .expect_err("query must be rejected"); + assert!(err.to_string().contains("query") || err.to_string().contains("fragment")); + } + + /// Rejects URLs that carry a fragment. + /// Failure mode caught: same shape as the query case; fragments are + /// usually stripped client-side but should not be silently accepted. + #[test] + fn parse_relayer_url_rejects_fragment() { + let err = parse_relayer_url("https://relayer.memwal.ai/#frag", false) + .expect_err("fragment must be rejected"); + assert!(err.to_string().contains("query") || err.to_string().contains("fragment")); + } + + /// Rejects URLs that embed a username. + /// Failure mode caught: an operator who pastes `https://user@host` + /// (e.g. from a curl command) would otherwise have the username + /// tagged onto every outbound request as a Basic-Auth identifier + /// — pointless against this relayer and an unintended exposure. + #[test] + fn parse_relayer_url_rejects_userinfo_username_only() { + let err = parse_relayer_url("https://user@relayer.memwal.ai", false) + .expect_err("userinfo must be rejected"); + assert!(err.to_string().contains("credentials")); + } + + /// Rejects URLs that embed a username and password. + /// Failure mode caught: the more dangerous variant of the userinfo + /// case — a real secret tagged onto every outbound request, AND a + /// secret that would have leaked through parse-error echoes before + /// the no-echo policy was put in place. + #[test] + fn parse_relayer_url_rejects_userinfo_with_password() { + let err = parse_relayer_url("https://user:secret@relayer.memwal.ai", false) + .expect_err("userinfo must be rejected"); + assert!(err.to_string().contains("credentials")); + } + + /// Parse-failure messages do NOT echo the raw input. + /// Failure mode caught: a future refactor reintroduces `{raw}` into + /// the error string and a typoed `https://user:secret@host/bad path` + /// (unparsable due to space) leaks the secret into logs and panic + /// messages. + #[test] + fn parse_relayer_url_does_not_echo_raw_on_parse_failure() { + let err = parse_relayer_url("https://user:secret@host/bad path", false) + .expect_err("malformed URL must Err"); + let msg = err.to_string(); + assert!(!msg.contains("secret"), "secret leaked into error: {msg}"); + assert!(!msg.contains("user"), "userinfo leaked into error: {msg}"); + } + + fn make_client(server_url: &str) -> MemWalClient { + MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), "") + } + + /// `health_check` returns `Ok(())` when `/health` returns the pinned version. + /// Failure mode caught: a version comparison that silently accepts any + /// response would surface here as a passing test against a deliberately + /// matching version mock; we then can flip the mock to verify failure + /// in the sibling tests. + #[tokio::test] + async fn health_check_ok_on_matching_version() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + serde_json::json!({"status": "ok", "version": MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let client = make_client(&server.url()); + assert!(client.health_check().await.is_ok()); + } + + /// `health_check` returns `Err` when `/health` reports a different + /// version. The MEMWAL_API_VERSION pin is the maintenance contract; + /// silently accepting a mismatch defeats the audit trail. + #[tokio::test] + async fn health_check_err_on_mismatched_version() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::json!({"status": "ok", "version": "9.9.9"}).to_string()) + .create_async() + .await; + let client = make_client(&server.url()); + let err = client + .health_check() + .await + .expect_err("version mismatch must Err"); + assert!( + err.to_string().contains("9.9.9"), + "error must mention the server version; got: {err}" + ); + } + + /// `health_check` returns `Err` when `/health` omits `version`. + /// Failure mode caught: a degraded relayer that drops the version field + /// would silently look healthy under the previous best-effort check. + #[tokio::test] + async fn health_check_err_when_version_missing() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(serde_json::json!({"status": "ok"}).to_string()) + .create_async() + .await; + let client = make_client(&server.url()); + let err = client + .health_check() + .await + .expect_err("missing version must Err"); + assert!( + err.to_string().contains("version"), + "error must mention version; got: {err}" + ); + } + + /// A 429 from the relayer surfaces as `MemWalError::RateLimited` with + /// the parsed `Retry-After` seconds, not the generic `Server` variant. + /// Failure mode caught: rate-limit responses look identical to other + /// upstream failures, so a DAG retry policy cannot distinguish "back + /// off" from "try a different relayer". + #[tokio::test] + async fn post_translates_429_to_rate_limited() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/forget") + .with_status(429) + .with_header("retry-after", "42") + .with_body("rate limited") + .create_async() + .await; + let client = make_client(&server.url()); + let err = client.forget(None).await.expect_err("429 must Err"); + match err { + MemWalError::RateLimited { retry_after_secs } => { + assert_eq!(retry_after_secs, Some(42)); + } + other => panic!("expected RateLimited, got {other}"), + } + } + + /// A 429 with no `Retry-After` header surfaces with `None` rather than + /// failing to parse. + /// Failure mode caught: a missing header is treated as a malformed + /// response and the call becomes Server(...) instead of RateLimited. + #[tokio::test] + async fn post_translates_429_without_retry_after() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/forget") + .with_status(429) + .with_body("rate limited") + .create_async() + .await; + let client = make_client(&server.url()); + let err = client.forget(None).await.expect_err("429 must Err"); + assert!(matches!( + err, + MemWalError::RateLimited { + retry_after_secs: None + } + )); + } + + /// A 500 from the relayer surfaces with a terse client-facing reason — + /// the upstream body is logged but NOT inlined into the error string. + /// Failure mode caught: a relayer that leaks account internals or + /// moderation messages in its 500 body would forward that text verbatim + /// to the unauthenticated /invoke caller. + #[tokio::test] + async fn post_terse_message_on_5xx() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("POST", "/api/forget") + .with_status(500) + .with_body("secret internal payload that should not leak") + .create_async() + .await; + let client = make_client(&server.url()); + let err = client.forget(None).await.expect_err("500 must Err"); + let msg = err.to_string(); + assert!(!msg.contains("secret internal payload")); + assert!(msg.contains("upstream")); + } + + /// `health_check` returns `Err` when the body is not JSON at all. + /// Failure mode caught: a misconfigured reverse proxy serving an HTML + /// error page would parse-fail silently and the call would short-circuit + /// to Ok under a permissive `if let Ok(...)` pattern. + #[tokio::test] + async fn health_check_err_on_non_json_body() { + let mut server = mockito::Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_body("nginx error page") + .create_async() + .await; + let client = make_client(&server.url()); + assert!(client.health_check().await.is_err()); + } +} diff --git a/offchain/tools/memory-memwal/src/error.rs b/offchain/tools/memory-memwal/src/error.rs new file mode 100644 index 0000000..99f71a6 --- /dev/null +++ b/offchain/tools/memory-memwal/src/error.rs @@ -0,0 +1,42 @@ +//! Shared error types for the MemWal tool crate. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub(crate) enum MemWalError { + #[error("Auth error: {0}")] + Auth(#[from] AuthError), + + #[error("HTTP request failed: {0}")] + Request(#[from] reqwest::Error), + + #[error("Server returned an error: {0}")] + Server(String), + + #[error("Job {0} failed on the server")] + JobFailed(String), + + #[error("Timed out waiting for job {0} to complete")] + Timeout(String), + + #[error("Configuration error: {0}")] + Config(String), + + #[error("rate-limited by relayer (retry after {retry_after_secs:?}s)")] + RateLimited { retry_after_secs: Option }, +} + +#[derive(Debug, Error)] +pub(crate) enum AuthError { + #[error("MEMWAL_DELEGATE_PRIVATE_KEY is not set or empty")] + MissingKey, + + #[error("Private key is not valid hex: {0}")] + InvalidHex(#[from] hex::FromHexError), + + #[error("Private key must be 32 bytes, got {0}")] + InvalidKeyLength(usize), + + #[error("System clock error: {0}")] + Clock(String), +} diff --git a/offchain/tools/memory-memwal/src/forget.rs b/offchain/tools/memory-memwal/src/forget.rs new file mode 100644 index 0000000..7b19088 --- /dev/null +++ b/offchain/tools/memory-memwal/src/forget.rs @@ -0,0 +1,194 @@ +//! # `xyz.taluslabs.memory.memwal.forget@1` +//! +//! Nexus Tool that deletes every memory in a MemWal namespace. The call is +//! owner-scoped — the relayer's auth middleware identifies the account from +//! the signed request; only memories belonging to that account are removed, +//! regardless of which namespace string is sent. + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// Namespace to clear. When omitted the server interprets it as + /// `"default"`. The relayer scopes the delete to the authenticated + /// account — you cannot delete other accounts' memories. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Namespace was cleared. `deleted` is the number of memories removed + /// (zero is a valid success — the namespace existed but was empty, or + /// did not exist at all). + Ok { + deleted: u64, + }, + Err { + reason: String, + }, +} + +pub(crate) struct ForgetMemories { + client: MemWalClient, +} + +impl NexusTool for ForgetMemories { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.forget@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/forget" + } + + fn description() -> &'static str { + "Delete every memory in a MemWal namespace owned by the authenticated account." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self.client.forget(input.namespace.as_deref()).await { + Ok(deleted) => Output::Ok { deleted }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> ForgetMemories { + ForgetMemories { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` surfaces the server's `deleted` count on a successful call. + /// Failure mode caught: server's deleted count is silently dropped or replaced with a constant. + #[tokio::test] + async fn invoke_returns_deleted_count_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/forget") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "deleted": 42, + "namespace": "scratch", + "owner": "0xowner" + }) + .to_string(), + ) + .create_async() + .await; + + match tool + .invoke(Input { + namespace: Some("scratch".into()), + }) + .await + { + Output::Ok { deleted } => assert_eq!(deleted, 42), + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok { deleted: 0 }` for an empty namespace — a valid success. + /// Failure mode caught: empty namespace mapped to Err, masking idempotent retries. + #[tokio::test] + async fn invoke_returns_zero_for_empty_namespace() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/forget") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"deleted": 0, "namespace": "default", "owner": "0xowner"}).to_string(), + ) + .create_async() + .await; + + match tool.invoke(Input { namespace: None }).await { + Output::Ok { deleted } => assert_eq!(deleted, 0), + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP 500 swallowed, returning a misleading zero count. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/forget") + .with_status(500) + .with_body("internal error") + .create_async() + .await; + + let output = tool.invoke(Input { namespace: None }).await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } +} diff --git a/offchain/tools/memory-memwal/src/main.rs b/offchain/tools/memory-memwal/src/main.rs new file mode 100644 index 0000000..e28676a --- /dev/null +++ b/offchain/tools/memory-memwal/src/main.rs @@ -0,0 +1,61 @@ +//! Nexus Tools for MemWal persistent memory. See `README.md` for the tool +//! list, env vars, and per-tool I/O contracts. +//! +//! Wire format pinned to `@mysten-incubation/memwal@0.0.4` (server Cargo +//! version `0.1.0`). [`client::MEMWAL_API_VERSION`] documents the +//! re-audit contract on tag bumps. + +use nexus_toolkit::bootstrap; + +mod analyze; +mod ask; +mod auth; +mod client; +mod error; +mod forget; +mod recall; +mod remember; +mod remember_bulk; +mod stats; + +fn main() { + // Install env_logger before anything else so dotenv/credential paths + // emit through `log::*`; `bootstrap!`'s own `try_init()` becomes a no-op. + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")) + .format_timestamp_millis() + .init(); + + // `--meta` is an introspection mode that emits the tool's FQN/URL/ + // schema list without standing up the runtime. It is invoked by the + // CI prepare step from a fresh Docker image with no env, so it must + // not require runtime credentials. Skip the credential validation + // path in this mode; nexus_toolkit's bootstrap! handles --meta and + // exits before any HTTP call is made. + let meta_only = std::env::args().any(|a| a == "--meta"); + + if !meta_only { + // dotenv + credential validation run single-threaded — `set_var` is + // unsound from a multi-threaded process. `main` is the only exit site. + client::load_dotenv_if_present(); + if let Err(reason) = client::validate_credentials_at_startup() { + log::error!("{} {reason}", client::ENV_PRIVATE_KEY); + std::process::exit(1); + } + } + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime") + .block_on(async { + bootstrap!([ + remember::RememberMemory, + remember_bulk::RememberBulkMemories, + recall::RecallMemories, + ask::AskMemory, + analyze::AnalyzeAndRemember, + forget::ForgetMemories, + stats::StatsForAccount, + ]) + }); +} diff --git a/offchain/tools/memory-memwal/src/recall.rs b/offchain/tools/memory-memwal/src/recall.rs new file mode 100644 index 0000000..d4e666d --- /dev/null +++ b/offchain/tools/memory-memwal/src/recall.rs @@ -0,0 +1,252 @@ +//! # `xyz.taluslabs.memory.memwal.recall@1` +//! +//! Nexus Tool that performs a semantic search over stored memories using a +//! natural-language query and returns the closest matches ranked by vector +//! distance (cosine similarity, lower = closer). + +use { + crate::client::{MemWalClient, MemoryResult}, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// Natural-language query to search for relevant memories. + query: String, + /// Maximum number of results to return. + limit: Option, + /// Namespace to search within. Searches the default namespace when omitted. + namespace: Option, +} + +/// A single memory returned by a recall query. +#[derive(Serialize, JsonSchema)] +pub(crate) struct RecalledMemory { + /// The stored text of the memory. + pub(crate) text: String, + /// Walrus blob identifier for the encrypted memory. + pub(crate) blob_id: String, + /// Cosine distance from the query vector — lower means more relevant. + pub(crate) distance: f64, +} + +impl From for RecalledMemory { + fn from(r: MemoryResult) -> Self { + Self { + text: r.text, + blob_id: r.blob_id, + distance: r.distance, + } + } +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Memories ranked by relevance. May be empty if nothing matched. + /// + /// The `namespace` field is the namespace that was searched, hoisted + /// to the variant since every result in a single recall belongs to + /// the same namespace (the relayer does not currently support + /// cross-namespace recall). + Ok { + results: Vec, + namespace: String, + }, + Err { + reason: String, + }, +} + +pub(crate) struct RecallMemories { + client: MemWalClient, +} + +impl NexusTool for RecallMemories { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.recall@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/recall" + } + + fn description() -> &'static str { + "Search stored memories by natural-language query and return the closest matches." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + let ns = input.namespace.as_deref().unwrap_or("default").to_string(); + + match self + .client + .recall(&input.query, input.limit, input.namespace.as_deref()) + .await + { + Ok(results) => Output::Ok { + results: results.into_iter().map(RecalledMemory::from).collect(), + namespace: ns, + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> RecallMemories { + RecallMemories { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns ranked results when the server responds with memories. + /// Failure mode caught: successful recall response incorrectly mapped to `Err`. + #[tokio::test] + async fn invoke_returns_results_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/recall") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "results": [ + { + "text": "Paris is the capital of France", + "blob_id": "blob-1", + "distance": 0.12 + } + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + query: "capital of France".into(), + limit: None, + namespace: None, + }) + .await; + match output { + Output::Ok { results, namespace } => { + assert_eq!(results.len(), 1); + assert_eq!(results[0].text, "Paris is the capital of France"); + assert_eq!(results[0].blob_id, "blob-1"); + assert!((results[0].distance - 0.12).abs() < f64::EPSILON); + assert_eq!(namespace, "default"); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok` with an empty list when no memories match. + /// Failure mode caught: empty result set treated as an error. + #[tokio::test] + async fn invoke_returns_empty_results_on_no_match() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/recall") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"results": []}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + query: "something obscure".into(), + limit: None, + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Ok { results, .. } if results.is_empty()), + "empty results must produce Ok with empty vec" + ); + } + + /// `invoke` returns `Err` on a server-side error. + /// Failure mode caught: HTTP 500 from the server swallowed, returns empty results. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/recall") + .with_status(500) + .with_body("internal error") + .create_async() + .await; + + let output = tool + .invoke(Input { + query: "anything".into(), + limit: None, + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } +} diff --git a/offchain/tools/memory-memwal/src/remember.rs b/offchain/tools/memory-memwal/src/remember.rs new file mode 100644 index 0000000..ddbc968 --- /dev/null +++ b/offchain/tools/memory-memwal/src/remember.rs @@ -0,0 +1,313 @@ +//! # `xyz.taluslabs.memory.memwal.remember@1` +//! +//! Nexus Tool that stores a single piece of text as a persistent memory in +//! MemWal. The call blocks until the memory is durably stored on Walrus and +//! returns the resulting blob ID, making it safe to chain in a Nexus DAG — +//! the next vertex will not activate until the write is confirmed. + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// The text to store as a memory. + text: String, + /// Namespace used to scope this memory. Defaults to `"default"` on the + /// server when omitted. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Memory was durably stored. `blob_id` is its Walrus blob identifier. + Ok { + blob_id: String, + }, + Err { + reason: String, + }, +} + +pub(crate) struct RememberMemory { + client: MemWalClient, +} + +impl NexusTool for RememberMemory { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.remember@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/remember" + } + + fn description() -> &'static str { + "Store a text memory in MemWal and return its blob ID once durably written." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + let job_id = match self + .client + .remember(&input.text, input.namespace.as_deref()) + .await + { + Ok(id) => id, + Err(e) => { + return Output::Err { + reason: e.to_string(), + } + } + }; + + match self.client.poll_job(&job_id).await { + Ok(blob_id) => Output::Ok { blob_id }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + mockito::{Matcher, Server}, + serde_json::json, + }; + + fn make_tool(server_url: &str) -> RememberMemory { + RememberMemory { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's `NexusTool::health` wrapper drops the + /// validate_key + health_check composition (e.g. a refactor that forgets + /// to await the inner future) — only an end-to-end test through the trait + /// catches that. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` returns `Ok { blob_id }` when the job completes successfully. + /// Failure mode caught: successful server response mapped to `Err` variant. + #[tokio::test] + async fn invoke_returns_blob_id_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-abc", "status": "pending"}).to_string()) + .create_async() + .await; + + let _m2 = server + .mock("GET", "/api/remember/job-abc") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"job_id": "job-abc", "status": "done", "blob_id": "blob-xyz"}).to_string(), + ) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "hello world".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Ok { blob_id } if blob_id == "blob-xyz"), + "expected Ok{{blob_id: blob-xyz}}" + ); + } + + /// `invoke` returns `Err` when the server signals job failure. + /// Failure mode caught: failed job silently returns empty blob_id instead of error. + #[tokio::test] + async fn invoke_returns_err_on_job_failure() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-fail", "status": "pending"}).to_string()) + .create_async() + .await; + + let _m2 = server + .mock("GET", "/api/remember/job-fail") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-fail", "status": "failed"}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "will fail".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "failed job must produce Err variant" + ); + } + + /// `invoke` returns `Err` when the POST to `/api/remember` itself fails (4xx/5xx). + /// Failure mode caught: HTTP error from the initial POST is swallowed. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/remember") + .with_status(500) + .with_body("internal error") + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "text that will not be stored".into(), + namespace: None, + }) + .await; + assert!( + matches!(output, Output::Err { .. }), + "server 500 must produce Err variant" + ); + } + + /// `invoke` returns `Err` when the server replies `"done"` without a + /// `blob_id` field — the missing identifier must NOT be silently coerced + /// to an empty string. + /// Failure mode caught: a regression of `unwrap_or_default()` in + /// `poll_job` would propagate `blob_id: ""` to downstream DAG vertices + /// as if the write had succeeded. + #[tokio::test] + async fn invoke_returns_err_when_done_missing_blob_id() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-noid", "status": "pending"}).to_string()) + .create_async() + .await; + + // status=done but no blob_id field. + let _m2 = server + .mock("GET", "/api/remember/job-noid") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "job-noid", "status": "done"}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "text".into(), + namespace: None, + }) + .await; + match output { + Output::Err { reason } => { + assert!( + reason.contains("blob_id"), + "error must mention missing blob_id; got: {reason}" + ); + } + Output::Ok { blob_id } => { + panic!("expected Err, got Ok(blob_id={blob_id:?}); silent empty-blob bug") + } + } + } + + /// `invoke` sends the `text` field in the POST body. + /// Failure mode caught: body serialisation drops the `text` field. + #[tokio::test] + async fn invoke_sends_correct_body() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m1 = server + .mock("POST", "/api/remember") + .match_body(Matcher::PartialJson(json!({"text": "exact text"}))) + .with_status(202) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "j1", "status": "pending"}).to_string()) + .create_async() + .await; + + let _m2 = server + .mock("GET", "/api/remember/j1") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(json!({"job_id": "j1", "status": "done", "blob_id": "b1"}).to_string()) + .create_async() + .await; + + let output = tool + .invoke(Input { + text: "exact text".into(), + namespace: None, + }) + .await; + assert!(matches!(output, Output::Ok { .. })); + } +} diff --git a/offchain/tools/memory-memwal/src/remember_bulk.rs b/offchain/tools/memory-memwal/src/remember_bulk.rs new file mode 100644 index 0000000..06dd433 --- /dev/null +++ b/offchain/tools/memory-memwal/src/remember_bulk.rs @@ -0,0 +1,330 @@ +//! # `xyz.taluslabs.memory.memwal.remember_bulk@1` +//! +//! Store up to 20 memories in one batched call. Returns blob_ids in input +//! order. ~10× more rate-limit-efficient than N separate `remember` calls +//! (weight 10 for up to 20 items vs 5 each). + +use { + crate::client::{MemWalClient, MAX_BULK_ITEMS}, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +/// One item in the bulk request. `namespace` is per-item: a single bulk call +/// can write to multiple namespaces in one go. +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct BulkItem { + /// The text to store. + text: String, + /// Namespace for this specific item. Defaults to `"default"` on the + /// server when omitted. + namespace: Option, +} + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// 1–20 items to store. The relayer rejects empty arrays and any batch + /// over MAX_BULK_ITEMS = 20. + items: Vec, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Every item was durably stored. `blob_ids` aligns positionally with the + /// input `items` array (i.e. `blob_ids[i]` is the blob for `items[i]`). + Ok { + blob_ids: Vec, + }, + Err { + reason: String, + }, +} + +pub(crate) struct RememberBulkMemories { + client: MemWalClient, +} + +impl NexusTool for RememberBulkMemories { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.remember_bulk@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/remember_bulk" + } + + fn description() -> &'static str { + "Store up to 20 memories in one batched call; returns one blob_id per item." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + // Validate batch size before signing anything — empty arrays waste a + // signed request slot; oversized batches surface as opaque HTTP 400 + // from the relayer without this guard. + if input.items.is_empty() { + return Output::Err { + reason: "bulk batch must contain at least 1 item".into(), + }; + } + if input.items.len() > MAX_BULK_ITEMS { + return Output::Err { + reason: format!( + "bulk batch capped at {MAX_BULK_ITEMS} items, got {}", + input.items.len() + ), + }; + } + + // Map our owned `Vec` into the borrowed `(&str, Option<&str>)` + // tuple shape the client expects. This avoids `Item<'_>` lifetime + // gymnastics by keeping the inputs alive in `items_refs`. + let items_refs: Vec<(&str, Option<&str>)> = input + .items + .iter() + .map(|i| (i.text.as_str(), i.namespace.as_deref())) + .collect(); + + let job_ids = match self.client.remember_bulk(&items_refs).await { + Ok(ids) => ids, + Err(e) => { + return Output::Err { + reason: e.to_string(), + }; + } + }; + + match self.client.poll_bulk_jobs(&job_ids).await { + Ok(blob_ids) => Output::Ok { blob_ids }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> RememberBulkMemories { + RememberBulkMemories { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + fn bulk_input(texts: &[&str]) -> Input { + Input { + items: texts + .iter() + .map(|t| BulkItem { + text: t.to_string(), + namespace: None, + }) + .collect(), + } + } + + /// `invoke` returns `Ok { blob_ids }` aligned with the input order when + /// every job reaches "done". + /// Failure mode caught: blob_ids returned in random order, breaking + /// callers that index-correlate items with results. + #[tokio::test] + async fn invoke_returns_blob_ids_in_input_order() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _bulk = server + .mock("POST", "/api/remember/bulk") + .with_status(202) + .with_header("content-type", "application/json") + .with_body( + json!({ + "job_ids": ["j1", "j2", "j3"], + "total": 3, + "status": "running" + }) + .to_string(), + ) + .create_async() + .await; + + // Server returns results in reversed order intentionally — the tool + // must reorder by job_id to align with the input order. + let _status = server + .mock("POST", "/api/remember/bulk/status") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "results": [ + {"job_id": "j3", "status": "done", "blob_id": "blob3"}, + {"job_id": "j2", "status": "done", "blob_id": "blob2"}, + {"job_id": "j1", "status": "done", "blob_id": "blob1"}, + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool.invoke(bulk_input(&["one", "two", "three"])).await; + match output { + Output::Ok { blob_ids } => { + assert_eq!(blob_ids, vec!["blob1", "blob2", "blob3"]); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Err` when any individual job ends `failed`. + /// Failure mode caught: a partial failure is silently presented as + /// success with truncated blob_ids, hiding data loss. + #[tokio::test] + async fn invoke_returns_err_on_any_job_failure() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _bulk = server + .mock("POST", "/api/remember/bulk") + .with_status(202) + .with_header("content-type", "application/json") + .with_body( + json!({"job_ids": ["jA", "jB"], "total": 2, "status": "running"}).to_string(), + ) + .create_async() + .await; + + let _status = server + .mock("POST", "/api/remember/bulk/status") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "results": [ + {"job_id": "jA", "status": "done", "blob_id": "blobA"}, + {"job_id": "jB", "status": "failed", "error": "encrypt timeout"}, + ] + }) + .to_string(), + ) + .create_async() + .await; + + let output = tool.invoke(bulk_input(&["a", "b"])).await; + assert!( + matches!(output, Output::Err { .. }), + "any failed job must produce Err" + ); + } + + /// `invoke` returns `Err` for an empty `items` array without hitting the network. + /// Failure mode caught: an empty batch silently round-trips, burning a + /// signed request slot and a rate-limit point for an opaque 400. + #[tokio::test] + async fn invoke_returns_err_on_empty_items() { + let server = Server::new_async().await; + let tool = make_tool(&server.url()); + let output = tool.invoke(bulk_input(&[])).await; + match output { + Output::Err { reason } => { + assert!( + reason.contains("1 item"), + "reason must mention the lower bound; got: {reason}" + ); + } + Output::Ok { .. } => panic!("empty batch must produce Err"), + } + } + + /// `invoke` returns `Err` for a batch of MAX_BULK_ITEMS+1 items without + /// contacting the relayer. + /// Failure mode caught: oversized batch reaches the relayer and gets an + /// opaque HTTP 400 instead of a tool-side cap explanation. + #[tokio::test] + async fn invoke_returns_err_on_oversized_batch() { + let server = Server::new_async().await; + let tool = make_tool(&server.url()); + let texts: Vec = (0..MAX_BULK_ITEMS + 1) + .map(|i| format!("item-{i}")) + .collect(); + let refs: Vec<&str> = texts.iter().map(String::as_str).collect(); + let output = tool.invoke(bulk_input(&refs)).await; + match output { + Output::Err { reason } => { + assert!( + reason.contains(&MAX_BULK_ITEMS.to_string()), + "reason must mention the cap; got: {reason}" + ); + } + Output::Ok { .. } => panic!("oversized batch must produce Err"), + } + } + + /// `invoke` returns `Err` when the initial bulk POST fails (4xx/5xx). + /// Failure mode caught: the initial-submit error is swallowed. + #[tokio::test] + async fn invoke_returns_err_on_submit_failure() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _bulk = server + .mock("POST", "/api/remember/bulk") + .with_status(400) + .with_body("too many items") + .create_async() + .await; + + let output = tool.invoke(bulk_input(&["x"])).await; + assert!( + matches!(output, Output::Err { .. }), + "submit-time 400 must produce Err" + ); + } +} diff --git a/offchain/tools/memory-memwal/src/stats.rs b/offchain/tools/memory-memwal/src/stats.rs new file mode 100644 index 0000000..be5db1f --- /dev/null +++ b/offchain/tools/memory-memwal/src/stats.rs @@ -0,0 +1,226 @@ +//! # `xyz.taluslabs.memory.memwal.stats@1` +//! +//! Nexus Tool that reports per-namespace usage for the authenticated MemWal +//! account: how many memories are stored and how many bytes they take. + +use { + crate::client::MemWalClient, + nexus_sdk::{fqn, ToolFqn}, + nexus_toolkit::*, + schemars::JsonSchema, + serde::{Deserialize, Serialize}, +}; + +#[derive(Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub(crate) struct Input { + /// Namespace to report on. Defaults to `"default"` on the server when + /// omitted. Only namespaces owned by the authenticated account return + /// meaningful data. + namespace: Option, +} + +#[derive(Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum Output { + /// Per-namespace usage figures. + Ok { + /// Number of memories stored in this namespace. + memory_count: i64, + /// Total encrypted byte size of those memories on Walrus. + storage_bytes: i64, + /// The resolved namespace (mirrors what the server interpreted). + namespace: String, + }, + Err { + reason: String, + }, +} + +pub(crate) struct StatsForAccount { + client: MemWalClient, +} + +impl NexusTool for StatsForAccount { + type Input = Input; + type Output = Output; + + async fn new() -> Self { + let client = MemWalClient::from_env().unwrap_or_else(|e| { + log::error!("relayer configuration invalid: {e}"); + panic!("relayer configuration invalid: {e}") + }); + Self { client } + } + + fn fqn() -> ToolFqn { + fqn!(concat!( + "xyz.taluslabs.memory.memwal.stats@", + env!("TOOL_FQN_VERSION") + )) + } + + fn path() -> &'static str { + "/memory/stats" + } + + fn description() -> &'static str { + "Report memory count and total stored bytes for a MemWal namespace." + } + + async fn health(&self) -> AnyResult { + self.client.validate_key().map_err(|e| anyhow::anyhow!(e))?; + self.client + .health_check() + .await + .map_err(|e| anyhow::anyhow!(e))?; + Ok(StatusCode::OK) + } + + async fn invoke(&self, input: Self::Input) -> Self::Output { + match self.client.stats(input.namespace.as_deref()).await { + Ok(s) => Output::Ok { + memory_count: s.memory_count, + storage_bytes: s.storage_bytes, + namespace: s.namespace, + }, + Err(e) => Output::Err { + reason: e.to_string(), + }, + } + } +} + +#[cfg(test)] +mod tests { + use {super::*, mockito::Server, serde_json::json}; + + fn make_tool(server_url: &str) -> StatsForAccount { + StatsForAccount { + client: MemWalClient::with_test_config(server_url, &hex::encode([0x42u8; 32]), ""), + } + } + + /// `health()` returns `Ok` when the relayer reports the matching API version. + /// Failure mode caught: the tool's NexusTool::health wrapper drops the + /// validate_key + health_check composition. + #[tokio::test] + async fn health_returns_ok_when_relayer_healthy() { + let mut server = Server::new_async().await; + let _m = server + .mock("GET", "/health") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({"status": "ok", "version": crate::client::MEMWAL_API_VERSION}).to_string(), + ) + .create_async() + .await; + let tool = make_tool(&server.url()); + assert!(tool.health().await.is_ok()); + } + + /// `invoke` echoes the server's count + bytes verbatim on a healthy response. + /// Failure mode caught: numeric fields silently zeroed or truncated on the boundary. + #[tokio::test] + async fn invoke_returns_counts_on_success() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/stats") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "memory_count": 17, + "storage_bytes": 5_242_880, + "namespace": "people", + "owner": "0xowner" + }) + .to_string(), + ) + .create_async() + .await; + + match tool + .invoke(Input { + namespace: Some("people".into()), + }) + .await + { + Output::Ok { + memory_count, + storage_bytes, + namespace, + } => { + assert_eq!(memory_count, 17); + assert_eq!(storage_bytes, 5_242_880); + assert_eq!(namespace, "people"); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Ok` with zeros for a namespace that exists but is empty. + /// Failure mode caught: zero-memory namespaces misclassified as Err. + #[tokio::test] + async fn invoke_handles_zero_counts() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/stats") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + json!({ + "memory_count": 0, + "storage_bytes": 0, + "namespace": "empty-ns", + "owner": "0xowner" + }) + .to_string(), + ) + .create_async() + .await; + + match tool + .invoke(Input { + namespace: Some("empty-ns".into()), + }) + .await + { + Output::Ok { + memory_count, + storage_bytes, + .. + } => { + assert_eq!(memory_count, 0); + assert_eq!(storage_bytes, 0); + } + Output::Err { reason } => panic!("unexpected Err: {reason}"), + } + } + + /// `invoke` returns `Err` on server error. + /// Failure mode caught: HTTP 500 swallowed, returning misleading zero stats. + #[tokio::test] + async fn invoke_returns_err_on_server_error() { + let mut server = Server::new_async().await; + let tool = make_tool(&server.url()); + + let _m = server + .mock("POST", "/api/stats") + .with_status(503) + .with_body("service unavailable") + .create_async() + .await; + + let output = tool.invoke(Input { namespace: None }).await; + assert!( + matches!(output, Output::Err { .. }), + "server 503 must produce Err variant" + ); + } +} diff --git a/offchain/tools/memory-memwal/tools.json b/offchain/tools/memory-memwal/tools.json new file mode 100644 index 0000000..8924f64 --- /dev/null +++ b/offchain/tools/memory-memwal/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "memory-memwal", + "command": "memory-memwal", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/payments-stripe/AUDIT.md b/offchain/tools/payments-stripe/AUDIT.md similarity index 100% rename from tools/payments-stripe/AUDIT.md rename to offchain/tools/payments-stripe/AUDIT.md diff --git a/tools/payments-stripe/Cargo.toml b/offchain/tools/payments-stripe/Cargo.toml similarity index 84% rename from tools/payments-stripe/Cargo.toml rename to offchain/tools/payments-stripe/Cargo.toml index 6c9f775..e953082 100644 --- a/tools/payments-stripe/Cargo.toml +++ b/offchain/tools/payments-stripe/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "payments-stripe" +path = "src/main.rs" + [dependencies] thiserror.workspace = true tokio.workspace = true @@ -26,3 +30,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/payments-stripe/README.md b/offchain/tools/payments-stripe/README.md similarity index 100% rename from tools/payments-stripe/README.md rename to offchain/tools/payments-stripe/README.md diff --git a/offchain/tools/payments-stripe/build.rs b/offchain/tools/payments-stripe/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/payments-stripe/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/payments-stripe/src/error.rs b/offchain/tools/payments-stripe/src/error.rs similarity index 100% rename from tools/payments-stripe/src/error.rs rename to offchain/tools/payments-stripe/src/error.rs diff --git a/tools/payments-stripe/src/main.rs b/offchain/tools/payments-stripe/src/main.rs similarity index 100% rename from tools/payments-stripe/src/main.rs rename to offchain/tools/payments-stripe/src/main.rs diff --git a/tools/payments-stripe/src/stripe_client.rs b/offchain/tools/payments-stripe/src/stripe_client.rs similarity index 100% rename from tools/payments-stripe/src/stripe_client.rs rename to offchain/tools/payments-stripe/src/stripe_client.rs diff --git a/tools/payments-stripe/src/tools/confirm_payment_intent.rs b/offchain/tools/payments-stripe/src/tools/confirm_payment_intent.rs similarity index 97% rename from tools/payments-stripe/src/tools/confirm_payment_intent.rs rename to offchain/tools/payments-stripe/src/tools/confirm_payment_intent.rs index bc91586..3b01790 100644 --- a/tools/payments-stripe/src/tools/confirm_payment_intent.rs +++ b/offchain/tools/payments-stripe/src/tools/confirm_payment_intent.rs @@ -63,7 +63,10 @@ impl NexusTool for ConfirmPaymentIntent { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.confirm-payment-intent@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.confirm-payment-intent@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/create_customer.rs b/offchain/tools/payments-stripe/src/tools/create_customer.rs similarity index 97% rename from tools/payments-stripe/src/tools/create_customer.rs rename to offchain/tools/payments-stripe/src/tools/create_customer.rs index 84c3413..cbf9afb 100644 --- a/tools/payments-stripe/src/tools/create_customer.rs +++ b/offchain/tools/payments-stripe/src/tools/create_customer.rs @@ -57,7 +57,10 @@ impl NexusTool for CreateCustomer { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.create-customer@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.create-customer@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/create_payment_intent.rs b/offchain/tools/payments-stripe/src/tools/create_payment_intent.rs similarity index 98% rename from tools/payments-stripe/src/tools/create_payment_intent.rs rename to offchain/tools/payments-stripe/src/tools/create_payment_intent.rs index f30fa57..0ddc107 100644 --- a/tools/payments-stripe/src/tools/create_payment_intent.rs +++ b/offchain/tools/payments-stripe/src/tools/create_payment_intent.rs @@ -73,7 +73,10 @@ impl NexusTool for CreatePaymentIntent { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.create-payment-intent@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.create-payment-intent@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/get_balance.rs b/offchain/tools/payments-stripe/src/tools/get_balance.rs similarity index 96% rename from tools/payments-stripe/src/tools/get_balance.rs rename to offchain/tools/payments-stripe/src/tools/get_balance.rs index afec74c..0c06785 100644 --- a/tools/payments-stripe/src/tools/get_balance.rs +++ b/offchain/tools/payments-stripe/src/tools/get_balance.rs @@ -50,7 +50,10 @@ impl NexusTool for GetBalance { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.get-balance@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.get-balance@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/get_payment_intent.rs b/offchain/tools/payments-stripe/src/tools/get_payment_intent.rs similarity index 97% rename from tools/payments-stripe/src/tools/get_payment_intent.rs rename to offchain/tools/payments-stripe/src/tools/get_payment_intent.rs index cd5a084..c059a6d 100644 --- a/tools/payments-stripe/src/tools/get_payment_intent.rs +++ b/offchain/tools/payments-stripe/src/tools/get_payment_intent.rs @@ -58,7 +58,10 @@ impl NexusTool for GetPaymentIntent { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.get-payment-intent@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.get-payment-intent@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/list_charges.rs b/offchain/tools/payments-stripe/src/tools/list_charges.rs similarity index 97% rename from tools/payments-stripe/src/tools/list_charges.rs rename to offchain/tools/payments-stripe/src/tools/list_charges.rs index 60177e0..3da217a 100644 --- a/tools/payments-stripe/src/tools/list_charges.rs +++ b/offchain/tools/payments-stripe/src/tools/list_charges.rs @@ -53,7 +53,10 @@ impl NexusTool for ListCharges { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.payments.stripe.list-charges@1") + fqn!(concat!( + "xyz.taluslabs.payments.stripe.list-charges@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/payments-stripe/src/tools/mod.rs b/offchain/tools/payments-stripe/src/tools/mod.rs similarity index 100% rename from tools/payments-stripe/src/tools/mod.rs rename to offchain/tools/payments-stripe/src/tools/mod.rs diff --git a/tools/payments-stripe/src/tools/models.rs b/offchain/tools/payments-stripe/src/tools/models.rs similarity index 100% rename from tools/payments-stripe/src/tools/models.rs rename to offchain/tools/payments-stripe/src/tools/models.rs diff --git a/offchain/tools/payments-stripe/tools.json b/offchain/tools/payments-stripe/tools.json new file mode 100644 index 0000000..bdcfd30 --- /dev/null +++ b/offchain/tools/payments-stripe/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "payments-stripe", + "command": "payments-stripe", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/social-twitter/Cargo.toml b/offchain/tools/social-twitter/Cargo.toml similarity index 84% rename from tools/social-twitter/Cargo.toml rename to offchain/tools/social-twitter/Cargo.toml index 0c08118..7d09c74 100644 --- a/tools/social-twitter/Cargo.toml +++ b/offchain/tools/social-twitter/Cargo.toml @@ -11,6 +11,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "social-twitter" +path = "src/main.rs" + [dependencies] oauth1-request = "0.6.1" base64 = "0.13.0" @@ -29,3 +33,7 @@ nexus-sdk.workspace = true [dev-dependencies] mockito = "1.7.0" + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/social-twitter/README.md b/offchain/tools/social-twitter/README.md similarity index 100% rename from tools/social-twitter/README.md rename to offchain/tools/social-twitter/README.md diff --git a/offchain/tools/social-twitter/build.rs b/offchain/tools/social-twitter/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/social-twitter/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/social-twitter/src/auth.rs b/offchain/tools/social-twitter/src/auth.rs similarity index 100% rename from tools/social-twitter/src/auth.rs rename to offchain/tools/social-twitter/src/auth.rs diff --git a/tools/social-twitter/src/direct_message/create_group_conversation.rs b/offchain/tools/social-twitter/src/direct_message/create_group_conversation.rs similarity index 99% rename from tools/social-twitter/src/direct_message/create_group_conversation.rs rename to offchain/tools/social-twitter/src/direct_message/create_group_conversation.rs index 7dd0dd0..0b32d11 100644 --- a/tools/social-twitter/src/direct_message/create_group_conversation.rs +++ b/offchain/tools/social-twitter/src/direct_message/create_group_conversation.rs @@ -66,7 +66,10 @@ impl NexusTool for CreateGroupDmConversation { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.create-group-conversation@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.create-group-conversation@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/get_conversation_messages.rs b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages.rs similarity index 98% rename from tools/social-twitter/src/direct_message/get_conversation_messages.rs rename to offchain/tools/social-twitter/src/direct_message/get_conversation_messages.rs index 962a30b..1d1d6a4 100644 --- a/tools/social-twitter/src/direct_message/get_conversation_messages.rs +++ b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages.rs @@ -160,7 +160,10 @@ impl NexusTool for GetConversationMessages { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-conversation-messages@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-conversation-messages@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs similarity index 99% rename from tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs rename to offchain/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs index 899b09f..64dbb86 100644 --- a/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs +++ b/offchain/tools/social-twitter/src/direct_message/get_conversation_messages_by_id.rs @@ -166,7 +166,10 @@ impl NexusTool for GetConversationMessagesById { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-conversation-messages-by-id@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-conversation-messages-by-id@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/mod.rs b/offchain/tools/social-twitter/src/direct_message/mod.rs similarity index 100% rename from tools/social-twitter/src/direct_message/mod.rs rename to offchain/tools/social-twitter/src/direct_message/mod.rs diff --git a/tools/social-twitter/src/direct_message/models.rs b/offchain/tools/social-twitter/src/direct_message/models.rs similarity index 100% rename from tools/social-twitter/src/direct_message/models.rs rename to offchain/tools/social-twitter/src/direct_message/models.rs diff --git a/tools/social-twitter/src/direct_message/send_direct_message.rs b/offchain/tools/social-twitter/src/direct_message/send_direct_message.rs similarity index 98% rename from tools/social-twitter/src/direct_message/send_direct_message.rs rename to offchain/tools/social-twitter/src/direct_message/send_direct_message.rs index 87ee63c..3a714d5 100644 --- a/tools/social-twitter/src/direct_message/send_direct_message.rs +++ b/offchain/tools/social-twitter/src/direct_message/send_direct_message.rs @@ -66,7 +66,10 @@ impl NexusTool for SendDirectMessage { // /:participant_id/dm_events fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.send-direct-message@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.send-direct-message@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs b/offchain/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs similarity index 99% rename from tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs rename to offchain/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs index 6b1bc53..853cfff 100644 --- a/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs +++ b/offchain/tools/social-twitter/src/direct_message/send_message_to_group_conversation.rs @@ -63,7 +63,10 @@ impl NexusTool for SendMessageToGroupConversation { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.send-message-to-group-conversation@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.send-message-to-group-conversation@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/error.rs b/offchain/tools/social-twitter/src/error.rs similarity index 100% rename from tools/social-twitter/src/error.rs rename to offchain/tools/social-twitter/src/error.rs diff --git a/tools/social-twitter/src/list/add_member.rs b/offchain/tools/social-twitter/src/list/add_member.rs similarity index 98% rename from tools/social-twitter/src/list/add_member.rs rename to offchain/tools/social-twitter/src/list/add_member.rs index 1b1f75d..b30c1d5 100644 --- a/tools/social-twitter/src/list/add_member.rs +++ b/offchain/tools/social-twitter/src/list/add_member.rs @@ -51,7 +51,10 @@ impl NexusTool for AddMember { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.add-member@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.add-member@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/create_list.rs b/offchain/tools/social-twitter/src/list/create_list.rs similarity index 99% rename from tools/social-twitter/src/list/create_list.rs rename to offchain/tools/social-twitter/src/list/create_list.rs index ac839e9..43fefe1 100644 --- a/tools/social-twitter/src/list/create_list.rs +++ b/offchain/tools/social-twitter/src/list/create_list.rs @@ -67,7 +67,10 @@ impl NexusTool for CreateList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.create-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.create-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/delete_list.rs b/offchain/tools/social-twitter/src/list/delete_list.rs similarity index 98% rename from tools/social-twitter/src/list/delete_list.rs rename to offchain/tools/social-twitter/src/list/delete_list.rs index 35e75f6..bc33477 100644 --- a/tools/social-twitter/src/list/delete_list.rs +++ b/offchain/tools/social-twitter/src/list/delete_list.rs @@ -57,7 +57,10 @@ impl NexusTool for DeleteList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.delete-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.delete-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_list.rs b/offchain/tools/social-twitter/src/list/get_list.rs similarity index 99% rename from tools/social-twitter/src/list/get_list.rs rename to offchain/tools/social-twitter/src/list/get_list.rs index fe36a2b..7a3a928 100644 --- a/tools/social-twitter/src/list/get_list.rs +++ b/offchain/tools/social-twitter/src/list/get_list.rs @@ -94,7 +94,10 @@ impl NexusTool for GetList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_list_members.rs b/offchain/tools/social-twitter/src/list/get_list_members.rs similarity index 99% rename from tools/social-twitter/src/list/get_list_members.rs rename to offchain/tools/social-twitter/src/list/get_list_members.rs index 6511085..51439f6 100644 --- a/tools/social-twitter/src/list/get_list_members.rs +++ b/offchain/tools/social-twitter/src/list/get_list_members.rs @@ -83,7 +83,10 @@ impl NexusTool for GetListMembers { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-list-members@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-list-members@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_list_tweets.rs b/offchain/tools/social-twitter/src/list/get_list_tweets.rs similarity index 99% rename from tools/social-twitter/src/list/get_list_tweets.rs rename to offchain/tools/social-twitter/src/list/get_list_tweets.rs index 4c40051..d7c89da 100644 --- a/tools/social-twitter/src/list/get_list_tweets.rs +++ b/offchain/tools/social-twitter/src/list/get_list_tweets.rs @@ -97,7 +97,10 @@ impl NexusTool for GetListTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-list-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-list-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/get_user_lists.rs b/offchain/tools/social-twitter/src/list/get_user_lists.rs similarity index 99% rename from tools/social-twitter/src/list/get_user_lists.rs rename to offchain/tools/social-twitter/src/list/get_user_lists.rs index 6fdeeb2..92f0466 100644 --- a/tools/social-twitter/src/list/get_user_lists.rs +++ b/offchain/tools/social-twitter/src/list/get_user_lists.rs @@ -85,7 +85,10 @@ impl NexusTool for GetUserLists { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-lists@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-lists@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/mod.rs b/offchain/tools/social-twitter/src/list/mod.rs similarity index 100% rename from tools/social-twitter/src/list/mod.rs rename to offchain/tools/social-twitter/src/list/mod.rs diff --git a/tools/social-twitter/src/list/models.rs b/offchain/tools/social-twitter/src/list/models.rs similarity index 100% rename from tools/social-twitter/src/list/models.rs rename to offchain/tools/social-twitter/src/list/models.rs diff --git a/tools/social-twitter/src/list/remove_member.rs b/offchain/tools/social-twitter/src/list/remove_member.rs similarity index 98% rename from tools/social-twitter/src/list/remove_member.rs rename to offchain/tools/social-twitter/src/list/remove_member.rs index 4c834d6..1ee6c1d 100644 --- a/tools/social-twitter/src/list/remove_member.rs +++ b/offchain/tools/social-twitter/src/list/remove_member.rs @@ -51,7 +51,10 @@ impl NexusTool for RemoveMember { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.remove-member@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.remove-member@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/list/update_list.rs b/offchain/tools/social-twitter/src/list/update_list.rs similarity index 99% rename from tools/social-twitter/src/list/update_list.rs rename to offchain/tools/social-twitter/src/list/update_list.rs index 0482775..6f90f27 100644 --- a/tools/social-twitter/src/list/update_list.rs +++ b/offchain/tools/social-twitter/src/list/update_list.rs @@ -61,7 +61,10 @@ impl NexusTool for UpdateList { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.update-list@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.update-list@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/main.rs b/offchain/tools/social-twitter/src/main.rs similarity index 100% rename from tools/social-twitter/src/main.rs rename to offchain/tools/social-twitter/src/main.rs diff --git a/tools/social-twitter/src/media/mod.rs b/offchain/tools/social-twitter/src/media/mod.rs similarity index 100% rename from tools/social-twitter/src/media/mod.rs rename to offchain/tools/social-twitter/src/media/mod.rs diff --git a/tools/social-twitter/src/media/models.rs b/offchain/tools/social-twitter/src/media/models.rs similarity index 100% rename from tools/social-twitter/src/media/models.rs rename to offchain/tools/social-twitter/src/media/models.rs diff --git a/tools/social-twitter/src/media/upload_media.rs b/offchain/tools/social-twitter/src/media/upload_media.rs similarity index 99% rename from tools/social-twitter/src/media/upload_media.rs rename to offchain/tools/social-twitter/src/media/upload_media.rs index f499fd1..4665d08 100644 --- a/tools/social-twitter/src/media/upload_media.rs +++ b/offchain/tools/social-twitter/src/media/upload_media.rs @@ -132,7 +132,10 @@ impl NexusTool for UploadMedia { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.upload-media@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.upload-media@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/delete_tweet.rs b/offchain/tools/social-twitter/src/tweet/delete_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/delete_tweet.rs rename to offchain/tools/social-twitter/src/tweet/delete_tweet.rs index 7c7ca92..505579c 100644 --- a/tools/social-twitter/src/tweet/delete_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/delete_tweet.rs @@ -58,7 +58,10 @@ impl NexusTool for DeleteTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.delete-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.delete-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_mentioned_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_mentioned_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_mentioned_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_mentioned_tweets.rs index 2f4a8ed..ebb6345 100644 --- a/tools/social-twitter/src/tweet/get_mentioned_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_mentioned_tweets.rs @@ -138,7 +138,10 @@ impl NexusTool for GetMentionedTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-mentioned-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-mentioned-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_recent_search_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_recent_search_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_recent_search_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_recent_search_tweets.rs index 96e7494..c4f6c05 100644 --- a/tools/social-twitter/src/tweet/get_recent_search_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_recent_search_tweets.rs @@ -209,7 +209,10 @@ impl NexusTool for GetRecentSearchTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-recent-search-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-recent-search-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_recent_tweet_count.rs b/offchain/tools/social-twitter/src/tweet/get_recent_tweet_count.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_recent_tweet_count.rs rename to offchain/tools/social-twitter/src/tweet/get_recent_tweet_count.rs index ba24a28..3a163f1 100644 --- a/tools/social-twitter/src/tweet/get_recent_tweet_count.rs +++ b/offchain/tools/social-twitter/src/tweet/get_recent_tweet_count.rs @@ -124,7 +124,10 @@ impl NexusTool for GetRecentTweetCount { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-recent-tweet-count@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-recent-tweet-count@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_tweet.rs b/offchain/tools/social-twitter/src/tweet/get_tweet.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_tweet.rs rename to offchain/tools/social-twitter/src/tweet/get_tweet.rs index d4a494e..f17a1f7 100644 --- a/tools/social-twitter/src/tweet/get_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/get_tweet.rs @@ -168,7 +168,10 @@ impl NexusTool for GetTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_tweets.rs index c6101ca..c6ad6aa 100644 --- a/tools/social-twitter/src/tweet/get_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_tweets.rs @@ -117,7 +117,10 @@ impl NexusTool for GetTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/get_user_tweets.rs b/offchain/tools/social-twitter/src/tweet/get_user_tweets.rs similarity index 99% rename from tools/social-twitter/src/tweet/get_user_tweets.rs rename to offchain/tools/social-twitter/src/tweet/get_user_tweets.rs index 78be040..535253d 100644 --- a/tools/social-twitter/src/tweet/get_user_tweets.rs +++ b/offchain/tools/social-twitter/src/tweet/get_user_tweets.rs @@ -144,7 +144,10 @@ impl NexusTool for GetUserTweets { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-tweets@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-tweets@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/like_tweet.rs b/offchain/tools/social-twitter/src/tweet/like_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/like_tweet.rs rename to offchain/tools/social-twitter/src/tweet/like_tweet.rs index 76b787e..0b84e48 100644 --- a/tools/social-twitter/src/tweet/like_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/like_tweet.rs @@ -56,7 +56,10 @@ impl NexusTool for LikeTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.like-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.like-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/mod.rs b/offchain/tools/social-twitter/src/tweet/mod.rs similarity index 100% rename from tools/social-twitter/src/tweet/mod.rs rename to offchain/tools/social-twitter/src/tweet/mod.rs diff --git a/tools/social-twitter/src/tweet/models.rs b/offchain/tools/social-twitter/src/tweet/models.rs similarity index 100% rename from tools/social-twitter/src/tweet/models.rs rename to offchain/tools/social-twitter/src/tweet/models.rs diff --git a/tools/social-twitter/src/tweet/post_tweet.rs b/offchain/tools/social-twitter/src/tweet/post_tweet.rs similarity index 99% rename from tools/social-twitter/src/tweet/post_tweet.rs rename to offchain/tools/social-twitter/src/tweet/post_tweet.rs index f3cc30b..3fe9fe2 100644 --- a/tools/social-twitter/src/tweet/post_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/post_tweet.rs @@ -94,7 +94,10 @@ impl NexusTool for PostTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.post-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.post-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/retweet_tweet.rs b/offchain/tools/social-twitter/src/tweet/retweet_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/retweet_tweet.rs rename to offchain/tools/social-twitter/src/tweet/retweet_tweet.rs index 4477448..48af3b0 100644 --- a/tools/social-twitter/src/tweet/retweet_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/retweet_tweet.rs @@ -63,7 +63,10 @@ impl NexusTool for RetweetTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.retweet-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.retweet-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/undo_retweet_tweet.rs b/offchain/tools/social-twitter/src/tweet/undo_retweet_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/undo_retweet_tweet.rs rename to offchain/tools/social-twitter/src/tweet/undo_retweet_tweet.rs index 43fb5d2..ca2d87f 100644 --- a/tools/social-twitter/src/tweet/undo_retweet_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/undo_retweet_tweet.rs @@ -60,7 +60,10 @@ impl NexusTool for UndoRetweetTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.undo-retweet-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.undo-retweet-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/tweet/unlike_tweet.rs b/offchain/tools/social-twitter/src/tweet/unlike_tweet.rs similarity index 98% rename from tools/social-twitter/src/tweet/unlike_tweet.rs rename to offchain/tools/social-twitter/src/tweet/unlike_tweet.rs index 512cfae..8842872 100644 --- a/tools/social-twitter/src/tweet/unlike_tweet.rs +++ b/offchain/tools/social-twitter/src/tweet/unlike_tweet.rs @@ -60,7 +60,10 @@ impl NexusTool for UnlikeTweet { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.unlike-tweet@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.unlike-tweet@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/twitter_client.rs b/offchain/tools/social-twitter/src/twitter_client.rs similarity index 100% rename from tools/social-twitter/src/twitter_client.rs rename to offchain/tools/social-twitter/src/twitter_client.rs diff --git a/tools/social-twitter/src/user/follow_user.rs b/offchain/tools/social-twitter/src/user/follow_user.rs similarity index 98% rename from tools/social-twitter/src/user/follow_user.rs rename to offchain/tools/social-twitter/src/user/follow_user.rs index 5e6fabe..fbd5408 100644 --- a/tools/social-twitter/src/user/follow_user.rs +++ b/offchain/tools/social-twitter/src/user/follow_user.rs @@ -63,7 +63,10 @@ impl NexusTool for FollowUser { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.follow-user@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.follow-user@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_user_by_id.rs b/offchain/tools/social-twitter/src/user/get_user_by_id.rs similarity index 99% rename from tools/social-twitter/src/user/get_user_by_id.rs rename to offchain/tools/social-twitter/src/user/get_user_by_id.rs index e722f30..1152cb9 100644 --- a/tools/social-twitter/src/user/get_user_by_id.rs +++ b/offchain/tools/social-twitter/src/user/get_user_by_id.rs @@ -144,7 +144,10 @@ impl NexusTool for GetUserById { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-by-id@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-by-id@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_user_by_username.rs b/offchain/tools/social-twitter/src/user/get_user_by_username.rs similarity index 99% rename from tools/social-twitter/src/user/get_user_by_username.rs rename to offchain/tools/social-twitter/src/user/get_user_by_username.rs index 5d96739..2441f4c 100644 --- a/tools/social-twitter/src/user/get_user_by_username.rs +++ b/offchain/tools/social-twitter/src/user/get_user_by_username.rs @@ -144,7 +144,10 @@ impl NexusTool for GetUserByUsername { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-user-by-username@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-user-by-username@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_users_by_id.rs b/offchain/tools/social-twitter/src/user/get_users_by_id.rs similarity index 99% rename from tools/social-twitter/src/user/get_users_by_id.rs rename to offchain/tools/social-twitter/src/user/get_users_by_id.rs index 5c9b1ed..7994c93 100644 --- a/tools/social-twitter/src/user/get_users_by_id.rs +++ b/offchain/tools/social-twitter/src/user/get_users_by_id.rs @@ -102,7 +102,10 @@ impl NexusTool for GetUsersById { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-users-by-id@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-users-by-id@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/get_users_by_username.rs b/offchain/tools/social-twitter/src/user/get_users_by_username.rs similarity index 99% rename from tools/social-twitter/src/user/get_users_by_username.rs rename to offchain/tools/social-twitter/src/user/get_users_by_username.rs index 160e92a..2f50e2c 100644 --- a/tools/social-twitter/src/user/get_users_by_username.rs +++ b/offchain/tools/social-twitter/src/user/get_users_by_username.rs @@ -75,7 +75,10 @@ impl NexusTool for GetUsersByUsername { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.get-users-by-username@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.get-users-by-username@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/social-twitter/src/user/mod.rs b/offchain/tools/social-twitter/src/user/mod.rs similarity index 100% rename from tools/social-twitter/src/user/mod.rs rename to offchain/tools/social-twitter/src/user/mod.rs diff --git a/tools/social-twitter/src/user/models.rs b/offchain/tools/social-twitter/src/user/models.rs similarity index 100% rename from tools/social-twitter/src/user/models.rs rename to offchain/tools/social-twitter/src/user/models.rs diff --git a/tools/social-twitter/src/user/unfollow_user.rs b/offchain/tools/social-twitter/src/user/unfollow_user.rs similarity index 98% rename from tools/social-twitter/src/user/unfollow_user.rs rename to offchain/tools/social-twitter/src/user/unfollow_user.rs index 30e4444..7edd8cd 100644 --- a/tools/social-twitter/src/user/unfollow_user.rs +++ b/offchain/tools/social-twitter/src/user/unfollow_user.rs @@ -60,7 +60,10 @@ impl NexusTool for UnfollowUser { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.social.twitter.unfollow-user@1") + fqn!(concat!( + "xyz.taluslabs.social.twitter.unfollow-user@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/social-twitter/tools.json b/offchain/tools/social-twitter/tools.json new file mode 100644 index 0000000..d5d37ca --- /dev/null +++ b/offchain/tools/social-twitter/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "social-twitter", + "command": "social-twitter", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/storage-walrus/Cargo.toml b/offchain/tools/storage-walrus/Cargo.toml similarity index 86% rename from tools/storage-walrus/Cargo.toml rename to offchain/tools/storage-walrus/Cargo.toml index a2f0fee..42acc1a 100644 --- a/tools/storage-walrus/Cargo.toml +++ b/offchain/tools/storage-walrus/Cargo.toml @@ -12,6 +12,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "walrus" +path = "src/main.rs" + [dependencies] schemars.workspace = true serde.workspace = true @@ -28,3 +32,7 @@ nexus-sdk = { workspace = true, features = ["walrus"] } [dev-dependencies] mockito.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" diff --git a/tools/storage-walrus/README.md b/offchain/tools/storage-walrus/README.md similarity index 100% rename from tools/storage-walrus/README.md rename to offchain/tools/storage-walrus/README.md diff --git a/offchain/tools/storage-walrus/build.rs b/offchain/tools/storage-walrus/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/storage-walrus/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/storage-walrus/src/client.rs b/offchain/tools/storage-walrus/src/client.rs similarity index 100% rename from tools/storage-walrus/src/client.rs rename to offchain/tools/storage-walrus/src/client.rs diff --git a/tools/storage-walrus/src/main.rs b/offchain/tools/storage-walrus/src/main.rs similarity index 100% rename from tools/storage-walrus/src/main.rs rename to offchain/tools/storage-walrus/src/main.rs diff --git a/tools/storage-walrus/src/read_file.rs b/offchain/tools/storage-walrus/src/read_file.rs similarity index 98% rename from tools/storage-walrus/src/read_file.rs rename to offchain/tools/storage-walrus/src/read_file.rs index bba8351..d44f962 100644 --- a/tools/storage-walrus/src/read_file.rs +++ b/offchain/tools/storage-walrus/src/read_file.rs @@ -67,7 +67,10 @@ impl NexusTool for ReadFile { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.read-file@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.read-file@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/read_json.rs b/offchain/tools/storage-walrus/src/read_json.rs similarity index 99% rename from tools/storage-walrus/src/read_json.rs rename to offchain/tools/storage-walrus/src/read_json.rs index 9711831..5fe8ed6 100644 --- a/tools/storage-walrus/src/read_json.rs +++ b/offchain/tools/storage-walrus/src/read_json.rs @@ -96,7 +96,10 @@ impl NexusTool for ReadJson { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.read-json@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.read-json@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/upload_file.rs b/offchain/tools/storage-walrus/src/upload_file.rs similarity index 99% rename from tools/storage-walrus/src/upload_file.rs rename to offchain/tools/storage-walrus/src/upload_file.rs index fb06b6d..f45feeb 100644 --- a/tools/storage-walrus/src/upload_file.rs +++ b/offchain/tools/storage-walrus/src/upload_file.rs @@ -93,7 +93,10 @@ impl NexusTool for UploadFile { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.upload-file@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.upload-file@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/upload_json.rs b/offchain/tools/storage-walrus/src/upload_json.rs similarity index 99% rename from tools/storage-walrus/src/upload_json.rs rename to offchain/tools/storage-walrus/src/upload_json.rs index aa5f6a3..a3316d0 100644 --- a/tools/storage-walrus/src/upload_json.rs +++ b/offchain/tools/storage-walrus/src/upload_json.rs @@ -98,7 +98,10 @@ impl NexusTool for UploadJson { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.upload-json@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.upload-json@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/tools/storage-walrus/src/utils.rs b/offchain/tools/storage-walrus/src/utils.rs similarity index 100% rename from tools/storage-walrus/src/utils.rs rename to offchain/tools/storage-walrus/src/utils.rs diff --git a/tools/storage-walrus/src/verify_blob.rs b/offchain/tools/storage-walrus/src/verify_blob.rs similarity index 98% rename from tools/storage-walrus/src/verify_blob.rs rename to offchain/tools/storage-walrus/src/verify_blob.rs index 0016788..033590b 100644 --- a/tools/storage-walrus/src/verify_blob.rs +++ b/offchain/tools/storage-walrus/src/verify_blob.rs @@ -67,7 +67,10 @@ impl NexusTool for VerifyBlob { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.storage.walrus.verify-blob@1") + fqn!(concat!( + "xyz.taluslabs.storage.walrus.verify-blob@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/storage-walrus/tools.json b/offchain/tools/storage-walrus/tools.json new file mode 100644 index 0000000..c26f0f1 --- /dev/null +++ b/offchain/tools/storage-walrus/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "walrus", + "command": "walrus", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/tools/templating-jinja/Cargo.toml b/offchain/tools/templating-jinja/Cargo.toml similarity index 76% rename from tools/templating-jinja/Cargo.toml rename to offchain/tools/templating-jinja/Cargo.toml index 34ae1a5..22cbdb7 100644 --- a/tools/templating-jinja/Cargo.toml +++ b/offchain/tools/templating-jinja/Cargo.toml @@ -10,6 +10,10 @@ authors.workspace = true keywords.workspace = true categories.workspace = true +[[bin]] +name = "templating-jinja" +path = "src/main.rs" + [dependencies] minijinja.workspace = true schemars.workspace = true @@ -19,4 +23,8 @@ tokio.workspace = true # === Nexus deps === nexus-toolkit.workspace = true -nexus-sdk.workspace = true \ No newline at end of file +nexus-sdk.workspace = true + +[build-dependencies] +serde_json.workspace = true +toml = "0.8" \ No newline at end of file diff --git a/tools/templating-jinja/README.md b/offchain/tools/templating-jinja/README.md similarity index 100% rename from tools/templating-jinja/README.md rename to offchain/tools/templating-jinja/README.md diff --git a/offchain/tools/templating-jinja/build.rs b/offchain/tools/templating-jinja/build.rs new file mode 100644 index 0000000..07c24d8 --- /dev/null +++ b/offchain/tools/templating-jinja/build.rs @@ -0,0 +1,50 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let tools_json_path = manifest_dir.join("tools.json"); + let cargo_toml_path = manifest_dir.join("Cargo.toml"); + + println!("cargo::rerun-if-changed={}", tools_json_path.display()); + println!("cargo::rerun-if-changed={}", cargo_toml_path.display()); + + let tools_json: serde_json::Value = serde_json::from_str( + &fs::read_to_string(&tools_json_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", tools_json_path.display())), + ) + .expect("tools.json must be valid JSON"); + + let command = tools_json["command"] + .as_str() + .expect("tools.json must have command"); + + let cargo_toml: toml::Value = toml::from_str( + &fs::read_to_string(&cargo_toml_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", cargo_toml_path.display())), + ) + .expect("Cargo.toml must be valid TOML"); + + let bin_name = cargo_toml + .get("bin") + .and_then(|b| b.as_array()) + .and_then(|bins| bins.first()) + .and_then(|b| b.get("name")) + .and_then(|n| n.as_str()); + + match bin_name { + Some(name) if name == command => {} + Some(name) => panic!( + "Binary name mismatch: Cargo.toml [[bin]] name = \"{name}\" \ + but tools.json command = \"{command}\". They must match." + ), + None => panic!( + "Cargo.toml has no [[bin]] section but tools.json specifies \ + command = \"{command}\". Add: [[bin]]\nname = \"{command}\"" + ), + } + + // FQN version is set by CI via Docker build-arg; defaults to "1" locally. + let version = env::var("TOOL_FQN_VERSION").unwrap_or_else(|_| "1".to_string()); + println!("cargo:rustc-env=TOOL_FQN_VERSION={version}"); + println!("cargo:rerun-if-env-changed=TOOL_FQN_VERSION"); +} diff --git a/tools/templating-jinja/src/main.rs b/offchain/tools/templating-jinja/src/main.rs similarity index 100% rename from tools/templating-jinja/src/main.rs rename to offchain/tools/templating-jinja/src/main.rs diff --git a/tools/templating-jinja/src/template.rs b/offchain/tools/templating-jinja/src/template.rs similarity index 99% rename from tools/templating-jinja/src/template.rs rename to offchain/tools/templating-jinja/src/template.rs index 7587a53..95c6d76 100644 --- a/tools/templating-jinja/src/template.rs +++ b/offchain/tools/templating-jinja/src/template.rs @@ -67,7 +67,10 @@ impl NexusTool for TemplatingJinja { } fn fqn() -> ToolFqn { - fqn!("xyz.taluslabs.templating.jinja@1") + fqn!(concat!( + "xyz.taluslabs.templating.jinja@", + env!("TOOL_FQN_VERSION") + )) } fn path() -> &'static str { diff --git a/offchain/tools/templating-jinja/tools.json b/offchain/tools/templating-jinja/tools.json new file mode 100644 index 0000000..986c0f0 --- /dev/null +++ b/offchain/tools/templating-jinja/tools.json @@ -0,0 +1,7 @@ +{ + "tool_name": "templating-jinja", + "command": "templating-jinja", + "environment": { + "RUST_LOG": "info" + } +} diff --git a/onchain/README.md b/onchain/README.md new file mode 100644 index 0000000..0481939 --- /dev/null +++ b/onchain/README.md @@ -0,0 +1,6 @@ +# onchain/ + +Reserved for onchain (Move / Sui package) tools and their CI. Empty +for now — see the offchain CI pipeline spec in +[docs/superpowers/specs/2026-05-19-offchain-tools-pipeline-design.md](../docs/superpowers/specs/2026-05-19-offchain-tools-pipeline-design.md) +for context. diff --git a/tools/payments-stripe/deploy/Dockerfile b/tools/payments-stripe/deploy/Dockerfile deleted file mode 100644 index 622fdd7..0000000 --- a/tools/payments-stripe/deploy/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -# ---- Builder ---------------------------------------------------------------- -FROM rust:1.83-bookworm AS builder -WORKDIR /build - -RUN apt-get update && apt-get install -y --no-install-recommends \ - pkg-config libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* - -# Copy the whole workspace — `members = ["tools/*"]` requires every -# sibling crate to exist for cargo's resolver. -COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ -COPY tools tools -COPY helpers helpers - -RUN cargo +stable build --release --package payments-stripe - -# ---- Runtime ---------------------------------------------------------------- -FROM gcr.io/distroless/cc-debian12:nonroot AS runtime -WORKDIR /app - -COPY --from=builder /build/target/release/payments-stripe /app/payments-stripe - -ENV BIND_ADDR=0.0.0.0:8080 -EXPOSE 8080 - -USER nonroot -ENTRYPOINT ["/app/payments-stripe"] diff --git a/tools/payments-stripe/deploy/cloud-run.mainnet.yaml b/tools/payments-stripe/deploy/cloud-run.mainnet.yaml deleted file mode 100644 index 7970b80..0000000 --- a/tools/payments-stripe/deploy/cloud-run.mainnet.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Credential model: this service holds NO Stripe API credentials. -# The only secrets mounted here are: -# - nexus-toolkit-config-mainnet-payments-stripe (Tool's own Ed25519 signing key) -# - nexus-allowed-leaders-mainnet (Leader public keys, not secret) -# Stripe `api_key` arrives in the per-request Input struct, sourced by -# the Leader at DAG-execution time. Adding any other secretKeyRef fails -# the auditor's C10 check. -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: "payments-stripe-mainnet" - labels: - network: mainnet - tool-fqn: "xyz.taluslabs.payments.stripe" - annotations: - run.googleapis.com/launch-stage: GA -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "1" - autoscaling.knative.dev/maxScale: "20" - run.googleapis.com/execution-environment: gen2 - spec: - serviceAccountName: "nexus-tools-mainnet@${PROJECT_ID}.iam.gserviceaccount.com" - timeoutSeconds: 30 - containerConcurrency: 80 - containers: - - name: tool - image: "${REGISTRY}/payments-stripe:${SHA}" - ports: - - name: http1 - containerPort: 8080 - env: - - name: BIND_ADDR - value: 0.0.0.0:8080 - - name: NEXUS_TOOLKIT_CONFIG_PATH - value: /etc/nexus/toolkit.json - - name: NEXUS_NETWORK - value: mainnet - volumeMounts: - - name: nexus-config - mountPath: /etc/nexus - readOnly: true - - name: allowed-leaders - mountPath: /etc/nexus/allowed_leaders - readOnly: true - resources: - limits: - cpu: "2" - memory: 1Gi - volumes: - - name: nexus-config - secret: - secretName: "nexus-toolkit-config-mainnet-payments-stripe" - - name: allowed-leaders - secret: - secretName: "nexus-allowed-leaders-mainnet" - traffic: - - percent: 100 - latestRevision: true diff --git a/tools/payments-stripe/deploy/cloud-run.testnet.yaml b/tools/payments-stripe/deploy/cloud-run.testnet.yaml deleted file mode 100644 index 59c22b0..0000000 --- a/tools/payments-stripe/deploy/cloud-run.testnet.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Credential model: this service holds NO Stripe API credentials. -# The only secrets mounted here are: -# - nexus-toolkit-config-testnet-payments-stripe (Tool's own Ed25519 signing key) -# - nexus-allowed-leaders-testnet (Leader public keys, not secret) -# Stripe `api_key` arrives in the per-request Input struct, sourced by -# the Leader at DAG-execution time. Adding any other secretKeyRef fails -# the auditor's C10 check. -apiVersion: serving.knative.dev/v1 -kind: Service -metadata: - name: "payments-stripe-testnet" - labels: - network: testnet - tool-fqn: "xyz.taluslabs.payments.stripe" - annotations: - run.googleapis.com/launch-stage: GA -spec: - template: - metadata: - annotations: - autoscaling.knative.dev/minScale: "0" - autoscaling.knative.dev/maxScale: "5" - run.googleapis.com/execution-environment: gen2 - spec: - serviceAccountName: "nexus-tools-testnet@${PROJECT_ID}.iam.gserviceaccount.com" - timeoutSeconds: 30 - containerConcurrency: 80 - containers: - - name: tool - image: "${REGISTRY}/payments-stripe:${SHA}" - ports: - - name: http1 - containerPort: 8080 - env: - - name: BIND_ADDR - value: 0.0.0.0:8080 - - name: NEXUS_TOOLKIT_CONFIG_PATH - value: /etc/nexus/toolkit.json - - name: NEXUS_NETWORK - value: testnet - volumeMounts: - - name: nexus-config - mountPath: /etc/nexus - readOnly: true - - name: allowed-leaders - mountPath: /etc/nexus/allowed_leaders - readOnly: true - resources: - limits: - cpu: "1" - memory: 512Mi - volumes: - - name: nexus-config - secret: - secretName: "nexus-toolkit-config-testnet-payments-stripe" - - name: allowed-leaders - secret: - secretName: "nexus-allowed-leaders-testnet" - traffic: - - percent: 100 - latestRevision: true diff --git a/tools/payments-stripe/deploy/register.sh b/tools/payments-stripe/deploy/register.sh deleted file mode 100755 index 65bfed5..0000000 --- a/tools/payments-stripe/deploy/register.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Idempotent registration of payments-stripe tools with Nexus. -# -# Usage: -# register.sh -# -# Reads tool paths from ../paths.json, fetches each tool's FQN from -# /meta, then registers (or updates) each FQN against the chosen Nexus -# network. - -set -euo pipefail - -URL="${1:?missing first arg: tool URL}" -NETWORK="${2:?missing second arg: testnet|mainnet}" - -case "$NETWORK" in - testnet|mainnet) ;; - *) echo "network must be 'testnet' or 'mainnet'"; exit 2 ;; -esac - -PATHS_FILE="$(dirname "$0")/../paths.json" -if [[ ! -f "$PATHS_FILE" ]]; then - echo "expected $PATHS_FILE to enumerate tool paths" >&2 - exit 2 -fi - -mapfile -t PATHS < <(jq -r '.[]' "$PATHS_FILE") - -for path in "${PATHS[@]}"; do - meta_url="${URL%/}${path}/meta" - meta="$(curl -fsS "$meta_url")" - fqn="$(echo "$meta" | jq -r .fqn)" - description="$(echo "$meta" | jq -r .description)" - - echo "registering $fqn at $URL$path on $NETWORK" - - if nexus tool list --network "$NETWORK" 2>/dev/null | grep -q "$fqn"; then - nexus tool update offchain \ - --network "$NETWORK" \ - --tool-fqn "$fqn" \ - --url "${URL%/}${path}" - else - nexus tool register offchain \ - --network "$NETWORK" \ - --tool-fqn "$fqn" \ - --url "${URL%/}${path}" \ - --description "$description" - fi -done diff --git a/tools/payments-stripe/paths.json b/tools/payments-stripe/paths.json deleted file mode 100644 index 28e0a40..0000000 --- a/tools/payments-stripe/paths.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - "/create-payment-intent", - "/get-payment-intent", - "/confirm-payment-intent", - "/create-customer", - "/get-balance", - "/list-charges" -]