[#83] add: gRPC parity test suite (Zainod vs. Lightwalletd)

.github/scripts/make-zcash-artifacts-executable.sh

@@ -2,8 +2,8 @@
 #
 # Make the zcash binaries downloaded into ./src executable on the test runners.
 #
-# The `zallet` launcher execs a sibling backend binary (`zallet-zaino` or
-# `zallet-zebra`), so those are made executable too when present.
+# The `zallet` launcher execs a sibling backend binary, so backend binaries are
+# made executable too when present.
 #
 # Inputs (environment):
 #   FILE_EXT  executable suffix, e.g. ".exe" (optional, default empty)
@@ -15,9 +15,9 @@ ext="${FILE_EXT:-}"
 chmod +x "./src/zebrad$ext"
 chmod +x "./src/zainod$ext"
 chmod +x "./src/zallet$ext"
-if [ -f "./src/zallet-zaino$ext" ]; then
-    chmod +x "./src/zallet-zaino$ext"
-fi
-if [ -f "./src/zallet-zebra$ext" ]; then
-    chmod +x "./src/zallet-zebra$ext"
-fi
+chmod +x "./src/lightwalletd$ext"
+for zallet_backend in zallet-zebra zallet-zaino; do
+    if [ -f "./src/${zallet_backend}$ext" ]; then
+        chmod +x "./src/${zallet_backend}$ext"
+    fi
+done

.github/scripts/stage-zallet-db-dump.sh

@@ -16,9 +16,29 @@ set -euo pipefail
 
 root="${DB_DUMP_ROOT:?DB_DUMP_ROOT must be set}"
 
-db_dump="$(find "$root" -path '*/zewif-zcashd-*/out/db_dump' -type f -print -quit)"
+candidate_roots=(
+    "$root"
+    backends/zaino/target/release/build
+    backends/zaino/target
+    target/release/build
+    target
+)
+existing_roots=()
+for candidate in "${candidate_roots[@]}"; do
+    if [ -d "$candidate" ]; then
+        existing_roots+=("$candidate")
+    fi
+done
+
+db_dump=""
+if [ "${#existing_roots[@]}" -gt 0 ]; then
+    db_dump="$(
+        find "${existing_roots[@]}" \
+            -path '*/zewif-zcashd-*/out/db_dump' -type f -print -quit
+    )"
+fi
 if [ -z "$db_dump" ]; then
-    echo "Vendored db_dump not found under $root/zewif-zcashd-*/out/" >&2
+    echo "Vendored db_dump not found under target/.../zewif-zcashd-*/out/" >&2
     exit 1
 fi
 

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ on:
       - zebra-interop-request
       - zaino-interop-request
       - zallet-interop-request
+      - lightwalletd-interop-request
 
 permissions:
   contents: read
@@ -455,6 +456,84 @@ jobs:
       job_name_prefix: Build zallet on
       is_zallet: true
 
+  build-lightwalletd:
+    name: Build lightwalletd on ${{ matrix.platform }}${{ matrix.required_suffix }}
+    needs: setup
+    runs-on: ${{ matrix.build_os }}
+    container:
+      image: ${{ matrix.container }}
+      env:
+        HOME: /root
+        PATH: /root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
+    continue-on-error: ${{ !matrix.required }}
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.setup.outputs.build_matrix) }}
+
+    steps:
+      - name: Check out integration-tests to access actions
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          path: integration-tests
+          persist-credentials: false
+
+      - name: Compute interop repo ids
+        id: repo-ids
+        uses: ./integration-tests/.github/actions/interop-repo-ids
+
+      - id: start-interop
+        uses: ./integration-tests/.github/actions/start-interop
+        with:
+          status-app-id: ${{ secrets.STATUS_APP_ID }}
+          status-app-private-key: ${{ secrets.STATUS_APP_PRIVATE_KEY }}
+          requesting-owner: ${{ steps.repo-ids.outputs.requesting-owner }}
+          requesting-repository: ${{ steps.repo-ids.outputs.requesting-repository }}
+          job-name: "Build lightwalletd on ${{ matrix.platform }}${{ matrix.required_suffix }}"
+
+      - name: Use specified zcash/lightwalletd commit
+        if: github.event.action == 'lightwalletd-interop-request'
+        shell: sh
+        env:
+          SHA: ${{ github.event.client_payload.sha }}
+        run: echo "LIGHTWALLETD_REF=${SHA}" >> $GITHUB_ENV
+
+      - name: Use zcash/lightwalletd current master
+        if: github.event.action != 'lightwalletd-interop-request'
+        run: echo "LIGHTWALLETD_REF=refs/heads/master" >> $GITHUB_ENV
+
+      - name: Check out zcash/lightwalletd
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: zcash/lightwalletd
+          ref: ${{ env.LIGHTWALLETD_REF }}
+          path: lightwalletd
+          persist-credentials: false
+
+      - name: Set up Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version-file: lightwalletd/go.mod
+          cache-dependency-path: lightwalletd/go.sum
+
+      - name: Build lightwalletd
+        run: go build -v -o lightwalletd${{ matrix.file_ext }} .
+        working-directory: ./lightwalletd
+
+      - name: Upload lightwalletd
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: lightwalletd-${{ matrix.name }}
+          path: |
+              ${{ format('./lightwalletd/lightwalletd{0}', matrix.file_ext) }}
+
+      - uses: ./integration-tests/.github/actions/finish-interop
+        if: always()
+        with:
+          app-token: ${{ steps.start-interop.outputs.app-token }}
+          requesting-owner: ${{ steps.repo-ids.outputs.requesting-owner }}
+          requesting-repository: ${{ steps.repo-ids.outputs.requesting-repository }}
+          job-name: "Build lightwalletd on ${{ matrix.platform }}${{ matrix.required_suffix }}"
+
   # Not working in Windows
   sec-hard:
     name: sec-hard ${{ matrix.platform }}${{ matrix.required_suffix }}
@@ -463,6 +542,7 @@ jobs:
       - build-zebra
       - build-zaino
       - build-zallet
+      - build-lightwalletd
     runs-on: ${{ matrix.test_os }}
     container:
       image: ${{ matrix.container }}
@@ -523,6 +603,12 @@ jobs:
           name: zallet-${{ matrix.name }}
           path: ./src
 
+      - name: Download lightwalletd artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: lightwalletd-${{ matrix.name }}
+          path: ./src
+
       - name: Make artifact executable
         if: runner.os != 'Windows'
         shell: bash
@@ -560,23 +646,41 @@ jobs:
         include: ${{ fromJson(needs.setup.outputs.rpc_test_matrix) }}
 
     steps:
+      - name: Select the test branch
+        id: test-branch
+        if: endsWith(github.event.action, '-interop-request')
+        env:
+          TEST_SHA: ${{ github.event.client_payload.test_sha }}
+        run: |
+          if [ "${TEST_SHA}" != "null" ] && [ -n "${TEST_SHA}" ]; then
+            echo "test_sha=${TEST_SHA}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Check out the selected test branch
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Empty on direct repo PRs, which selects the default.
+          ref: ${{ steps.test-branch.outputs.test_sha }}
+          persist-credentials: false
+
       - name: Set up Python
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
         with:
           python-version: ${{ needs.setup.outputs.python_version }}
 
-      - name: Cache Python dependencies for RPC tests
+      - name: Cache uv project environment for RPC tests
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           enableCrossOsArchive: true
-          path: venv
-          key: test-rpc-venv-${{ matrix.name }}-py${{ needs.setup.outputs.python_version }}
+          path: .venv
+          key: test-rpc-venv-${{ matrix.name }}-py${{ needs.setup.outputs.python_version }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
 
-      - name: Get Python dependencies for RPC tests
+      - name: Install uv
+        run: python3 -m pip install uv
+
+      - name: Sync Python dependencies for RPC tests
         run: |
-          python3 -m venv ./venv
-          . ./venv/bin/activate
-          pip install zmq asyncio base58 toml embit
+          uv sync --frozen
 
   test-rpc:
     name: RPC tests ${{ matrix.platform }} ${{ matrix.backend }} ${{ matrix.shard }}${{ matrix.required_suffix }}
@@ -585,6 +689,7 @@ jobs:
       - build-zebra
       - build-zaino
       - build-zallet
+      - build-lightwalletd
       - rpc-depends
     runs-on: ${{ matrix.test_os }}
     container:
@@ -641,18 +746,19 @@ jobs:
         with:
           python-version: ${{ needs.setup.outputs.python_version }}
 
-      - name: Cache Python dependencies for RPC tests
+      - name: Cache uv project environment for RPC tests
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           enableCrossOsArchive: true
-          path: venv
-          key: test-rpc-venv-${{ matrix.name }}-py${{ needs.setup.outputs.python_version }}
+          path: .venv
+          key: test-rpc-venv-${{ matrix.name }}-py${{ needs.setup.outputs.python_version }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+
+      - name: Install uv
+        run: python3 -m pip install uv
 
-      - name: Get Python dependencies for RPC tests if not cached
+      - name: Sync Python dependencies for RPC tests
         run: |
-          python3 -m venv ./venv
-          . ./venv/bin/activate
-          pip install zmq asyncio base58 toml embit
+          uv sync --frozen
 
       - name: Download zebrad artifact
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -672,6 +778,12 @@ jobs:
           name: zallet-${{ matrix.name }}
           path: ./src
 
+      - name: Download lightwalletd artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: lightwalletd-${{ matrix.name }}
+          path: ./src
+
       - name: Make artifact executable
         if: runner.os != 'Windows'
         shell: bash
@@ -744,8 +856,7 @@ jobs:
           if all_passed == False:
               sys.exit(1)
           EOF
-          . ./venv/bin/activate
-          ZALLET_BACKEND=${{ matrix.backend }} ZEBRAD=$(pwd)/${{ format('src/zebrad{0}', matrix.file_ext) }} ZAINOD=$(pwd)/${{ format('src/zainod{0}', matrix.file_ext) }} ZALLET=$(pwd)/${{ format('src/zallet{0}', matrix.file_ext) }} SRC_DIR=$(pwd) python3 ./subclass.py
+          ZALLET_BACKEND=${{ matrix.backend }} ZEBRAD=$(pwd)/${{ format('src/zebrad{0}', matrix.file_ext) }} ZAINOD=$(pwd)/${{ format('src/zainod{0}', matrix.file_ext) }} ZALLET=$(pwd)/${{ format('src/zallet{0}', matrix.file_ext) }} LIGHTWALLETD=$(pwd)/${{ format('src/lightwalletd{0}', matrix.file_ext) }} SRC_DIR=$(pwd) uv run python3 ./subclass.py
 
       - uses: ./.github/actions/finish-interop
         if: always()

.gitignore

@@ -5,10 +5,13 @@
 *.orig
 *.pyc
 .vscode
-
+.claude
+.codex
 qa/pull-tester/tests_config.py
 qa/pull-tester/tests_config.ini
 qa/cache/*
+qa/rpc-tests/cache/grpc_comparison/
+qa/rpc-tests/cache/grpc_comparison_stage1/
 src/*
 
 poetry.lock