From a553e5266e3d890aaf3089fbb1ae75a8f4888f71 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 14:15:10 -0300 Subject: [PATCH 01/21] Testing connectivity from GHA to Canton Devnet --- .github/workflows/test-tailscale.yml | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/test-tailscale.yml diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml new file mode 100644 index 000000000..7e935d4d2 --- /dev/null +++ b/.github/workflows/test-tailscale.yml @@ -0,0 +1,34 @@ +name: Test Tailscale Connectivity +run-name: Test Tailscale connectivity to Canton Devnet + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + test-tailscale: + name: Test Tailscale Connectivity + timeout-minutes: 10 + runs-on: ubuntu-latest + # Optional: set environment if using GitHub Environment secrets (e.g. canton-devnet) + # environment: canton-devnet + permissions: + id-token: write # Required for OIDC + contents: read + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Connect to Tailscale + uses: tailscale/github-action@v2 + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} + audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} + tags: tag:bcy-validators-gha + + - name: Test Canton Devnet connectivity + run: | + curl -f -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant + echo "✅ Canton Devnet is accessible via Tailscale" From 2573a3744548cee3f2aaeb88f356a39acf00a265 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 14:23:57 -0300 Subject: [PATCH 02/21] Updated GHA Tailscale action version to v4 --- .github/workflows/test-tailscale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 7e935d4d2..3dfc6dbc9 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -22,7 +22,7 @@ jobs: uses: actions/checkout@v4 - name: Connect to Tailscale - uses: tailscale/github-action@v2 + uses: tailscale/github-action@v4 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} From 435c0e6b593d766ce742d07967f2419f1cf216fd Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 14:53:59 -0300 Subject: [PATCH 03/21] Adding debug steps for tailscale --- .github/workflows/test-tailscale.yml | 29 ++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 3dfc6dbc9..487a8dd41 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -11,8 +11,8 @@ jobs: name: Test Tailscale Connectivity timeout-minutes: 10 runs-on: ubuntu-latest - # Optional: set environment if using GitHub Environment secrets (e.g. canton-devnet) - # environment: canton-devnet + # Required: your Tailscale OIDC subject is repo:...:environment:* so the job must use an environment + environment: canton-devnet permissions: id-token: write # Required for OIDC contents: read @@ -21,6 +21,9 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + # If you see "sudo failed with exit code 1", the real error is often + # "requested tags [...] are invalid or not permitted". Create the tag + # in Tailscale Admin → ACL (Visual) → Tags and set owner to autogroup:admin. - name: Connect to Tailscale uses: tailscale/github-action@v4 with: @@ -28,7 +31,25 @@ jobs: audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} tags: tag:bcy-validators-gha + - name: Troubleshoot DNS (Tailscale resolution + 20s sleep) + run: | + echo "=== Tailscale DNS status ===" + tailscale dns status || true + echo "" + echo "=== Resolving canton-devnet.bcy-v.metalhosts.com ===" + tailscale dns resolve canton-devnet.bcy-v.metalhosts.com 2>/dev/null || true + echo "" + echo "=== System resolver (getent) ===" + getent hosts canton-devnet.bcy-v.metalhosts.com || true + echo "" + echo "=== /etc/resolv.conf ===" + cat /etc/resolv.conf + echo "" + echo "=== Sleeping 20s before curl ===" + sleep 20 + - name: Test Canton Devnet connectivity run: | - curl -f -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant - echo "✅ Canton Devnet is accessible via Tailscale" + curl -v -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true + echo "" + echo "✅ Check logs above for resolution IP and HTTP response (403 may be server-side auth)" From e3ef120ae3a9e869efd4ddd0801893d2b84b37fc Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 15:12:25 -0300 Subject: [PATCH 04/21] tailscale troubleshooting --- .github/workflows/test-tailscale.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 487a8dd41..73105464e 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -12,7 +12,7 @@ jobs: timeout-minutes: 10 runs-on: ubuntu-latest # Required: your Tailscale OIDC subject is repo:...:environment:* so the job must use an environment - environment: canton-devnet + # environment: canton-devnet permissions: id-token: write # Required for OIDC contents: read @@ -34,7 +34,7 @@ jobs: - name: Troubleshoot DNS (Tailscale resolution + 20s sleep) run: | echo "=== Tailscale DNS status ===" - tailscale dns status || true + sudo tailscale dns status || true echo "" echo "=== Resolving canton-devnet.bcy-v.metalhosts.com ===" tailscale dns resolve canton-devnet.bcy-v.metalhosts.com 2>/dev/null || true @@ -52,4 +52,4 @@ jobs: run: | curl -v -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true echo "" - echo "✅ Check logs above for resolution IP and HTTP response (403 may be server-side auth)" + echo "✅ Check logs above for resolution IP and HTTP response (403 may be server-side auth)" \ No newline at end of file From 87fb0243ad855b78564ece2739c43f59a222acec Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 15:47:34 -0300 Subject: [PATCH 05/21] More troubleshooting --- .github/workflows/test-tailscale.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 73105464e..3e591689d 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -50,6 +50,12 @@ jobs: - name: Test Canton Devnet connectivity run: | + echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com (expect 100.x.x.x for Tailscale) ===" + getent hosts canton-devnet.bcy-v.metalhosts.com || true + echo "" + echo "=== IP route show table all |grep tailscale ===" + ip route show table all |grep tailscale || true + echo "" curl -v -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true echo "" - echo "✅ Check logs above for resolution IP and HTTP response (403 may be server-side auth)" \ No newline at end of file + echo "✅ If you see 403: request reached AWS (awselb) but was rejected server-side (ELB/WAF/app allowlist or auth)." \ No newline at end of file From 47a4a63374cb5c4eaf98e1c3e87fee7ea7c473a0 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 15:59:08 -0300 Subject: [PATCH 06/21] tailscale up/down workoaround --- .github/workflows/test-tailscale.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 3e591689d..84ca414d2 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -50,6 +50,9 @@ jobs: - name: Test Canton Devnet connectivity run: | + echo "=== tailscale down && tailscale up workaround ===" + tailscale down && tailscale up + echo "" echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com (expect 100.x.x.x for Tailscale) ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true echo "" From 1a4b38551f2605cd29d1e5a8140ef67d3b4b734d Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 16:01:13 -0300 Subject: [PATCH 07/21] Fixed troubleshooting typos --- .github/workflows/test-tailscale.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 84ca414d2..d79f050bf 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -51,13 +51,13 @@ jobs: - name: Test Canton Devnet connectivity run: | echo "=== tailscale down && tailscale up workaround ===" - tailscale down && tailscale up + sudo tailscale down && sudo tailscale up echo "" echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com (expect 100.x.x.x for Tailscale) ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true echo "" - echo "=== IP route show table all |grep tailscale ===" - ip route show table all |grep tailscale || true + echo "=== IP route show table all ===" + ip route show table all || true echo "" curl -v -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true echo "" From e023130370100cea7c8dec75f9709e2d107cd7d8 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 16:24:46 -0300 Subject: [PATCH 08/21] Added tailscale --accept-routes=true --- .github/workflows/test-tailscale.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index d79f050bf..91b47e592 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -31,10 +31,17 @@ jobs: audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} tags: tag:bcy-validators-gha + # Required for app connectors: Linux nodes don't accept advertised routes by default. + # This allows the runner to receive routes from the app connector (e.g. canton-devnet IPs via tailscale0). + - name: Accept app connector routes + run: | + sudo tailscale set --accept-routes=true + sudo tailscale set --operator=$USER + - name: Troubleshoot DNS (Tailscale resolution + 20s sleep) run: | echo "=== Tailscale DNS status ===" - sudo tailscale dns status || true + tailscale dns status || true echo "" echo "=== Resolving canton-devnet.bcy-v.metalhosts.com ===" tailscale dns resolve canton-devnet.bcy-v.metalhosts.com 2>/dev/null || true From a088716053056df666c2a2c1b330cd58db1c78a8 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 16:27:44 -0300 Subject: [PATCH 09/21] more troubleshooting --- .github/workflows/test-tailscale.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 91b47e592..04d2b677e 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -40,6 +40,9 @@ jobs: - name: Troubleshoot DNS (Tailscale resolution + 20s sleep) run: | + echo "=== tailscale set --accept-routes=true ===" + tailscale set --accept-routes=true + echo "" echo "=== Tailscale DNS status ===" tailscale dns status || true echo "" @@ -57,8 +60,10 @@ jobs: - name: Test Canton Devnet connectivity run: | - echo "=== tailscale down && tailscale up workaround ===" - sudo tailscale down && sudo tailscale up + # echo "=== tailscale down && tailscale up workaround ===" + # sudo tailscale down && sudo tailscale up + echo "=== tailscale set --accept-routes=true ===" + tailscale set --accept-routes=true echo "" echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com (expect 100.x.x.x for Tailscale) ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true From fae29bc2554a48eb21be3aa2bb25958b814ddef8 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 16:33:59 -0300 Subject: [PATCH 10/21] Updated GH action --- .github/workflows/test-tailscale.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 04d2b677e..941078ccc 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -25,7 +25,7 @@ jobs: # "requested tags [...] are invalid or not permitted". Create the tag # in Tailscale Admin → ACL (Visual) → Tags and set owner to autogroup:admin. - name: Connect to Tailscale - uses: tailscale/github-action@v4 + uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} @@ -40,9 +40,6 @@ jobs: - name: Troubleshoot DNS (Tailscale resolution + 20s sleep) run: | - echo "=== tailscale set --accept-routes=true ===" - tailscale set --accept-routes=true - echo "" echo "=== Tailscale DNS status ===" tailscale dns status || true echo "" @@ -60,12 +57,7 @@ jobs: - name: Test Canton Devnet connectivity run: | - # echo "=== tailscale down && tailscale up workaround ===" - # sudo tailscale down && sudo tailscale up - echo "=== tailscale set --accept-routes=true ===" - tailscale set --accept-routes=true - echo "" - echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com (expect 100.x.x.x for Tailscale) ===" + echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true echo "" echo "=== IP route show table all ===" From 5d9d5359a0c17e1f2f153dcdbefeafcd60aa0bb7 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 20 Feb 2026 16:34:45 -0300 Subject: [PATCH 11/21] comment --- .github/workflows/test-tailscale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 941078ccc..d840f3775 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -25,7 +25,7 @@ jobs: # "requested tags [...] are invalid or not permitted". Create the tag # in Tailscale Admin → ACL (Visual) → Tags and set owner to autogroup:admin. - name: Connect to Tailscale - uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 + uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 # v4.1.1 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} From d40990b3d4c288eea64ada70ca1382dd2642d0b5 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Thu, 26 Feb 2026 17:24:40 -0300 Subject: [PATCH 12/21] Manually adding the tailscale routes for troubleshooting --- .github/workflows/test-tailscale.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index d840f3775..100a1381f 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -60,6 +60,11 @@ jobs: echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true echo "" + echo "=== Adding LB routes manually (test if missing routes cause 403) ===" + sudo ip route add 99.79.185.238/32 dev tailscale0 2>/dev/null || true + sudo ip route add 16.54.113.197/32 dev tailscale0 2>/dev/null || true + sudo ip route add 3.99.93.80/32 dev tailscale0 2>/dev/null || true + echo "" echo "=== IP route show table all ===" ip route show table all || true echo "" From 6fc36209ce87019a07385defe9827611e1dfef2b Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Thu, 26 Feb 2026 17:38:24 -0300 Subject: [PATCH 13/21] Additional troubleshooting --- .github/workflows/test-tailscale.yml | 38 +++++++++++----------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 100a1381f..f7f61ae8b 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -32,42 +32,34 @@ jobs: tags: tag:bcy-validators-gha # Required for app connectors: Linux nodes don't accept advertised routes by default. - # This allows the runner to receive routes from the app connector (e.g. canton-devnet IPs via tailscale0). - name: Accept app connector routes run: | sudo tailscale set --accept-routes=true sudo tailscale set --operator=$USER - - name: Troubleshoot DNS (Tailscale resolution + 20s sleep) + # Workaround for missing routes on ephemeral runners (tailscale/github-action#266). + # Forces tailscaled to reconnect and re-fetch route advertisements from the coordination server. + - name: Reconnect Tailscale to pick up app connector routes run: | - echo "=== Tailscale DNS status ===" - tailscale dns status || true + echo "=== tailscale down && tailscale up ===" + sudo tailscale down + sudo tailscale up --accept-routes echo "" - echo "=== Resolving canton-devnet.bcy-v.metalhosts.com ===" - tailscale dns resolve canton-devnet.bcy-v.metalhosts.com 2>/dev/null || true + echo "=== Waiting 10s for routes to propagate ===" + sleep 10 echo "" - echo "=== System resolver (getent) ===" - getent hosts canton-devnet.bcy-v.metalhosts.com || true - echo "" - echo "=== /etc/resolv.conf ===" - cat /etc/resolv.conf + echo "=== Tailscale status ===" + tailscale status || true echo "" - echo "=== Sleeping 20s before curl ===" - sleep 20 + echo "=== Routes (grep tailscale0) ===" + ip route show table all | grep tailscale || true - name: Test Canton Devnet connectivity run: | - echo "=== IP that will be used for canton-devnet.bcy-v.metalhosts.com ===" + echo "=== Resolved IPs ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true echo "" - echo "=== Adding LB routes manually (test if missing routes cause 403) ===" - sudo ip route add 99.79.185.238/32 dev tailscale0 2>/dev/null || true - sudo ip route add 16.54.113.197/32 dev tailscale0 2>/dev/null || true - sudo ip route add 3.99.93.80/32 dev tailscale0 2>/dev/null || true - echo "" - echo "=== IP route show table all ===" + echo "=== All routes ===" ip route show table all || true echo "" - curl -v -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true - echo "" - echo "✅ If you see 403: request reached AWS (awselb) but was rejected server-side (ELB/WAF/app allowlist or auth)." \ No newline at end of file + curl -v --connect-timeout 30 -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true \ No newline at end of file From ebf50d958565c9cf49b5d28e2252144c0a46bed5 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Thu, 26 Feb 2026 18:06:15 -0300 Subject: [PATCH 14/21] Debug --- .github/workflows/test-tailscale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index f7f61ae8b..6f494638a 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -43,7 +43,7 @@ jobs: run: | echo "=== tailscale down && tailscale up ===" sudo tailscale down - sudo tailscale up --accept-routes + sudo tailscale up --reset --accept-routes echo "" echo "=== Waiting 10s for routes to propagate ===" sleep 10 From fbd48d8d4f9d98bee01f91dcbae0456a4af9073c Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Thu, 26 Feb 2026 18:13:53 -0300 Subject: [PATCH 15/21] More debug.... --- .github/workflows/test-tailscale.yml | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 6f494638a..e734191f9 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -21,37 +21,27 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - # If you see "sudo failed with exit code 1", the real error is often - # "requested tags [...] are invalid or not permitted". Create the tag - # in Tailscale Admin → ACL (Visual) → Tags and set owner to autogroup:admin. - name: Connect to Tailscale uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 # v4.1.1 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} tags: tag:bcy-validators-gha + args: --accept-routes - # Required for app connectors: Linux nodes don't accept advertised routes by default. - - name: Accept app connector routes + - name: Wait for app connector route discovery run: | - sudo tailscale set --accept-routes=true sudo tailscale set --operator=$USER - - # Workaround for missing routes on ephemeral runners (tailscale/github-action#266). - # Forces tailscaled to reconnect and re-fetch route advertisements from the coordination server. - - name: Reconnect Tailscale to pick up app connector routes - run: | - echo "=== tailscale down && tailscale up ===" - sudo tailscale down - sudo tailscale up --reset --accept-routes - echo "" - echo "=== Waiting 10s for routes to propagate ===" - sleep 10 - echo "" echo "=== Tailscale status ===" tailscale status || true echo "" - echo "=== Routes (grep tailscale0) ===" + echo "=== Trigger DNS resolution to kick app connector route discovery ===" + getent hosts canton-devnet.bcy-v.metalhosts.com || true + echo "" + echo "=== Waiting 15s for routes to propagate ===" + sleep 15 + echo "" + echo "=== Routes (grep tailscale) ===" ip route show table all | grep tailscale || true - name: Test Canton Devnet connectivity From eec0d7941571ea04fc0ea003625355cf72a4dabf Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 27 Feb 2026 14:01:39 -0300 Subject: [PATCH 16/21] Debug --- .github/workflows/test-tailscale.yml | 53 ++++++++++++++++++---------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index e734191f9..2730bbbf9 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -11,10 +11,8 @@ jobs: name: Test Tailscale Connectivity timeout-minutes: 10 runs-on: ubuntu-latest - # Required: your Tailscale OIDC subject is repo:...:environment:* so the job must use an environment - # environment: canton-devnet permissions: - id-token: write # Required for OIDC + id-token: write contents: read steps: @@ -27,29 +25,48 @@ jobs: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} tags: tag:bcy-validators-gha - args: --accept-routes - - name: Wait for app connector route discovery + - name: Configure Tailscale for app connector routes run: | + sudo tailscale set --accept-routes=true sudo tailscale set --operator=$USER - echo "=== Tailscale status ===" + + - name: "[DEBUG] 1 - Tailscale node info" + run: | + echo "=== Tailscale status (this node + peers) ===" tailscale status || true echo "" - echo "=== Trigger DNS resolution to kick app connector route discovery ===" - getent hosts canton-devnet.bcy-v.metalhosts.com || true + echo "=== This node's Tailscale IP ===" + tailscale ip -4 || true echo "" - echo "=== Waiting 15s for routes to propagate ===" - sleep 15 + echo "=== Tailscale debug prefs (confirm AcceptRoutes=true) ===" + tailscale debug prefs 2>&1 | head -30 || true + + - name: "[DEBUG] 2 - DNS configuration (Split DNS shows app connector DoH)" + run: | + echo "=== Tailscale DNS status ===" + tailscale dns status || true echo "" - echo "=== Routes (grep tailscale) ===" - ip route show table all | grep tailscale || true + echo "=== /etc/resolv.conf ===" + cat /etc/resolv.conf - - name: Test Canton Devnet connectivity + - name: "[DEBUG] 3 - DNS resolution (LB IPs returned by app connector DoH)" run: | - echo "=== Resolved IPs ===" + echo "=== getent hosts canton-devnet.bcy-v.metalhosts.com ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true + + - name: "[DEBUG] 4 - Wait 15s for app connector route advertisement" + run: sleep 15 + + - name: "[DEBUG] 5 - Routes (expect LB IPs via tailscale0, but they are MISSING)" + run: | + echo "=== All routes via tailscale ===" + ip route show table all | grep tailscale || echo "(no tailscale routes found)" echo "" - echo "=== All routes ===" - ip route show table all || true - echo "" - curl -v --connect-timeout 30 -H "Content-Type: application/json" https://canton-devnet.bcy-v.metalhosts.com/metrics/participant || true \ No newline at end of file + echo "=== Full routing table ===" + ip route show table all + + - name: "[DEBUG] 6 - Connectivity test (403 = reached ALB via public internet, not via app connector)" + run: | + curl -v --connect-timeout 30 -H "Content-Type: application/json" \ + https://canton-devnet.bcy-v.metalhosts.com/metrics/participant 2>&1 || true From d605cc35bb2633d0fd3eae4ac2ff1f2184796376 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Thu, 23 Apr 2026 14:39:19 -0300 Subject: [PATCH 17/21] Debug checks --- .github/workflows/test-tailscale.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 2730bbbf9..dc10ccfdc 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -54,11 +54,21 @@ jobs: run: | echo "=== getent hosts canton-devnet.bcy-v.metalhosts.com ===" getent hosts canton-devnet.bcy-v.metalhosts.com || true + - name: "[DEBUG] 4 - Get route to canton-devnet.bcy-v.metalhosts.com IP" + run: | + echo "=== route get 35.182.74.119 ===" + route get 35.182.74.119 || true + + - name: "[DEBUG] 5 - DNS resolution (LB IPs returned by app connector DoH)" + run: | + echo "=== getent hosts canton-devnet.bcy-v.metalhosts.com ===" + getent hosts canton-devnet.bcy-v.metalhosts.com || true + - - name: "[DEBUG] 4 - Wait 15s for app connector route advertisement" + - name: "[DEBUG] 6 - Wait 15s for app connector route advertisement" run: sleep 15 - - name: "[DEBUG] 5 - Routes (expect LB IPs via tailscale0, but they are MISSING)" + - name: "[DEBUG] 7 - Routes (expect LB IPs via tailscale0, but they are MISSING)" run: | echo "=== All routes via tailscale ===" ip route show table all | grep tailscale || echo "(no tailscale routes found)" @@ -66,7 +76,7 @@ jobs: echo "=== Full routing table ===" ip route show table all - - name: "[DEBUG] 6 - Connectivity test (403 = reached ALB via public internet, not via app connector)" + - name: "[DEBUG] 8 - Connectivity test (403 = reached ALB via public internet, not via app connector)" run: | curl -v --connect-timeout 30 -H "Content-Type: application/json" \ https://canton-devnet.bcy-v.metalhosts.com/metrics/participant 2>&1 || true From 1f622876e45c906bb33b0de1c2d7cd0541b612bc Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 8 May 2026 16:55:29 -0300 Subject: [PATCH 18/21] Testing new canton workflow with auth --- .github/workflows/test-tailscale.yml | 120 ++++++++++++++------------- 1 file changed, 62 insertions(+), 58 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index dc10ccfdc..856c39da6 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -1,20 +1,57 @@ +# Required secrets: +# TS_OAUTH_CLIENT_ID_CANTON — Tailscale OAuth client ID (repo-wide) +# TS_AUDIENCE_CANTON — Tailscale audience tag (repo-wide) +# +# Per environment (devnet, testnet, mainnet): +# CANTON_OKTA_AUTHORIZER_ — Okta issuer URL (e.g. https://trial.okta.com/oauth2/default) +# CANTON_OKTA_CLIENT_ID_ — Okta OAuth client ID for CI/CD +# CANTON_OKTA_CLIENT_SECRET_ — Okta OAuth client secret for CI/CD + name: Test Tailscale Connectivity -run-name: Test Tailscale connectivity to Canton Devnet +run-name: "Test Tailscale connectivity to Canton (${{ inputs.canton_environment || 'devnet' }})" on: - pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: + inputs: + canton_environment: + description: "Canton environment to test" + type: choice + options: + - devnet + - testnet + - mainnet + default: devnet jobs: test-tailscale: - name: Test Tailscale Connectivity + name: "Test Tailscale Connectivity (${{ inputs.canton_environment }})" timeout-minutes: 10 runs-on: ubuntu-latest permissions: - id-token: write + id-token: write contents: read + env: + CANTON_ENV: ${{ inputs.canton_environment }} + CANTON_OKTA_AUTHORIZER: >- + ${{ + inputs.canton_environment == 'testnet' && secrets.CANTON_OKTA_AUTHORIZER_TESTNET || + inputs.canton_environment == 'mainnet' && secrets.CANTON_OKTA_AUTHORIZER_MAINNET || + secrets.CANTON_OKTA_AUTHORIZER_DEVNET + }} + CANTON_OKTA_CLIENT_ID: >- + ${{ + inputs.canton_environment == 'testnet' && secrets.CANTON_OKTA_CLIENT_ID_TESTNET || + inputs.canton_environment == 'mainnet' && secrets.CANTON_OKTA_CLIENT_ID_MAINNET || + secrets.CANTON_OKTA_CLIENT_ID_DEVNET + }} + CANTON_OKTA_CLIENT_SECRET: >- + ${{ + inputs.canton_environment == 'testnet' && secrets.CANTON_OKTA_CLIENT_SECRET_TESTNET || + inputs.canton_environment == 'mainnet' && secrets.CANTON_OKTA_CLIENT_SECRET_MAINNET || + secrets.CANTON_OKTA_CLIENT_SECRET_DEVNET + }} + steps: - name: Checkout code uses: actions/checkout@v4 @@ -22,61 +59,28 @@ jobs: - name: Connect to Tailscale uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 # v4.1.1 with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON_DEVNET }} - audience: ${{ secrets.TS_AUDIENCE_CANTON_DEVNET }} + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON }} + audience: ${{ secrets.TS_AUDIENCE_CANTON }} tags: tag:bcy-validators-gha - - name: Configure Tailscale for app connector routes - run: | - sudo tailscale set --accept-routes=true - sudo tailscale set --operator=$USER - - - name: "[DEBUG] 1 - Tailscale node info" - run: | - echo "=== Tailscale status (this node + peers) ===" - tailscale status || true - echo "" - echo "=== This node's Tailscale IP ===" - tailscale ip -4 || true - echo "" - echo "=== Tailscale debug prefs (confirm AcceptRoutes=true) ===" - tailscale debug prefs 2>&1 | head -30 || true - - - name: "[DEBUG] 2 - DNS configuration (Split DNS shows app connector DoH)" - run: | - echo "=== Tailscale DNS status ===" - tailscale dns status || true - echo "" - echo "=== /etc/resolv.conf ===" - cat /etc/resolv.conf - - - name: "[DEBUG] 3 - DNS resolution (LB IPs returned by app connector DoH)" - run: | - echo "=== getent hosts canton-devnet.bcy-v.metalhosts.com ===" - getent hosts canton-devnet.bcy-v.metalhosts.com || true - - name: "[DEBUG] 4 - Get route to canton-devnet.bcy-v.metalhosts.com IP" - run: | - echo "=== route get 35.182.74.119 ===" - route get 35.182.74.119 || true - - - name: "[DEBUG] 5 - DNS resolution (LB IPs returned by app connector DoH)" - run: | - echo "=== getent hosts canton-devnet.bcy-v.metalhosts.com ===" - getent hosts canton-devnet.bcy-v.metalhosts.com || true - - - - name: "[DEBUG] 6 - Wait 15s for app connector route advertisement" - run: sleep 15 - - - name: "[DEBUG] 7 - Routes (expect LB IPs via tailscale0, but they are MISSING)" + - name: Get Okta access token + id: okta run: | - echo "=== All routes via tailscale ===" - ip route show table all | grep tailscale || echo "(no tailscale routes found)" - echo "" - echo "=== Full routing table ===" - ip route show table all + RESPONSE=$(curl -s --fail -X POST \ + "${CANTON_OKTA_AUTHORIZER}/v1/token" \ + -u "${CANTON_OKTA_CLIENT_ID}:${CANTON_OKTA_CLIENT_SECRET}" \ + -d "grant_type=client_credentials&scope=canton") + TOKEN=$(echo "$RESPONSE" | jq -r '.access_token') + if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then + echo "::error::Failed to obtain Okta access token" + exit 1 + fi + echo "::add-mask::${TOKEN}" + echo "access_token=${TOKEN}" >> "$GITHUB_OUTPUT" - - name: "[DEBUG] 8 - Connectivity test (403 = reached ALB via public internet, not via app connector)" + - name: Verify Canton connectivity run: | - curl -v --connect-timeout 30 -H "Content-Type: application/json" \ - https://canton-devnet.bcy-v.metalhosts.com/metrics/participant 2>&1 || true + curl --fail --connect-timeout 30 \ + -H "Authorization: Bearer ${{ steps.okta.outputs.access_token }}" \ + -H "Content-Type: application/json" \ + "https://canton-${CANTON_ENV}.bcy-v.metalhosts.com/api/json/v2/parties/ccipOwner::122009b21cd8f52c55fc9854f7b1771837fa661f47529956ed40708bf656905b5721" From 682e8c2005fa0c6c8801fa59e4e78f8aadb23d7b Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Fri, 8 May 2026 17:05:50 -0300 Subject: [PATCH 19/21] Updated scope --- .github/workflows/test-tailscale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 856c39da6..2ddbf5aa1 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -69,7 +69,7 @@ jobs: RESPONSE=$(curl -s --fail -X POST \ "${CANTON_OKTA_AUTHORIZER}/v1/token" \ -u "${CANTON_OKTA_CLIENT_ID}:${CANTON_OKTA_CLIENT_SECRET}" \ - -d "grant_type=client_credentials&scope=canton") + -d "grant_type=client_credentials&scope=daml_ledger_api") TOKEN=$(echo "$RESPONSE" | jq -r '.access_token') if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then echo "::error::Failed to obtain Okta access token" From 705be71116aedaade8ead1ef91b97871c6012741 Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Wed, 24 Jun 2026 17:57:32 -0300 Subject: [PATCH 20/21] Testing connectivity against RPC proxy --- .github/workflows/test-tailscale.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 2ddbf5aa1..2290db3cc 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -84,3 +84,14 @@ jobs: -H "Authorization: Bearer ${{ steps.okta.outputs.access_token }}" \ -H "Content-Type: application/json" \ "https://canton-${CANTON_ENV}.bcy-v.metalhosts.com/api/json/v2/parties/ccipOwner::122009b21cd8f52c55fc9854f7b1771837fa661f47529956ed40708bf656905b5721" + + - name: Verify RPC Proxy connectivity + curl -s -X POST "https://rpcs.cldev.sh/ethereum/sepolia" \ + -H "Content-Type: application/json" \ + --data '{ + "jsonrpc":"2.0", + "method":"eth_blockNumber", + "params":[], + "id":1 + }' + \ No newline at end of file From 8601015ac1b0b62a1b0d448381c8bba2610b7e2a Mon Sep 17 00:00:00 2001 From: Nahuel Batista Date: Wed, 24 Jun 2026 18:00:27 -0300 Subject: [PATCH 21/21] Fixed RPC connectivity job --- .github/workflows/test-tailscale.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-tailscale.yml b/.github/workflows/test-tailscale.yml index 2290db3cc..acf0c097a 100644 --- a/.github/workflows/test-tailscale.yml +++ b/.github/workflows/test-tailscale.yml @@ -84,8 +84,9 @@ jobs: -H "Authorization: Bearer ${{ steps.okta.outputs.access_token }}" \ -H "Content-Type: application/json" \ "https://canton-${CANTON_ENV}.bcy-v.metalhosts.com/api/json/v2/parties/ccipOwner::122009b21cd8f52c55fc9854f7b1771837fa661f47529956ed40708bf656905b5721" - + - name: Verify RPC Proxy connectivity + run: | curl -s -X POST "https://rpcs.cldev.sh/ethereum/sepolia" \ -H "Content-Type: application/json" \ --data '{ @@ -93,5 +94,4 @@ jobs: "method":"eth_blockNumber", "params":[], "id":1 - }' - \ No newline at end of file + }' \ No newline at end of file