Skip to content

Commit 868e697

Browse files
PigbibiPigbibi
authored
Add Binance dispatch retry guard (#50)
1 parent ad89878 commit 868e697

3 files changed

Lines changed: 104 additions & 20 deletions

File tree

‎.github/workflows/install-dispatch-guard.yml‎

Lines changed: 61 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ jobs:
6464
6565
ENV_FILE="${BINANCE_PLATFORM_DISPATCH_ENV:-$HOME/.config/binance-platform/dispatch.env}"
6666
LOG_FILE="${BINANCE_PLATFORM_DISPATCH_LOG:-$HOME/binance-quant/ops/dispatch-runtime.log}"
67+
DISPATCH_MAX_ATTEMPTS="${DISPATCH_MAX_ATTEMPTS:-4}"
68+
DISPATCH_RETRY_BASE_SECONDS="${DISPATCH_RETRY_BASE_SECONDS:-15}"
6769
6870
notify_telegram() {
6971
local message="$1"
@@ -96,6 +98,13 @@ jobs:
9698
exit 1
9799
}
98100
101+
is_retryable_http_status() {
102+
case "$1" in
103+
000|500|502|503|504) return 0 ;;
104+
*) return 1 ;;
105+
esac
106+
}
107+
99108
mkdir -p "$(dirname "$LOG_FILE")"
100109
101110
[ -f "$ENV_FILE" ] || fail_dispatch "missing env file" "$ENV_FILE"
@@ -108,32 +117,64 @@ jobs:
108117
fi
109118
done
110119
120+
if ! [[ "$DISPATCH_MAX_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then
121+
fail_dispatch "invalid DISPATCH_MAX_ATTEMPTS" "$DISPATCH_MAX_ATTEMPTS"
122+
fi
123+
if ! [[ "$DISPATCH_RETRY_BASE_SECONDS" =~ ^[0-9]+$ ]]; then
124+
fail_dispatch "invalid DISPATCH_RETRY_BASE_SECONDS" "$DISPATCH_RETRY_BASE_SECONDS"
125+
fi
126+
111127
response_file=$(mktemp)
112128
stderr_file=$(mktemp)
113129
trap 'rm -f "$response_file" "$stderr_file"' EXIT
114130
115-
set +e
116-
http_status=$(curl --silent --show-error \
117-
--output "$response_file" \
118-
--write-out "%{http_code}" \
119-
-X POST \
120-
-H "Accept: application/vnd.github+json" \
121-
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
122-
-H "X-GitHub-Api-Version: 2022-11-28" \
123-
"https://api.github.com/repos/${REPO}/actions/workflows/${WORKFLOW}/dispatches" \
124-
-d "{\"ref\":\"${REF}\"}" 2>"$stderr_file")
125-
curl_exit=$?
126-
set -e
127-
128-
if [ "$curl_exit" -ne 0 ]; then
129-
fail_dispatch "curl exited ${curl_exit}" "$(head -c 800 "$stderr_file")"
130-
fi
131+
attempt=1
132+
while [ "$attempt" -le "$DISPATCH_MAX_ATTEMPTS" ]; do
133+
: > "$response_file"
134+
: > "$stderr_file"
135+
136+
set +e
137+
http_status=$(curl --silent --show-error \
138+
--connect-timeout 20 \
139+
--max-time 60 \
140+
--output "$response_file" \
141+
--write-out "%{http_code}" \
142+
-X POST \
143+
-H "Accept: application/vnd.github+json" \
144+
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
145+
-H "X-GitHub-Api-Version: 2022-11-28" \
146+
"https://api.github.com/repos/${REPO}/actions/workflows/${WORKFLOW}/dispatches" \
147+
-d "{\"ref\":\"${REF}\"}" 2>"$stderr_file")
148+
curl_exit=$?
149+
set -e
150+
151+
if [ "$curl_exit" -eq 0 ] && [ "$http_status" = "204" ]; then
152+
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) dispatched ${REPO}/${WORKFLOW} ref=${REF} attempt=${attempt}/${DISPATCH_MAX_ATTEMPTS}" >> "$LOG_FILE"
153+
exit 0
154+
fi
131155
132-
if [ "$http_status" != "204" ]; then
133-
fail_dispatch "GitHub dispatch returned HTTP ${http_status}" "$(head -c 800 "$response_file")"
134-
fi
156+
retryable=false
157+
if [ "$curl_exit" -ne 0 ]; then
158+
reason="curl exited ${curl_exit}"
159+
details="$(head -c 800 "$stderr_file")"
160+
retryable=true
161+
else
162+
reason="GitHub dispatch returned HTTP ${http_status}"
163+
details="$(head -c 800 "$response_file")"
164+
if is_retryable_http_status "$http_status"; then
165+
retryable=true
166+
fi
167+
fi
135168
136-
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) dispatched ${REPO}/${WORKFLOW} ref=${REF}" >> "$LOG_FILE"
169+
if [ "$attempt" -ge "$DISPATCH_MAX_ATTEMPTS" ] || [ "$retryable" != "true" ]; then
170+
fail_dispatch "$reason" "$details"
171+
fi
172+
173+
delay=$((DISPATCH_RETRY_BASE_SECONDS * attempt))
174+
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) dispatch retry ${attempt}/${DISPATCH_MAX_ATTEMPTS}: ${reason} ${details}" >> "$LOG_FILE"
175+
sleep "$delay"
176+
attempt=$((attempt + 1))
177+
done
137178
EOF
138179
139180
chmod 700 "$SCRIPT_PATH"

‎docs/operator_runbook.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ The monthly execution pool is locked to the accepted upstream `version` / `as_of
7171
- `main.yml` is `workflow_dispatch` only.
7272
- GitHub Actions no longer owns the hourly cadence for runtime execution in this repo.
7373
- Production cadence should come from one external scheduler, for example VPS cron calling the GitHub Actions dispatch API.
74+
- The VPS dispatch guard retries bounded transient failures such as network errors and GitHub `500`/`502`/`503`/`504`, but still alerts immediately for configuration and permission failures.
7475
- Avoid overlapping dispatches from multiple schedulers or from a second manual run while the current runtime job is still in progress.
7576

7677
## Degraded Mode Ladder

‎tests/test_dispatch_guard_workflow.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
from __future__ import annotations
2+
3+
import unittest
4+
from pathlib import Path
5+
6+
7+
ROOT = Path(__file__).resolve().parents[1]
8+
WORKFLOW = ROOT / ".github" / "workflows" / "install-dispatch-guard.yml"
9+
10+
11+
class DispatchGuardWorkflowTests(unittest.TestCase):
12+
@classmethod
13+
def setUpClass(cls) -> None:
14+
cls.workflow_text = WORKFLOW.read_text(encoding="utf-8")
15+
16+
def test_dispatch_guard_retries_transient_failures(self) -> None:
17+
text = self.workflow_text
18+
19+
self.assertIn('DISPATCH_MAX_ATTEMPTS="${DISPATCH_MAX_ATTEMPTS:-4}"', text)
20+
self.assertIn('DISPATCH_RETRY_BASE_SECONDS="${DISPATCH_RETRY_BASE_SECONDS:-15}"', text)
21+
self.assertIn("is_retryable_http_status()", text)
22+
self.assertIn("000|500|502|503|504) return 0 ;;", text)
23+
self.assertIn('retryable=true', text)
24+
self.assertIn('sleep "$delay"', text)
25+
self.assertIn('dispatch retry ${attempt}/${DISPATCH_MAX_ATTEMPTS}', text)
26+
27+
def test_dispatch_guard_keeps_non_retryable_failures_immediate(self) -> None:
28+
text = self.workflow_text
29+
30+
self.assertIn('if [ "$attempt" -ge "$DISPATCH_MAX_ATTEMPTS" ] || [ "$retryable" != "true" ]; then', text)
31+
self.assertIn('fail_dispatch "$reason" "$details"', text)
32+
self.assertIn('GitHub dispatch returned HTTP ${http_status}', text)
33+
34+
def test_dispatch_guard_bounds_curl_runtime(self) -> None:
35+
text = self.workflow_text
36+
37+
self.assertIn("--connect-timeout 20", text)
38+
self.assertIn("--max-time 60", text)
39+
40+
41+
if __name__ == "__main__":
42+
unittest.main()

0 commit comments

Comments
 (0)