From 32e362f93f7475bcacd7b0dd33bad7c92f6a7601 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 27 Jun 2026 04:58:26 +0800 Subject: [PATCH] Use Binance Vision fallback for GitHub Actions BTC downloads. GitHub-hosted runners receive HTTP 451 from api.binance.com, so the publish script now tries data-api.binance.vision and binance.us before falling back to the primary endpoint. Co-authored-by: Cursor --- scripts/publish_ibit_btc_platform_handoff.py | 53 ++++++++++++++----- .../test_publish_ibit_btc_platform_handoff.py | 8 +++ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/scripts/publish_ibit_btc_platform_handoff.py b/scripts/publish_ibit_btc_platform_handoff.py index 06e6483..ce3d594 100644 --- a/scripts/publish_ibit_btc_platform_handoff.py +++ b/scripts/publish_ibit_btc_platform_handoff.py @@ -10,8 +10,10 @@ from datetime import UTC, date, datetime, timedelta from pathlib import Path -BINANCE_BTCUSDT_DAILY_URL = ( - "https://api.binance.com/api/v3/klines?symbol=BTCUSDT&interval=1d&limit=800" +BINANCE_BTCUSDT_DAILY_URLS = ( + "https://data-api.binance.vision/api/v3/klines?symbol=BTCUSDT&interval=1d&limit=800", + "https://api.binance.us/api/v3/klines?symbol=BTCUSDT&interval=1d&limit=800", + "https://api.binance.com/api/v3/klines?symbol=BTCUSDT&interval=1d&limit=800", ) DEFAULT_CONSUMER = "us_equity:ibit_smart_dca" DEFAULT_STRATEGY = "ibit_smart_dca" @@ -35,17 +37,11 @@ def resolve_as_of(*, csv_path: Path, requested: str | None, today: date | None = return rows[-1] -def fetch_binance_btc_daily_csv(output_path: Path) -> int: +def fetch_binance_btc_daily_csv(output_path: Path) -> tuple[int, str]: """Download BTCUSDT daily OHLCV from Binance public API into a local CSV.""" output_path.parent.mkdir(parents=True, exist_ok=True) - try: - with urllib.request.urlopen(BINANCE_BTCUSDT_DAILY_URL, timeout=30) as response: - payload = json.loads(response.read().decode()) - except urllib.error.URLError as exc: - raise RuntimeError(f"failed to download Binance BTCUSDT daily klines: {exc}") from exc - - if not payload: - raise RuntimeError("Binance BTCUSDT daily klines response was empty") + payload, source_url = _download_binance_btcusdt_daily_klines() + provider_name = _provider_name_for_url(source_url) rows: list[dict[str, object]] = [] for entry in payload: @@ -70,7 +66,33 @@ def fetch_binance_btc_daily_csv(output_path: Path) -> int: writer.writeheader() writer.writerows(rows) - return len(rows) + print(f"downloaded BTCUSDT daily rows from {provider_name}") + return len(rows), provider_name + + +def _download_binance_btcusdt_daily_klines() -> tuple[list[object], str]: + errors: list[str] = [] + for url in BINANCE_BTCUSDT_DAILY_URLS: + try: + with urllib.request.urlopen(url, timeout=30) as response: + payload = json.loads(response.read().decode()) + except urllib.error.URLError as exc: + errors.append(f"{url}: {exc}") + continue + if not payload: + errors.append(f"{url}: empty response") + continue + return payload, url + joined = "; ".join(errors) or "no Binance endpoints configured" + raise RuntimeError(f"failed to download Binance BTCUSDT daily klines: {joined}") + + +def _provider_name_for_url(url: str) -> str: + if "binance.vision" in url: + return "binance_vision_public" + if "binance.us" in url: + return "binance_us_public" + return "binance_public" def build_ibit_btc_platform_handoff( @@ -80,6 +102,7 @@ def build_ibit_btc_platform_handoff( as_of: str, code_commit: str, source_version: str, + provider: str = "binance_vision_public", consumer: str = DEFAULT_CONSUMER, strategy: str = DEFAULT_STRATEGY, ) -> dict[str, Path]: @@ -104,7 +127,7 @@ def build_ibit_btc_platform_handoff( "--as-of", as_of, "--provider", - "binance_public", + provider, "--provider-dataset", "btcusdt_daily_klines", "--source-version", @@ -222,8 +245,9 @@ def main(argv: list[str] | None = None) -> int: input_csv = args.input_csv or (work_dir / "inputs" / "btc_daily.csv") code_commit = args.code_commit or __import__("os").environ.get("GITHUB_SHA", "0" * 40) + provider = "local_csv" if args.input_csv is None: - row_count = fetch_binance_btc_daily_csv(input_csv) + row_count, provider = fetch_binance_btc_daily_csv(input_csv) print(f"downloaded {row_count} BTCUSDT daily rows to {input_csv}") elif not input_csv.is_file(): print(f"error: input CSV not found: {input_csv}", file=sys.stderr) @@ -238,6 +262,7 @@ def main(argv: list[str] | None = None) -> int: as_of=as_of, code_commit=code_commit, source_version=args.source_version, + provider=provider, consumer=args.consumer, strategy=args.strategy, ) diff --git a/tests/test_publish_ibit_btc_platform_handoff.py b/tests/test_publish_ibit_btc_platform_handoff.py index 6d29ea0..fb7112c 100644 --- a/tests/test_publish_ibit_btc_platform_handoff.py +++ b/tests/test_publish_ibit_btc_platform_handoff.py @@ -7,6 +7,8 @@ import pytest from scripts.publish_ibit_btc_platform_handoff import ( + BINANCE_BTCUSDT_DAILY_URLS, + _provider_name_for_url, default_as_of, resolve_as_of, ) @@ -54,3 +56,9 @@ def test_resolve_as_of_raises_for_empty_csv(tmp_path: Path) -> None: with pytest.raises(ValueError, match="no dates found"): resolve_as_of(csv_path=csv_path, requested="2026-06-25") + + +def test_provider_name_for_url() -> None: + assert _provider_name_for_url(BINANCE_BTCUSDT_DAILY_URLS[0]) == "binance_vision_public" + assert _provider_name_for_url(BINANCE_BTCUSDT_DAILY_URLS[1]) == "binance_us_public" + assert _provider_name_for_url(BINANCE_BTCUSDT_DAILY_URLS[2]) == "binance_public"