From dd5eaecfe5bce11138bd396c17361867b0d09ccd Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 11 Jul 2026 03:00:19 +0800 Subject: [PATCH] fix(ci): persist validated lifecycle baselines Co-Authored-By: Codex --- .github/workflows/drift-check.yml | 73 +++------------ scripts/run_walk_forward_backtest.py | 117 +++++++++++++++++++++--- tests/test_drift_workflow_config.py | 15 +++ tests/test_run_walk_forward_backtest.py | 87 ++++++++++++++++++ 4 files changed, 216 insertions(+), 76 deletions(-) create mode 100644 tests/test_drift_workflow_config.py create mode 100644 tests/test_run_walk_forward_backtest.py diff --git a/.github/workflows/drift-check.yml b/.github/workflows/drift-check.yml index 002d96b..1e82ba8 100644 --- a/.github/workflows/drift-check.yml +++ b/.github/workflows/drift-check.yml @@ -14,65 +14,14 @@ permissions: jobs: drift: - runs-on: ubuntu-latest - timeout-minutes: 15 - env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - STRATEGY_DOMAIN: hk_equity - - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Checkout QuantPlatformKit - uses: actions/checkout@v6 - with: - repository: QuantStrategyLab/QuantPlatformKit - ref: main - path: external/QuantPlatformKit - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - set -euo pipefail - python -m pip install --upgrade pip - python -m pip install -e . pandas - python -m pip install --no-deps -e external/QuantPlatformKit - - - name: Run drift detection - run: quant-lifecycle drift --domain hk_equity --no-alerts - - - name: Sync drift alerts to GitHub Issues - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_REPOSITORY: ${{ github.repository }} - run: python scripts/run_drift_github_issues.py - - - name: Checkout AIAuditBridge - uses: actions/checkout@v6 - with: - repository: QuantStrategyLab/AIAuditBridge - ref: main - path: external/AIAuditBridge - - - name: Dual-review critical drift - env: - AIAUDIT_BRIDGE_ROOT: external/AIAuditBridge - CODEX_AUDIT_SERVICE_URL: ${{ secrets.CODEX_AUDIT_SERVICE_URL }} - AI_GATEWAY_SERVICE_URL: ${{ vars.AI_GATEWAY_SERVICE_URL }} - run: | - script="external/AIAuditBridge/scripts/run_drift_dual_review.py" - if [ ! -f "$script" ]; then - echo "::notice::dual-review scripts unavailable; skipping until AIAuditBridge is merged" - exit 0 - fi - if [ -z "${CODEX_AUDIT_SERVICE_URL:-}" ]; then - echo "::notice::CODEX_AUDIT_SERVICE_URL not configured; skipping dual-review dispatch" - exit 0 - fi - PYTHONPATH=external/AIAuditBridge python "$script" \ - --domain "${STRATEGY_DOMAIN}" --dispatch + uses: QuantStrategyLab/QuantPlatformKit/.github/workflows/reusable-drift-check.yml@335c7a22bc3f570bd5705427ccc40172eda6b289 + with: + strategy_domain: hk_equity + caller_event_name: ${{ github.event_name }} + caller_pr_head_repository: ${{ github.event.pull_request.head.repo.full_name || '' }} + snapshot_repository: QuantStrategyLab/HkEquitySnapshotPipelines + snapshot_checkout_path: external/HkEquitySnapshotPipelines + ai_gateway_service_url: ${{ vars.AI_GATEWAY_SERVICE_URL }} + secrets: + codex_audit_service_url: ${{ secrets.CODEX_AUDIT_SERVICE_URL }} + snapshot_repository_token: ${{ secrets.SNAPSHOT_REPOSITORY_TOKEN }} diff --git a/scripts/run_walk_forward_backtest.py b/scripts/run_walk_forward_backtest.py index 4007efc..49bea81 100644 --- a/scripts/run_walk_forward_backtest.py +++ b/scripts/run_walk_forward_backtest.py @@ -4,12 +4,18 @@ from __future__ import annotations import argparse +import copy +import hashlib import json +import tempfile from datetime import date from pathlib import Path from typing import Any +import pandas as pd + from hk_equity_strategies.backtest.orchestrator_runner import SUPPORTED_PROFILES, build_backtest_runner +from hk_equity_strategies.backtest.orchestrator_runner import _synthetic_market_history as _runner_synthetic_market_history from hk_equity_strategies.strategies.hk_equity_combo import PROFILE_NAME as HK_EQUITY_COMBO_PROFILE from hk_equity_strategies.strategies.hk_global_etf_tactical_rotation import DEFAULT_MIN_HISTORY_DAYS @@ -25,6 +31,7 @@ "combo_mode": "dynamic", }, } +MIN_SYNTHETIC_DAYS = 700 def _result_payload(item: Any) -> dict[str, Any]: @@ -40,13 +47,53 @@ def _result_payload(item: Any) -> dict[str, Any]: } +def _baseline_param_set_id( + profile: str, + params: dict[str, Any], +) -> str: + fingerprint = hashlib.sha256(json.dumps(params, sort_keys=True, default=str).encode("utf-8")).hexdigest()[:12] + return f"{profile}_baseline_{fingerprint}" + + +def _baseline_identity_params( + params: dict[str, Any], + *, + synthetic_days: int | None, +) -> dict[str, Any]: + identity = copy.deepcopy(params) + if synthetic_days is not None: + identity["_synthetic_days"] = synthetic_days + return identity + + +def _clone_market_history(market_history: pd.DataFrame | None) -> pd.DataFrame | None: + return market_history.copy(deep=True) if market_history is not None else None + + +def _shared_market_history(params: dict[str, Any], synthetic_days: int, market_history: pd.DataFrame | None) -> pd.DataFrame: + if market_history is not None: + return _clone_market_history(market_history) + min_history_days = int(params.get("min_history_days", DEFAULT_MIN_HISTORY_DAYS)) + if int(synthetic_days) < min_history_days: + raise ValueError(f"synthetic_days must be >= {min_history_days} for profile={params!r}") + return _runner_synthetic_market_history(days=int(synthetic_days)) + + +def _build_runner(*, profile: str, synthetic_days: int, market_history: pd.DataFrame | None = None): + return build_backtest_runner( + profile, + market_history=market_history, + synthetic_days=synthetic_days, + ) + + def run_walk_forward( *, profile: str, windows: tuple[tuple[date, date], ...] = DEFAULT_WINDOWS, synthetic_days: int = 700, store_root: Path | None = None, - market_history: Any = None, + market_history: pd.DataFrame | None = None, ) -> dict[str, Any]: from quant_platform_kit.strategy_lifecycle.backtest_orchestrator import BacktestOrchestrator from quant_platform_kit.strategy_lifecycle.performance_store import PerformanceStore @@ -55,27 +102,69 @@ def run_walk_forward( raise ValueError(f"unsupported profile={profile!r}; supported={sorted(SUPPORTED_PROFILES)}") params = dict(PROFILE_DEFAULTS.get(profile, {"min_history_days": DEFAULT_MIN_HISTORY_DAYS})) - runner = build_backtest_runner( - profile, - market_history=market_history, - synthetic_days=synthetic_days, + store_root = store_root or Path("/tmp/hk_equity_wf_store") + store_root.mkdir(parents=True, exist_ok=True) + baseline_params = copy.deepcopy(params) + shared_market_history = _shared_market_history(baseline_params, synthetic_days, market_history) + with tempfile.TemporaryDirectory(prefix=f"{profile}_wf_", dir=store_root) as scratch_dir: + scratch_store = PerformanceStore(local_root=Path(scratch_dir)) + scratch_orchestrator = BacktestOrchestrator(store=scratch_store) + scratch_orchestrator.register_runner( + "hk_equity", + _build_runner( + profile=profile, + market_history=_clone_market_history(shared_market_history), + synthetic_days=synthetic_days, + ), + ) + baseline_runner = _build_runner( + profile=profile, + market_history=_clone_market_history(shared_market_history), + synthetic_days=synthetic_days, + ) + baseline_raw = baseline_runner.run( + profile, + copy.deepcopy(baseline_params), + start_date=None, + end_date=None, + ) + via_orch = scratch_orchestrator.run( + profile, + domain="hk_equity", + params=copy.deepcopy(baseline_params), + param_set_id=f"{profile}_full_compare", + start_date=None, + end_date=None, + ) + wf_params = copy.deepcopy(baseline_params) + wf_results = scratch_orchestrator.walk_forward( + profile, + domain="hk_equity", + params=wf_params, + windows=windows, + param_set_id=f"{profile}_wf", + ) + baseline_store_params = _baseline_identity_params( + baseline_params, + synthetic_days=synthetic_days if market_history is None else None, ) - store = PerformanceStore(local_root=store_root or Path("/tmp/hk_equity_wf_store")) + store = PerformanceStore(local_root=store_root) orchestrator = BacktestOrchestrator(store=store) - orchestrator.register_runner("hk_equity", runner) - - baseline = runner.run(profile, params) - wf_results = orchestrator.walk_forward( - profile, + baseline = orchestrator.persist_result( + baseline_raw, + strategy_profile=profile, domain="hk_equity", - params=params, - windows=windows, - param_set_id=f"{profile}_wf", + params=baseline_params, + param_set_id=_baseline_param_set_id( + profile, + baseline_store_params, + ), ) return { "strategy_profile": profile, "domain": "hk_equity", "baseline": _result_payload(baseline), + "orchestrator_full_window": _result_payload(via_orch), "walk_forward_folds": [_result_payload(item) for item in wf_results], "source": "BacktestOrchestrator.walk_forward", } diff --git a/tests/test_drift_workflow_config.py b/tests/test_drift_workflow_config.py new file mode 100644 index 0000000..281926e --- /dev/null +++ b/tests/test_drift_workflow_config.py @@ -0,0 +1,15 @@ +from pathlib import Path + + +def test_drift_workflow_wires_snapshot_repo_and_lifecycle_env() -> None: + workflow = (Path(__file__).resolve().parents[1] / ".github" / "workflows" / "drift-check.yml").read_text(encoding="utf-8") + + assert "uses: QuantStrategyLab/QuantPlatformKit/.github/workflows/reusable-drift-check.yml@335c7a22bc3f570bd5705427ccc40172eda6b289" in workflow + assert "strategy_domain: hk_equity" in workflow + assert "caller_event_name: ${{ github.event_name }}" in workflow + assert "caller_pr_head_repository: ${{ github.event.pull_request.head.repo.full_name || '' }}" in workflow + assert "snapshot_repository: QuantStrategyLab/HkEquitySnapshotPipelines" in workflow + assert "snapshot_checkout_path: external/HkEquitySnapshotPipelines" in workflow + assert "ai_gateway_service_url: ${{ vars.AI_GATEWAY_SERVICE_URL }}" in workflow + assert "codex_audit_service_url: ${{ secrets.CODEX_AUDIT_SERVICE_URL }}" in workflow + assert "snapshot_repository_token: ${{ secrets.SNAPSHOT_REPOSITORY_TOKEN }}" in workflow diff --git a/tests/test_run_walk_forward_backtest.py b/tests/test_run_walk_forward_backtest.py new file mode 100644 index 0000000..9a1576c --- /dev/null +++ b/tests/test_run_walk_forward_backtest.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pandas as pd +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "src" +QPK_SRC = ROOT.parent / "QuantPlatformKit" / "src" +if str(QPK_SRC) not in sys.path: + sys.path.insert(0, str(QPK_SRC)) +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from hk_equity_strategies.backtest.orchestrator_runner import _synthetic_market_history +from scripts.run_walk_forward_backtest import _baseline_param_set_id, _clone_market_history, run_walk_forward + + +def test_run_walk_forward_persists_lifecycle_baseline(tmp_path: Path) -> None: + payload = run_walk_forward( + profile="hk_global_etf_tactical_rotation", + synthetic_days=700, + store_root=tmp_path, + ) + + records = [ + json.loads(path.read_text(encoding="utf-8")) + for path in (tmp_path / "backtest" / "hk_equity" / "hk_global_etf_tactical_rotation").glob("*.json") + ] + + assert payload["baseline"]["sharpe_ratio"] is not None + baseline_records = [record for record in records if "_baseline_" in record["param_set_id"]] + assert baseline_records + assert all(record["params"] == {"min_history_days": 260} for record in baseline_records) + assert not any("_wf" in record["param_set_id"] for record in records) + assert payload["orchestrator_full_window"]["sharpe_ratio"] is not None + assert payload["walk_forward_folds"] + + +def test_clone_market_history_returns_independent_dataframe() -> None: + history = _synthetic_market_history(days=10) + + cloned = _clone_market_history(history) + + assert cloned is not history + assert cloned is not None + cloned.loc[:, "close"] = 0.0 + assert not cloned["close"].equals(history["close"]) + + +def test_run_walk_forward_accepts_explicit_market_history(tmp_path: Path) -> None: + history = _synthetic_market_history(days=1200) + + payload = run_walk_forward( + profile="hk_global_etf_tactical_rotation", + synthetic_days=700, + store_root=tmp_path, + market_history=history, + ) + + assert payload["baseline"]["sharpe_ratio"] is not None + assert isinstance(history, pd.DataFrame) + + +def test_baseline_param_set_id_tracks_runner_inputs() -> None: + first = _baseline_param_set_id( + "hk_global_etf_tactical_rotation", + {"min_history_days": 260, "_synthetic_days": 700}, + ) + second = _baseline_param_set_id( + "hk_global_etf_tactical_rotation", + {"min_history_days": 260, "_synthetic_days": 900}, + ) + + assert first != second + + +def test_run_walk_forward_rejects_too_short_synthetic_history(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="synthetic_days must be >= 260"): + run_walk_forward( + profile="hk_global_etf_tactical_rotation", + synthetic_days=220, + store_root=tmp_path, + )