diff --git a/.github/workflows/ai_review.yml b/.github/workflows/ai_review.yml index 503590c..e3677c0 100644 --- a/.github/workflows/ai_review.yml +++ b/.github/workflows/ai_review.yml @@ -3,78 +3,124 @@ name: AI Monthly Review "on": issues: types: [opened] + workflow_dispatch: + inputs: + issue_number: + description: "Issue number to review" + required: true jobs: ai-review: - if: contains(github.event.issue.labels.*.name, 'monthly-review') + if: contains(github.event.issue.labels.*.name, 'monthly-review') || inputs.issue_number != '' runs-on: ubuntu-latest permissions: - contents: write + contents: read + id-token: write issues: write - pull-requests: write steps: - name: Checkout uses: actions/checkout@v5 + - name: Load review issue context + id: issue_context + run: | + python3 - <<'PY' + import json + import os + import urllib.request + + repo = os.environ["GITHUB_REPOSITORY"] + issue_number = os.environ["ISSUE_NUMBER"] + token = os.environ["GITHUB_TOKEN"] + api_url = f"https://api.github.com/repos/{repo}/issues/{issue_number}" + request = urllib.request.Request( + api_url, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "crypto-leader-rotation-ai-review", + }, + ) + with urllib.request.urlopen(request) as response: + issue = json.load(response) + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + print("issue_title< 7)? - - Is dry_run=True expected or a sign that publish was not actually enabled? - - Are ranking scores reasonable (no extreme values or suspicious clustering)? - - Is pool_size consistent with the expected value (5)? - - 3. **Downstream Impact (BinanceQuant)** - - BinanceQuant is a downstream execution engine that selects the top 2 symbols - from this pool for trend rotation trading on Binance Spot. - - Note any symbols that are new to the pool vs last month (if determinable from context). - - Flag if any pool symbols might have low liquidity or high volatility concerns - for a real-money execution system. - - Remind the operator that pool changes will affect live positions at the next - BinanceQuant hourly cycle after the Firestore document updates. - - 4. **Operator Action Items** - - Summarize the operator checklist items from the report. - - Add any additional follow-up items based on your analysis. - - If publish was a dry run, note that the operator should confirm whether - a real publish is intended. - - 5. **Code Improvements (Optional)** - - If you identify concrete, low-risk improvements to scripts or configuration - in this repository, you may open a Pull Request. - - Do NOT modify strategy logic, ranking parameters, or universe filters. - - Acceptable improvements: documentation, validation checks, error messages, - report formatting, workflow configuration. - - Do NOT auto-merge. The operator will review and decide. + 1. **Release Consistency & Contract Health** + - Do release metadata, symbols, pool_size, version, and validation state agree? + - Are there any warnings, stale artifacts, or publish flags that weaken trust in this release? + + 2. **Selector Quality Review** + - Does the official 5-symbol pool look reasonable for a Binance Spot mainstream leader universe? + - Does the ranking preview show healthy score separation, or suspicious clustering/tie behavior? + - Are any symbols potentially borderline on liquidity, stability, or crowding risk for downstream live use? + + 3. **Research Coverage & Evidence Gaps** + - Is shadow/challenger coverage present? + - If not present, explicitly say that strategy-optimization evidence is incomplete. + - If present, explain whether official baseline still looks justified versus challengers. + + 4. **Downstream Impact (BinancePlatform)** + - What are the likely downstream implications for BinancePlatform, which consumes this pool monthly? + - Highlight any pool changes or symbol characteristics that could affect downstream rotation behavior or turnover. + + 5. **Strategy Optimization Directions** + - Suggest the 1-3 highest-value next research directions in this repository. + - Prefer ideas such as challenger track design, shadow build coverage, ranking weight review, regime-weight review, universe-filter tuning, and validation additions. + - Do not suggest arbitrary complexity; prioritize explainable, low-risk, testable improvements. + + 6. **Operator Action Items** + - Summarize immediate operator checks for this month. + - Separate "safe to keep production as-is" from "needs more research evidence". ## Output Format @@ -85,13 +131,15 @@ jobs: ### Release Consistency ... - ### Anomalies + ### Selector Quality ... - ### Downstream Impact (BinanceQuant) + ### Research Coverage Gaps ... - ### Operator Action Items + ### Downstream Impact (BinancePlatform) ... - ### Code Improvement Suggestions + ### Strategy Optimization Directions + ... + ### Operator Action Items ... --- @@ -99,11 +147,24 @@ jobs: ### 发布一致性检查 ... - ### 异常检测 + ### 选币器质量评估 ... - ### 下游影响(BinanceQuant) + ### 研究覆盖缺口 ... - ### 操作员待办事项 + ### 下游影响(BinancePlatform) + ... + ### 策略优化方向 ... - ### 代码改进建议 + ### 操作员待办事项 ... + + - name: Post AI review issue comment + if: steps.claude_review.outcome == 'success' + run: | + python3 scripts/post_monthly_ai_review_comment.py \ + --repo "${GITHUB_REPOSITORY}" \ + --issue-number "${{ inputs.issue_number || github.event.issue.number }}" \ + --execution-file "${{ steps.claude_review.outputs.execution_file }}" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/monthly_publish.yml b/.github/workflows/monthly_publish.yml index dae75c4..51bdbee 100644 --- a/.github/workflows/monthly_publish.yml +++ b/.github/workflows/monthly_publish.yml @@ -13,6 +13,7 @@ jobs: - Linux - X64 permissions: + actions: write contents: write id-token: write issues: write @@ -91,9 +92,11 @@ jobs: - name: Create Monthly Review Issue if: success() && env.PUBLISH_ENABLED != 'false' + id: review_issue env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_OUTPUT: ${{ github.output }} run: | set -euo pipefail python - <<'PY' @@ -154,6 +157,46 @@ jobs: }, ) print(f"Created issue #{issue['number']}: {issue['html_url']}") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + print(f"issue_number={issue['number']}", file=output) + print(f"issue_url={issue['html_url']}", file=output) + PY + + - name: Trigger AI Monthly Review + if: success() && env.PUBLISH_ENABLED != 'false' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + ISSUE_NUMBER: ${{ steps.review_issue.outputs.issue_number }} + run: | + set -euo pipefail + python - <<'PY' + import json + import os + import urllib.request + + token = os.environ["GITHUB_TOKEN"] + repository = os.environ["GITHUB_REPOSITORY"] + ref_name = os.environ["GITHUB_REF_NAME"] + issue_number = os.environ["ISSUE_NUMBER"] + payload = {"ref": ref_name, "inputs": {"issue_number": issue_number}} + request = urllib.request.Request( + f"https://api.github.com/repos/{repository}/actions/workflows/ai_review.yml/dispatches", + data=json.dumps(payload).encode("utf-8"), + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "crypto-leader-rotation-monthly-publish", + }, + ) + with urllib.request.urlopen(request) as response: + if response.status not in (201, 204): + raise RuntimeError(f"Unexpected dispatch status: {response.status}") + print(f"Triggered ai_review.yml for issue #{issue_number} on ref {ref_name}") PY - name: Write Release Heartbeat diff --git a/scripts/post_monthly_ai_review_comment.py b/scripts/post_monthly_ai_review_comment.py new file mode 100644 index 0000000..143747e --- /dev/null +++ b/scripts/post_monthly_ai_review_comment.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + + +COMMENT_MARKER = "" +DEFAULT_API_URL = "https://api.github.com" + + +def extract_latest_assistant_text(execution_log: list[dict[str, Any]]) -> str: + for turn in reversed(execution_log): + if turn.get("type") != "assistant": + continue + + content_items = turn.get("message", {}).get("content", []) + text_parts = [ + item.get("text", "").strip() + for item in content_items + if item.get("type") == "text" and item.get("text", "").strip() + ] + if text_parts: + return "\n\n".join(text_parts).strip() + + raise ValueError("No assistant review text found in Claude execution log") + + +def build_comment_body(review_text: str, run_url: str | None = None) -> str: + body = f"{COMMENT_MARKER}\n## Claude Monthly Strategy Review\n\n{review_text.strip()}" + if run_url: + body += f"\n\n---\n_Generated by AI Monthly Review workflow: {run_url}_" + return body + + +def github_request( + method: str, + url: str, + token: str, + payload: dict[str, Any] | None = None, +) -> Any: + data = None + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "crypto-leader-rotation-monthly-ai-review", + } + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + + request = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(request) as response: + charset = response.headers.get_content_charset("utf-8") + raw = response.read().decode(charset) + return json.loads(raw) if raw else None + + +def upsert_issue_comment( + *, + api_url: str, + repo: str, + issue_number: int, + token: str, + body: str, +) -> None: + comments_url = f"{api_url}/repos/{repo}/issues/{issue_number}/comments" + comments = github_request("GET", comments_url, token) + existing = next( + ( + comment + for comment in comments + if COMMENT_MARKER in comment.get("body", "") + ), + None, + ) + + if existing: + github_request( + "PATCH", + f"{api_url}/repos/{repo}/issues/comments/{existing['id']}", + token, + {"body": body}, + ) + print(f"Updated issue comment {existing['id']}") + return + + github_request("POST", comments_url, token, {"body": body}) + print(f"Created issue comment for issue #{issue_number}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Extract Claude review text from execution output and upsert an issue comment.", + ) + parser.add_argument("--repo", required=True, help="owner/repo") + parser.add_argument("--issue-number", required=True, type=int) + parser.add_argument("--execution-file", required=True, type=Path) + parser.add_argument("--api-url", default=DEFAULT_API_URL) + parser.add_argument("--run-url", default="") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("GITHUB_TOKEN is required", file=sys.stderr) + return 1 + + execution_log = json.loads(args.execution_file.read_text(encoding="utf-8")) + review_text = extract_latest_assistant_text(execution_log) + body = build_comment_body(review_text, args.run_url or None) + + try: + upsert_issue_comment( + api_url=args.api_url.rstrip("/"), + repo=args.repo, + issue_number=args.issue_number, + token=token, + body=body, + ) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + print(f"GitHub API request failed: {exc.code} {detail}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_monthly_report_bundle.py b/scripts/run_monthly_report_bundle.py index 6d7152d..b310165 100644 --- a/scripts/run_monthly_report_bundle.py +++ b/scripts/run_monthly_report_bundle.py @@ -127,6 +127,20 @@ def render_ai_review_input(bundle: dict[str, Any], release_status_md: str, month - Pool size: {bundle['pool_size']} - Symbols: {", ".join(bundle['symbols']) or 'n/a'} +## Review intent + +- This is an upstream selector review for CryptoLeaderRotation, not a downstream execution report. +- The main question is whether the current monthly 5-symbol pool still looks like a sound production selector output, and what additional research evidence is still missing. +- Shadow / challenger coverage should be used for strategy-optimization judgment when available. If missing, treat optimization evidence as incomplete rather than forcing a strong conclusion. +- Downstream BinancePlatform consumes this pool monthly and then applies its own execution logic on top. + +## Strategy review questions + +1. Does the official pool look internally consistent with the ranking preview and release metadata? +2. Are score spread and selected symbols reasonable for a Binance Spot mainstream leader selector? +3. Is shadow / challenger evidence present, and if not, what is the highest-value missing comparison? +4. What are the most useful next low-risk research directions before changing production selector logic? + ## Release Status Summary {release_status_md} diff --git a/tests/test_monthly_publish_workflow_config.py b/tests/test_monthly_publish_workflow_config.py index ddf789c..2a740b1 100644 --- a/tests/test_monthly_publish_workflow_config.py +++ b/tests/test_monthly_publish_workflow_config.py @@ -6,12 +6,14 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] WORKFLOW_PATH = PROJECT_ROOT / ".github" / "workflows" / "monthly_publish.yml" +AI_REVIEW_WORKFLOW_PATH = PROJECT_ROOT / ".github" / "workflows" / "ai_review.yml" class MonthlyPublishWorkflowConfigTests(unittest.TestCase): def test_publish_targets_use_vars_only(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertIn("actions: write", workflow) self.assertIn("GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}", workflow) self.assertIn("GCS_BUCKET: ${{ vars.GCS_BUCKET }}", workflow) self.assertIn("credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}", workflow) @@ -24,8 +26,29 @@ def test_monthly_review_issue_creation_does_not_require_gh_cli(self) -> None: self.assertNotIn("gh label create", workflow) self.assertNotIn("gh issue create", workflow) + self.assertNotIn("gh workflow run", workflow) self.assertIn("https://api.github.com/repos/{repository}", workflow) self.assertIn('GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}', workflow) + self.assertIn("issue_number=", workflow) + self.assertIn("/actions/workflows/ai_review.yml/dispatches", workflow) + + def test_ai_review_workflow_supports_dispatch_and_comment_posting(self) -> None: + workflow = AI_REVIEW_WORKFLOW_PATH.read_text(encoding="utf-8") + + self.assertIn("workflow_dispatch:", workflow) + self.assertIn("issue_number:", workflow) + self.assertIn("id-token: write", workflow) + self.assertIn("Load review issue context", workflow) + self.assertIn("api.github.com/repos/{repo}/issues/{issue_number}", workflow) + self.assertIn("github_token: ${{ secrets.GITHUB_TOKEN }}", workflow) + self.assertIn("This is a strategy-optimization review, not only a release QA check.", workflow) + self.assertIn("If shadow or challenger tracks are missing, say evidence is incomplete", workflow) + self.assertIn("Strategy Optimization Directions", workflow) + self.assertIn("post_monthly_ai_review_comment.py", workflow) + self.assertIn("steps.claude_review.outputs.execution_file", workflow) + self.assertNotIn("model:", workflow) + self.assertNotIn("allowed_tools:", workflow) + self.assertNotIn("custom_instructions:", workflow) if __name__ == "__main__": diff --git a/tests/test_monthly_report_bundle.py b/tests/test_monthly_report_bundle.py index e1c5248..c002831 100644 --- a/tests/test_monthly_report_bundle.py +++ b/tests/test_monthly_report_bundle.py @@ -57,6 +57,10 @@ def test_write_bundle_copies_files_and_writes_manifest(self) -> None: self.assertIn("monthly_telegram.txt", manifest["artifact_files"]) self.assertTrue((bundle_dir / "ai_review_input.md").exists()) self.assertTrue((bundle_dir / "job_summary.md").exists()) + ai_review_input = (bundle_dir / "ai_review_input.md").read_text(encoding="utf-8") + self.assertIn("upstream selector review", ai_review_input) + self.assertIn("Shadow / challenger coverage", ai_review_input) + self.assertIn("Strategy review questions", ai_review_input) if __name__ == "__main__": diff --git a/tests/test_post_monthly_ai_review_comment.py b/tests/test_post_monthly_ai_review_comment.py new file mode 100644 index 0000000..eae16aa --- /dev/null +++ b/tests/test_post_monthly_ai_review_comment.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import unittest + +from scripts.post_monthly_ai_review_comment import ( + COMMENT_MARKER, + build_comment_body, + extract_latest_assistant_text, +) + + +class PostMonthlyAiReviewCommentTests(unittest.TestCase): + def test_extract_latest_assistant_text_returns_last_text_reply(self) -> None: + execution_log = [ + {"type": "system", "subtype": "init"}, + { + "type": "assistant", + "message": { + "content": [ + {"type": "text", "text": "Working on it."}, + ] + }, + }, + { + "type": "assistant", + "message": { + "content": [ + {"type": "tool_use", "name": "Read", "input": {"file_path": "x"}}, + {"type": "text", "text": "## English\nReview\n\n## 中文\n评审"}, + ] + }, + }, + {"type": "result", "subtype": "success"}, + ] + + review_text = extract_latest_assistant_text(execution_log) + + self.assertEqual(review_text, "## English\nReview\n\n## 中文\n评审") + + def test_build_comment_body_includes_marker_and_run_link(self) -> None: + body = build_comment_body("Review content", "https://github.com/example/repo/actions/runs/1") + + self.assertIn(COMMENT_MARKER, body) + self.assertIn("## Claude Monthly Strategy Review", body) + self.assertIn("Review content", body) + self.assertIn("actions/runs/1", body) + + +if __name__ == "__main__": + unittest.main()