From f9c0e54d6a9058016dce30b194f1d709a967d5c6 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 2 Apr 2026 02:03:13 +0800 Subject: [PATCH] feat: add gpt secondary monthly review --- .github/workflows/ai_review.yml | 51 +++- scripts/post_monthly_ai_review_comment.py | 42 ++-- scripts/render_monthly_ai_review.py | 120 ++++++++++ scripts/run_openai_secondary_review.py | 223 ++++++++++++++++++ tests/test_monthly_publish_workflow_config.py | 7 +- tests/test_post_monthly_ai_review_comment.py | 2 +- tests/test_render_monthly_ai_review.py | 60 +++++ tests/test_run_openai_secondary_review.py | 50 ++++ 8 files changed, 529 insertions(+), 26 deletions(-) create mode 100644 scripts/render_monthly_ai_review.py create mode 100644 scripts/run_openai_secondary_review.py create mode 100644 tests/test_render_monthly_ai_review.py create mode 100644 tests/test_run_openai_secondary_review.py diff --git a/.github/workflows/ai_review.yml b/.github/workflows/ai_review.yml index 0866866..dab16ff 100644 --- a/.github/workflows/ai_review.yml +++ b/.github/workflows/ai_review.yml @@ -28,10 +28,12 @@ jobs: - name: Load review issue context id: issue_context run: | + mkdir -p data/output/ai_review python3 - <<'PY' import json import os import urllib.request + from pathlib import Path repo = os.environ["GITHUB_REPOSITORY"] issue_number = os.environ["ISSUE_NUMBER"] @@ -49,6 +51,16 @@ jobs: with urllib.request.urlopen(request) as response: issue = json.load(response) + issue_context = { + "number": issue["number"], + "title": issue["title"], + "body": issue["body"], + } + Path("data/output/ai_review/issue_context.json").write_text( + json.dumps(issue_context, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: print("issue_title< str: - for turn in reversed(execution_log): - if turn.get("type") != "assistant": - continue - - content_items = turn.get("message", {}).get("content", []) - text_parts = [ - item.get("text", "").strip() - for item in content_items - if item.get("type") == "text" and item.get("text", "").strip() - ] - if text_parts: - return "\n\n".join(text_parts).strip() - - raise ValueError("No assistant review text found in Claude execution log") - - -def build_comment_body(review_text: str, run_url: str | None = None) -> str: - body = f"{COMMENT_MARKER}\n## Claude Monthly Strategy Review\n\n{review_text.strip()}" +def build_comment_body(review_markdown: str, run_url: str | None = None) -> str: + body = f"{COMMENT_MARKER}\n## AI Monthly Review\n\n{review_markdown.strip()}" if run_url: body += f"\n\n---\n_Generated by AI Monthly Review workflow: {run_url}_" return body @@ -101,7 +89,8 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--repo", required=True, help="owner/repo") parser.add_argument("--issue-number", required=True, type=int) - parser.add_argument("--execution-file", required=True, type=Path) + parser.add_argument("--execution-file", type=Path) + parser.add_argument("--review-file", type=Path) parser.add_argument("--api-url", default=DEFAULT_API_URL) parser.add_argument("--run-url", default="") return parser.parse_args() @@ -114,9 +103,16 @@ def main() -> int: print("GITHUB_TOKEN is required", file=sys.stderr) return 1 - execution_log = json.loads(args.execution_file.read_text(encoding="utf-8")) - review_text = extract_latest_assistant_text(execution_log) - body = build_comment_body(review_text, args.run_url or None) + if args.review_file is not None: + review_markdown = args.review_file.read_text(encoding="utf-8") + elif args.execution_file is not None: + execution_log = json.loads(args.execution_file.read_text(encoding="utf-8")) + review_markdown = extract_latest_assistant_text(execution_log) + else: + print("Either --review-file or --execution-file is required", file=sys.stderr) + return 1 + + body = build_comment_body(review_markdown, args.run_url or None) try: upsert_issue_comment( diff --git a/scripts/render_monthly_ai_review.py b/scripts/render_monthly_ai_review.py new file mode 100644 index 0000000..e010782 --- /dev/null +++ b/scripts/render_monthly_ai_review.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def extract_latest_assistant_text(execution_log: list[dict[str, Any]]) -> str: + for turn in reversed(execution_log): + if turn.get("type") != "assistant": + continue + + content_items = turn.get("message", {}).get("content", []) + text_parts = [ + item.get("text", "").strip() + for item in content_items + if item.get("type") == "text" and item.get("text", "").strip() + ] + if text_parts: + return "\n\n".join(text_parts).strip() + + raise ValueError("No assistant review text found in execution log") + + +def load_primary_review_markdown(*, execution_file: Path | None, primary_review_file: Path | None) -> str: + if primary_review_file is not None: + return primary_review_file.read_text(encoding="utf-8").strip() + if execution_file is not None: + execution_log = json.loads(execution_file.read_text(encoding="utf-8")) + return extract_latest_assistant_text(execution_log) + raise ValueError("Either execution_file or primary_review_file is required") + + +def render_secondary_review_markdown(payload: dict[str, Any]) -> str: + lines: list[str] = [ + f"## Secondary Review ({payload.get('provider_display_name', 'GPT')})", + "", + f"- Verdict: `{payload['verdict']}`", + f"- Risk Level: `{payload['risk_level']}`", + f"- Production Recommendation: `{payload['production_recommendation']}`", + f"- Summary: {payload['summary']}", + ] + + findings = [item.strip() for item in payload.get("key_findings", []) if str(item).strip()] + if findings: + lines.extend(["", "### Key Findings"]) + lines.extend(f"- {item}" for item in findings) + + actions = payload.get("recommended_actions", []) + if actions: + lines.extend(["", "### Recommended Actions"]) + for action in actions: + flags: list[str] = [] + if action.get("auto_pr_safe"): + flags.append("auto-pr-safe") + if action.get("experiment_only"): + flags.append("experiment-only") + flag_text = f" [{', '.join(flags)}]" if flags else "" + lines.append( + "- " + f"{action['title']} " + f"({action['owner_repo']}, risk={action['risk_level']}){flag_text}: {action['summary']}" + ) + + follow_up_checks = [item.strip() for item in payload.get("follow_up_checks", []) if str(item).strip()] + if follow_up_checks: + lines.extend(["", "### Follow-up Checks"]) + lines.extend(f"- {item}" for item in follow_up_checks) + + return "\n".join(lines).strip() + + +def build_full_review_markdown( + primary_review_text: str, + *, + primary_title: str, + secondary_review_payload: dict[str, Any] | None = None, +) -> str: + lines = [f"## {primary_title}", "", primary_review_text.strip()] + if secondary_review_payload is not None: + lines.extend(["", "---", "", render_secondary_review_markdown(secondary_review_payload)]) + return "\n".join(lines).strip() + "\n" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Render markdown for the monthly AI review from primary and optional secondary review outputs.", + ) + parser.add_argument("--output-file", required=True, type=Path) + parser.add_argument("--execution-file", type=Path) + parser.add_argument("--primary-review-file", type=Path) + parser.add_argument("--secondary-review-file", type=Path) + parser.add_argument("--primary-title", default="Claude Primary Review") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + primary_review_text = load_primary_review_markdown( + execution_file=args.execution_file, + primary_review_file=args.primary_review_file, + ) + secondary_review_payload = None + if args.secondary_review_file is not None: + secondary_review_payload = json.loads(args.secondary_review_file.read_text(encoding="utf-8")) + + markdown = build_full_review_markdown( + primary_review_text, + primary_title=args.primary_title, + secondary_review_payload=secondary_review_payload, + ) + args.output_file.parent.mkdir(parents=True, exist_ok=True) + args.output_file.write_text(markdown, encoding="utf-8") + print(f"review_markdown={args.output_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_openai_secondary_review.py b/scripts/run_openai_secondary_review.py new file mode 100644 index 0000000..85d7010 --- /dev/null +++ b/scripts/run_openai_secondary_review.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + + +OPENAI_API_URL = "https://api.openai.com/v1/chat/completions" +SUPPORTED_REVIEW_KINDS = {"upstream_selector", "execution_runtime"} + +SECONDARY_REVIEW_SCHEMA: dict[str, Any] = { + "type": "object", + "additionalProperties": False, + "properties": { + "review_kind": {"type": "string"}, + "provider": {"type": "string"}, + "provider_display_name": {"type": "string"}, + "model": {"type": "string"}, + "verdict": {"type": "string", "enum": ["agree", "partial_agree", "disagree"]}, + "risk_level": {"type": "string", "enum": ["low", "medium", "high"]}, + "production_recommendation": { + "type": "string", + "enum": ["keep_production_as_is", "research_only", "needs_attention"], + }, + "summary": {"type": "string"}, + "key_findings": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "maxItems": 5, + }, + "recommended_actions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "properties": { + "title": {"type": "string"}, + "owner_repo": { + "type": "string", + "enum": ["CryptoLeaderRotation", "CryptoStrategies", "BinancePlatform"], + }, + "risk_level": {"type": "string", "enum": ["low", "medium", "high"]}, + "auto_pr_safe": {"type": "boolean"}, + "experiment_only": {"type": "boolean"}, + "summary": {"type": "string"}, + }, + "required": [ + "title", + "owner_repo", + "risk_level", + "auto_pr_safe", + "experiment_only", + "summary", + ], + }, + "maxItems": 5, + }, + "follow_up_checks": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 5, + }, + }, + "required": [ + "review_kind", + "provider", + "provider_display_name", + "model", + "verdict", + "risk_level", + "production_recommendation", + "summary", + "key_findings", + "recommended_actions", + "follow_up_checks", + ], +} + + +def build_system_prompt(review_kind: str) -> str: + if review_kind == "upstream_selector": + return ( + "You are the independent secondary reviewer for CryptoLeaderRotation, an upstream selector " + "repository that publishes a monthly 5-symbol Binance Spot leader pool. Review the issue body " + "and the Claude primary review, then return only valid JSON matching the provided schema. " + "Do not simply echo Claude. Re-check whether release consistency, selector quality, " + "shadow/challenger evidence, and downstream BinancePlatform impact actually support the same conclusion. " + "Use recommended_actions for concrete next steps, and only mark auto_pr_safe=true for low-risk " + "changes like workflow, telemetry, report wording, tests, or challenger/shadow configuration." + ) + if review_kind == "execution_runtime": + return ( + "You are the independent secondary reviewer for BinancePlatform, a downstream Binance Spot execution " + "engine. Review the issue body and the Claude primary review, then return only valid JSON matching " + "the provided schema. Do not simply echo Claude. Re-check whether execution health, gating/no-trade " + "reasons, degraded mode, circuit breaker behavior, and cash-flow context support the same conclusion. " + "Use recommended_actions for concrete next steps, and only mark auto_pr_safe=true for low-risk " + "changes like workflow, telemetry, report wording, tests, or diagnostics." + ) + raise ValueError(f"Unsupported review kind: {review_kind}") + + +def build_user_prompt(issue_title: str, issue_body: str, primary_review_text: str) -> str: + return ( + "Independently review the monthly report below, then compare it against the Claude primary review. " + "If Claude looks too strong or too weak, say so in verdict/summary/findings.\n\n" + f"## Issue Title\n{issue_title.strip()}\n\n" + f"## Issue Body\n{issue_body.strip()}\n\n" + f"## Claude Primary Review\n{primary_review_text.strip()}\n" + ) + + +def build_request_payload( + *, + model: str, + review_kind: str, + issue_title: str, + issue_body: str, + primary_review_text: str, +) -> dict[str, Any]: + return { + "model": model, + "messages": [ + {"role": "system", "content": build_system_prompt(review_kind)}, + { + "role": "user", + "content": build_user_prompt(issue_title, issue_body, primary_review_text), + }, + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "secondary_monthly_review", + "strict": True, + "schema": SECONDARY_REVIEW_SCHEMA, + }, + }, + } + + +def extract_completion_content(response_payload: dict[str, Any]) -> str: + choices = response_payload.get("choices") or [] + if not choices: + raise ValueError("OpenAI response did not include choices") + + message = choices[0].get("message") or {} + content = message.get("content") + if not isinstance(content, str) or not content.strip(): + raise ValueError("OpenAI response did not include text content") + return content.strip() + + +def call_openai(payload: dict[str, Any], api_key: str) -> dict[str, Any]: + request = urllib.request.Request( + OPENAI_API_URL, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + with urllib.request.urlopen(request) as response: + charset = response.headers.get_content_charset("utf-8") + return json.loads(response.read().decode(charset)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run an OpenAI secondary review for the monthly AI review workflow.", + ) + parser.add_argument("--review-kind", required=True, choices=sorted(SUPPORTED_REVIEW_KINDS)) + parser.add_argument("--issue-context-file", required=True, type=Path) + parser.add_argument("--primary-review-file", required=True, type=Path) + parser.add_argument("--output-file", required=True, type=Path) + parser.add_argument("--model", default=os.environ.get("OPENAI_MODEL", "gpt-5.4-mini")) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + print("OPENAI_API_KEY is required", file=sys.stderr) + return 1 + + issue_context = json.loads(args.issue_context_file.read_text(encoding="utf-8")) + primary_review_text = args.primary_review_file.read_text(encoding="utf-8") + payload = build_request_payload( + model=args.model, + review_kind=args.review_kind, + issue_title=str(issue_context.get("title", "")), + issue_body=str(issue_context.get("body", "")), + primary_review_text=primary_review_text, + ) + + try: + response_payload = call_openai(payload, api_key) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + print(f"OpenAI API request failed: {exc.code} {detail}", file=sys.stderr) + return 1 + + review_payload = json.loads(extract_completion_content(response_payload)) + review_payload["review_kind"] = args.review_kind + review_payload["provider"] = "openai" + review_payload["provider_display_name"] = "GPT Secondary Review" + review_payload["model"] = args.model + + args.output_file.parent.mkdir(parents=True, exist_ok=True) + args.output_file.write_text(json.dumps(review_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(f"secondary_review={args.output_file}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_monthly_publish_workflow_config.py b/tests/test_monthly_publish_workflow_config.py index dedd395..b3bbaf5 100644 --- a/tests/test_monthly_publish_workflow_config.py +++ b/tests/test_monthly_publish_workflow_config.py @@ -52,8 +52,13 @@ def test_ai_review_workflow_supports_dispatch_and_comment_posting(self) -> None: self.assertIn("If shadow or challenger tracks are missing, say evidence is incomplete", workflow) self.assertIn("Strategy Optimization Directions", workflow) self.assertIn("post_monthly_ai_review_comment.py", workflow) + self.assertIn("render_monthly_ai_review.py", workflow) + self.assertIn("run_openai_secondary_review.py", workflow) self.assertIn("steps.claude_review.outputs.execution_file", workflow) - self.assertNotIn("model:", workflow) + self.assertIn("OPENAI_API_KEY", workflow) + self.assertIn("OPENAI_SECONDARY_MODEL", workflow) + self.assertIn("secondary_review.json", workflow) + self.assertIn("actions/upload-artifact@v7", workflow) self.assertNotIn("allowed_tools:", workflow) self.assertNotIn("custom_instructions:", workflow) diff --git a/tests/test_post_monthly_ai_review_comment.py b/tests/test_post_monthly_ai_review_comment.py index eae16aa..538c3b7 100644 --- a/tests/test_post_monthly_ai_review_comment.py +++ b/tests/test_post_monthly_ai_review_comment.py @@ -41,7 +41,7 @@ def test_build_comment_body_includes_marker_and_run_link(self) -> None: body = build_comment_body("Review content", "https://github.com/example/repo/actions/runs/1") self.assertIn(COMMENT_MARKER, body) - self.assertIn("## Claude Monthly Strategy Review", body) + self.assertIn("## AI Monthly Review", body) self.assertIn("Review content", body) self.assertIn("actions/runs/1", body) diff --git a/tests/test_render_monthly_ai_review.py b/tests/test_render_monthly_ai_review.py new file mode 100644 index 0000000..292cb2e --- /dev/null +++ b/tests/test_render_monthly_ai_review.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import unittest + +from scripts.render_monthly_ai_review import build_full_review_markdown, render_secondary_review_markdown + + +class RenderMonthlyAiReviewTests(unittest.TestCase): + def test_render_secondary_review_markdown_includes_actions_and_flags(self) -> None: + payload = { + "provider_display_name": "GPT Secondary Review", + "verdict": "partial_agree", + "risk_level": "medium", + "production_recommendation": "research_only", + "summary": "Evidence is directionally fine but still incomplete.", + "key_findings": ["Shadow coverage is still thin."], + "recommended_actions": [ + { + "title": "Add another challenger track", + "owner_repo": "CryptoLeaderRotation", + "risk_level": "low", + "auto_pr_safe": True, + "experiment_only": True, + "summary": "Improve monthly evidence before changing production.", + } + ], + "follow_up_checks": ["Compare challenger turnover before next promotion."], + } + + markdown = render_secondary_review_markdown(payload) + + self.assertIn("## Secondary Review (GPT Secondary Review)", markdown) + self.assertIn("`partial_agree`", markdown) + self.assertIn("auto-pr-safe", markdown) + self.assertIn("experiment-only", markdown) + self.assertIn("Compare challenger turnover", markdown) + + def test_build_full_review_markdown_includes_primary_and_secondary_sections(self) -> None: + markdown = build_full_review_markdown( + "## English\nPrimary review", + primary_title="Claude Primary Review", + secondary_review_payload={ + "provider_display_name": "GPT Secondary Review", + "verdict": "agree", + "risk_level": "low", + "production_recommendation": "keep_production_as_is", + "summary": "Looks consistent.", + "key_findings": ["No blocking issue found."], + "recommended_actions": [], + "follow_up_checks": [], + }, + ) + + self.assertIn("## Claude Primary Review", markdown) + self.assertIn("## Secondary Review (GPT Secondary Review)", markdown) + self.assertIn("## English", markdown) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_openai_secondary_review.py b/tests/test_run_openai_secondary_review.py new file mode 100644 index 0000000..c25c0dc --- /dev/null +++ b/tests/test_run_openai_secondary_review.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +import unittest + +from scripts.run_openai_secondary_review import ( + build_request_payload, + build_system_prompt, + extract_completion_content, +) + + +class RunOpenAiSecondaryReviewTests(unittest.TestCase): + def test_build_system_prompt_for_upstream_selector_mentions_shadow_and_binanceplatform(self) -> None: + prompt = build_system_prompt("upstream_selector") + + self.assertIn("CryptoLeaderRotation", prompt) + self.assertIn("shadow/challenger", prompt) + self.assertIn("BinancePlatform", prompt) + + def test_build_request_payload_uses_structured_json_schema(self) -> None: + payload = build_request_payload( + model="gpt-5.4-mini", + review_kind="upstream_selector", + issue_title="Monthly Review", + issue_body="body", + primary_review_text="primary", + ) + + self.assertEqual(payload["model"], "gpt-5.4-mini") + self.assertEqual(payload["response_format"]["type"], "json_schema") + self.assertTrue(payload["response_format"]["json_schema"]["strict"]) + self.assertIn("messages", payload) + + def test_extract_completion_content_reads_first_choice_message(self) -> None: + response_payload = { + "choices": [ + { + "message": { + "content": json.dumps({"summary": "ok"}), + } + } + ] + } + + self.assertEqual(extract_completion_content(response_payload), '{"summary": "ok"}') + + +if __name__ == "__main__": + unittest.main()