From 9fa0035756257c264460cb3ce71864d605edc28b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 09:49:47 +0000 Subject: [PATCH 1/6] Initial plan From e135262093eb94f268e332a639fcb28048c9bdce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 09:56:21 +0000 Subject: [PATCH 2/6] fix: fall back when copilot cli generation fails Agent-Logs-Url: https://github.com/DevSecNinja/wazzup/sessions/f3f2c1a6-a8eb-4a20-92ba-f562e9163b2e Co-authored-by: DevSecNinja <14926452+DevSecNinja@users.noreply.github.com> --- src/wazzup/ai.py | 121 ++++++++++++++++++++++++++--------------------- tests/test_ai.py | 64 +++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 54 deletions(-) diff --git a/src/wazzup/ai.py b/src/wazzup/ai.py index adf4283..9105512 100644 --- a/src/wazzup/ai.py +++ b/src/wazzup/ai.py @@ -289,23 +289,36 @@ def curate_items(self, request: CurationRequest) -> CurationResponse: "--no-ask-user", ] ) - result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) - if result.returncode != 0: - details = [] - if result.stdout.strip(): - details.append(f"stdout: {result.stdout.strip()}") - if result.stderr.strip(): - details.append(f"stderr: {result.stderr.strip()}") - detail_text = "\n" + "\n".join(details) if details else "" - raise RuntimeError( - f"Copilot CLI curation failed with exit code {result.returncode}. " - "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, " - "or use AI_PROVIDER=fake." - f"{detail_text}" + try: + result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) + if result.returncode != 0: + details = [] + if result.stdout.strip(): + details.append(f"stdout: {result.stdout.strip()}") + if result.stderr.strip(): + details.append(f"stderr: {result.stderr.strip()}") + detail_text = "\n" + "\n".join(details) if details else "" + raise RuntimeError( + f"Copilot CLI curation failed with exit code {result.returncode}. " + "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, " + "or use AI_PROVIDER=fake." + f"{detail_text}" + ) + if not output_path.exists(): + raise RuntimeError("Copilot CLI did not write curation-output.json") + payload = json.loads(output_path.read_text(encoding="utf-8")) + except (RuntimeError, json.JSONDecodeError) as exc: + fallback = FakeCurationProvider().curate_items(request) + return CurationResponse( + selected_ids=fallback.selected_ids, + provider={ + **fallback.provider, + "type": "copilot-cli-fallback", + "fallbackFrom": self.name, + "fallbackReason": str(exc), + "validated": True, + }, ) - if not output_path.exists(): - raise RuntimeError("Copilot CLI did not write curation-output.json") - payload = json.loads(output_path.read_text(encoding="utf-8")) selected_ids = payload.get("selectedIds") if not isinstance(selected_ids, list) or not all(isinstance(item_id, str) for item_id in selected_ids): fallback = FakeCurationProvider() @@ -389,45 +402,45 @@ def generate_structured_summary(self, request: SummaryRequest) -> SummaryRespons "--no-ask-user", ] ) - result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) - if result.returncode != 0: - details = [] - if result.stdout.strip(): - details.append(f"stdout: {result.stdout.strip()}") - if result.stderr.strip(): - details.append(f"stderr: {result.stderr.strip()}") - detail_text = "\n" + "\n".join(details) if details else "" - raise RuntimeError( - f"Copilot CLI failed with exit code {result.returncode}. " - "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, " - "or use AI_PROVIDER=fake." - f"{detail_text}" - ) - if not output_path.exists(): - raise RuntimeError("Copilot CLI did not write summary.json") - payload = json.loads(output_path.read_text(encoding="utf-8")) - provider = { - "type": self.name, - "model": payload.get("model", self.model or "copilot-cli"), - "agent": self.agent or None, - "promptVersion": "summary-v1", - "validated": True, - } - try: - return response_from_payload(payload, provider=provider) - except ValueError as exc: - fallback = FakeSummaryProvider().generate_structured_summary(request) - return SummaryResponse( - headline=fallback.headline, - sections=fallback.sections, - provider={ - **fallback.provider, - "type": "copilot-cli-fallback", - "fallbackFrom": self.name, - "fallbackReason": str(exc), + try: + result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) + if result.returncode != 0: + details = [] + if result.stdout.strip(): + details.append(f"stdout: {result.stdout.strip()}") + if result.stderr.strip(): + details.append(f"stderr: {result.stderr.strip()}") + detail_text = "\n" + "\n".join(details) if details else "" + raise RuntimeError( + f"Copilot CLI failed with exit code {result.returncode}. " + "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, " + "or use AI_PROVIDER=fake." + f"{detail_text}" + ) + if not output_path.exists(): + raise RuntimeError("Copilot CLI did not write summary.json") + payload = json.loads(output_path.read_text(encoding="utf-8")) + provider = { + "type": self.name, + "model": payload.get("model", self.model or "copilot-cli"), + "agent": self.agent or None, + "promptVersion": "summary-v1", "validated": True, - }, - ) + } + return response_from_payload(payload, provider=provider) + except (RuntimeError, ValueError, json.JSONDecodeError) as exc: + fallback = FakeSummaryProvider().generate_structured_summary(request) + return SummaryResponse( + headline=fallback.headline, + sections=fallback.sections, + provider={ + **fallback.provider, + "type": "copilot-cli-fallback", + "fallbackFrom": self.name, + "fallbackReason": str(exc), + "validated": True, + }, + ) class CopilotCliTransparencyReportProvider: diff --git a/tests/test_ai.py b/tests/test_ai.py index b43f3d4..9449d1e 100644 --- a/tests/test_ai.py +++ b/tests/test_ai.py @@ -408,6 +408,38 @@ def fake_run(_command, capture_output, cwd, env, text): # type: ignore[no-untyp self.assertIn("fallbackReason", response.provider) self.assertTrue(response.sections[0]["bullets"]) + @patch("wazzup.ai.subprocess.run") + @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") + def test_copilot_summary_falls_back_on_runtime_failure(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] + previous_token = os.environ.get("COPILOT_GITHUB_TOKEN") + os.environ["COPILOT_GITHUB_TOKEN"] = "test-token" + source = load_sources("config/sources.yml")[0] + item = parse_feed(source, Path("tests/fixtures/microsoft-security-blog.xml").read_bytes())[0] + scored = score_items([item], [source], load_app_config("config/interests.yml"), datetime(2026, 5, 6, tzinfo=UTC)) + run_mock.return_value = Mock(returncode=1, stdout="failed", stderr="upstream error") + try: + response = CopilotCliSummaryProvider().generate_structured_summary( + SummaryRequest( + kind="hourly", + window_start="2026-05-06T00:00:00Z", + window_end="2026-05-06T21:00:00Z", + generated_at="2026-05-06T21:00:00Z", + timezone="Europe/Amsterdam", + summary_language="en", + items=scored, + ) + ) + finally: + if previous_token is None: + os.environ.pop("COPILOT_GITHUB_TOKEN", None) + else: + os.environ["COPILOT_GITHUB_TOKEN"] = previous_token + + self.assertEqual("copilot-cli-fallback", response.provider["type"]) + self.assertEqual("copilot-cli", response.provider["fallbackFrom"]) + self.assertIn("exit code 1", response.provider["fallbackReason"]) + self.assertTrue(response.sections[0]["bullets"]) + class AiCurationProviderTests(unittest.TestCase): def test_curation_provider_defaults_to_fake(self) -> None: @@ -605,6 +637,38 @@ def fake_run(_command, capture_output, cwd, env, text): # type: ignore[no-untyp self.assertIn("fallbackReason", response.provider) self.assertTrue(response.selected_ids) + @patch("wazzup.ai.subprocess.run") + @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") + def test_copilot_cli_curation_falls_back_on_runtime_failure(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] + previous_token = os.environ.get("COPILOT_GITHUB_TOKEN") + os.environ["COPILOT_GITHUB_TOKEN"] = "test-token" + source = load_sources("config/sources.yml")[0] + item = parse_feed(source, Path("tests/fixtures/microsoft-security-blog.xml").read_bytes())[0] + scored = score_items([item], [source], load_app_config("config/interests.yml"), datetime(2026, 5, 6, tzinfo=UTC)) + run_mock.return_value = Mock(returncode=1, stdout="failed", stderr="upstream error") + try: + response = CopilotCliCurationProvider().curate_items( + CurationRequest( + kind="hourly", + window_start="2026-05-06T20:00:00Z", + window_end="2026-05-06T21:00:00Z", + generated_at="2026-05-06T21:00:00Z", + timezone="Europe/Amsterdam", + items=scored, + max_items=12, + ) + ) + finally: + if previous_token is None: + os.environ.pop("COPILOT_GITHUB_TOKEN", None) + else: + os.environ["COPILOT_GITHUB_TOKEN"] = previous_token + + self.assertEqual("copilot-cli-fallback", response.provider["type"]) + self.assertEqual("copilot-cli", response.provider["fallbackFrom"]) + self.assertIn("exit code 1", response.provider["fallbackReason"]) + self.assertTrue(response.selected_ids) + class AiTransparencyReportProviderTests(unittest.TestCase): def test_transparency_provider_defaults_to_fake(self) -> None: From 102b659fa6fb7fc06398c280ea418fde4b22c1b7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 14:25:14 +0000 Subject: [PATCH 3/6] fix: retry copilot cli before failing Agent-Logs-Url: https://github.com/DevSecNinja/wazzup/sessions/8cf0349d-d3f8-4484-9586-ce6d58b41e81 Co-authored-by: DevSecNinja <14926452+DevSecNinja@users.noreply.github.com> --- src/wazzup/ai.py | 74 +++++++++++++++++++----------------------------- tests/test_ai.py | 57 ++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 75 deletions(-) diff --git a/src/wazzup/ai.py b/src/wazzup/ai.py index 9105512..67df6f1 100644 --- a/src/wazzup/ai.py +++ b/src/wazzup/ai.py @@ -96,6 +96,7 @@ def generate_transparency_report(self, request: TransparencyReportRequest) -> Tr DEFAULT_COPILOT_AGENT = "wazzup-writer" DEFAULT_COPILOT_CURATOR_AGENT = "wazzup-curator" DEFAULT_COPILOT_TRANSPARENCY_AGENT = "wazzup-transparency-reporter" +COPILOT_CLI_MAX_ATTEMPTS = 2 MAX_SUMMARY_HEADLINE_LENGTH = 80 MAX_SUMMARY_TITLE_LENGTH = 96 MAX_SUMMARY_DESCRIPTION_LENGTH = 220 @@ -231,6 +232,28 @@ def generate_transparency_report(self, request: TransparencyReportRequest) -> Tr ) +def run_copilot_cli(command: list[str], run_env: dict[str, str], failure_label: str) -> subprocess.CompletedProcess[str]: + result: subprocess.CompletedProcess[str] | None = None + for _attempt in range(COPILOT_CLI_MAX_ATTEMPTS): + result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) + if result.returncode == 0: + return result + if result is None: + raise RuntimeError(f"{failure_label} was not attempted") + + details = [] + if result.stdout.strip(): + details.append(f"stdout: {result.stdout.strip()}") + if result.stderr.strip(): + details.append(f"stderr: {result.stderr.strip()}") + detail_text = "\n" + "\n".join(details) if details else "" + raise RuntimeError( + f"{failure_label} failed with exit code {result.returncode} after {COPILOT_CLI_MAX_ATTEMPTS} attempts. " + "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, or use AI_PROVIDER=fake." + f"{detail_text}" + ) + + class CopilotCliCurationProvider: name = "copilot-cli" @@ -289,36 +312,10 @@ def curate_items(self, request: CurationRequest) -> CurationResponse: "--no-ask-user", ] ) - try: - result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) - if result.returncode != 0: - details = [] - if result.stdout.strip(): - details.append(f"stdout: {result.stdout.strip()}") - if result.stderr.strip(): - details.append(f"stderr: {result.stderr.strip()}") - detail_text = "\n" + "\n".join(details) if details else "" - raise RuntimeError( - f"Copilot CLI curation failed with exit code {result.returncode}. " - "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, " - "or use AI_PROVIDER=fake." - f"{detail_text}" - ) - if not output_path.exists(): - raise RuntimeError("Copilot CLI did not write curation-output.json") - payload = json.loads(output_path.read_text(encoding="utf-8")) - except (RuntimeError, json.JSONDecodeError) as exc: - fallback = FakeCurationProvider().curate_items(request) - return CurationResponse( - selected_ids=fallback.selected_ids, - provider={ - **fallback.provider, - "type": "copilot-cli-fallback", - "fallbackFrom": self.name, - "fallbackReason": str(exc), - "validated": True, - }, - ) + run_copilot_cli(command, run_env, "Copilot CLI curation") + if not output_path.exists(): + raise RuntimeError("Copilot CLI did not write curation-output.json") + payload = json.loads(output_path.read_text(encoding="utf-8")) selected_ids = payload.get("selectedIds") if not isinstance(selected_ids, list) or not all(isinstance(item_id, str) for item_id in selected_ids): fallback = FakeCurationProvider() @@ -403,20 +400,7 @@ def generate_structured_summary(self, request: SummaryRequest) -> SummaryRespons ] ) try: - result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) - if result.returncode != 0: - details = [] - if result.stdout.strip(): - details.append(f"stdout: {result.stdout.strip()}") - if result.stderr.strip(): - details.append(f"stderr: {result.stderr.strip()}") - detail_text = "\n" + "\n".join(details) if details else "" - raise RuntimeError( - f"Copilot CLI failed with exit code {result.returncode}. " - "Verify COPILOT_GITHUB_TOKEN has Copilot Requests permission, " - "or use AI_PROVIDER=fake." - f"{detail_text}" - ) + run_copilot_cli(command, run_env, "Copilot CLI") if not output_path.exists(): raise RuntimeError("Copilot CLI did not write summary.json") payload = json.loads(output_path.read_text(encoding="utf-8")) @@ -428,7 +412,7 @@ def generate_structured_summary(self, request: SummaryRequest) -> SummaryRespons "validated": True, } return response_from_payload(payload, provider=provider) - except (RuntimeError, ValueError, json.JSONDecodeError) as exc: + except ValueError as exc: fallback = FakeSummaryProvider().generate_structured_summary(request) return SummaryResponse( headline=fallback.headline, diff --git a/tests/test_ai.py b/tests/test_ai.py index 9449d1e..85938af 100644 --- a/tests/test_ai.py +++ b/tests/test_ai.py @@ -11,6 +11,7 @@ CopilotCliCurationProvider, CopilotCliSummaryProvider, CopilotCliTransparencyReportProvider, + COPILOT_CLI_MAX_ATTEMPTS, CurationRequest, DEFAULT_COPILOT_AGENT, DEFAULT_COPILOT_CURATOR_AGENT, @@ -410,7 +411,7 @@ def fake_run(_command, capture_output, cwd, env, text): # type: ignore[no-untyp @patch("wazzup.ai.subprocess.run") @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") - def test_copilot_summary_falls_back_on_runtime_failure(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] + def test_copilot_summary_retries_then_fails_on_runtime_failure(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] previous_token = os.environ.get("COPILOT_GITHUB_TOKEN") os.environ["COPILOT_GITHUB_TOKEN"] = "test-token" source = load_sources("config/sources.yml")[0] @@ -418,27 +419,25 @@ def test_copilot_summary_falls_back_on_runtime_failure(self, _which, run_mock) - scored = score_items([item], [source], load_app_config("config/interests.yml"), datetime(2026, 5, 6, tzinfo=UTC)) run_mock.return_value = Mock(returncode=1, stdout="failed", stderr="upstream error") try: - response = CopilotCliSummaryProvider().generate_structured_summary( - SummaryRequest( - kind="hourly", - window_start="2026-05-06T00:00:00Z", - window_end="2026-05-06T21:00:00Z", - generated_at="2026-05-06T21:00:00Z", - timezone="Europe/Amsterdam", - summary_language="en", - items=scored, + with self.assertRaisesRegex(RuntimeError, f"after {COPILOT_CLI_MAX_ATTEMPTS} attempts"): + CopilotCliSummaryProvider().generate_structured_summary( + SummaryRequest( + kind="hourly", + window_start="2026-05-06T00:00:00Z", + window_end="2026-05-06T21:00:00Z", + generated_at="2026-05-06T21:00:00Z", + timezone="Europe/Amsterdam", + summary_language="en", + items=scored, + ) ) - ) finally: if previous_token is None: os.environ.pop("COPILOT_GITHUB_TOKEN", None) else: os.environ["COPILOT_GITHUB_TOKEN"] = previous_token - self.assertEqual("copilot-cli-fallback", response.provider["type"]) - self.assertEqual("copilot-cli", response.provider["fallbackFrom"]) - self.assertIn("exit code 1", response.provider["fallbackReason"]) - self.assertTrue(response.sections[0]["bullets"]) + self.assertEqual(COPILOT_CLI_MAX_ATTEMPTS, run_mock.call_count) class AiCurationProviderTests(unittest.TestCase): @@ -639,7 +638,7 @@ def fake_run(_command, capture_output, cwd, env, text): # type: ignore[no-untyp @patch("wazzup.ai.subprocess.run") @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") - def test_copilot_cli_curation_falls_back_on_runtime_failure(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] + def test_copilot_cli_curation_retries_then_fails_on_runtime_failure(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] previous_token = os.environ.get("COPILOT_GITHUB_TOKEN") os.environ["COPILOT_GITHUB_TOKEN"] = "test-token" source = load_sources("config/sources.yml")[0] @@ -647,27 +646,25 @@ def test_copilot_cli_curation_falls_back_on_runtime_failure(self, _which, run_mo scored = score_items([item], [source], load_app_config("config/interests.yml"), datetime(2026, 5, 6, tzinfo=UTC)) run_mock.return_value = Mock(returncode=1, stdout="failed", stderr="upstream error") try: - response = CopilotCliCurationProvider().curate_items( - CurationRequest( - kind="hourly", - window_start="2026-05-06T20:00:00Z", - window_end="2026-05-06T21:00:00Z", - generated_at="2026-05-06T21:00:00Z", - timezone="Europe/Amsterdam", - items=scored, - max_items=12, + with self.assertRaisesRegex(RuntimeError, f"after {COPILOT_CLI_MAX_ATTEMPTS} attempts"): + CopilotCliCurationProvider().curate_items( + CurationRequest( + kind="hourly", + window_start="2026-05-06T20:00:00Z", + window_end="2026-05-06T21:00:00Z", + generated_at="2026-05-06T21:00:00Z", + timezone="Europe/Amsterdam", + items=scored, + max_items=12, + ) ) - ) finally: if previous_token is None: os.environ.pop("COPILOT_GITHUB_TOKEN", None) else: os.environ["COPILOT_GITHUB_TOKEN"] = previous_token - self.assertEqual("copilot-cli-fallback", response.provider["type"]) - self.assertEqual("copilot-cli", response.provider["fallbackFrom"]) - self.assertIn("exit code 1", response.provider["fallbackReason"]) - self.assertTrue(response.selected_ids) + self.assertEqual(COPILOT_CLI_MAX_ATTEMPTS, run_mock.call_count) class AiTransparencyReportProviderTests(unittest.TestCase): From a7c654d8614fd925145346fb2531edad51e0450c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 14:27:40 +0000 Subject: [PATCH 4/6] fix: clarify copilot retry failure handling Agent-Logs-Url: https://github.com/DevSecNinja/wazzup/sessions/8cf0349d-d3f8-4484-9586-ce6d58b41e81 Co-authored-by: DevSecNinja <14926452+DevSecNinja@users.noreply.github.com> --- src/wazzup/ai.py | 22 +++++++++++----------- tests/test_ai.py | 8 ++++++-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/wazzup/ai.py b/src/wazzup/ai.py index 67df6f1..a545c35 100644 --- a/src/wazzup/ai.py +++ b/src/wazzup/ai.py @@ -399,18 +399,18 @@ def generate_structured_summary(self, request: SummaryRequest) -> SummaryRespons "--no-ask-user", ] ) + run_copilot_cli(command, run_env, "Copilot CLI") + if not output_path.exists(): + raise RuntimeError("Copilot CLI did not write summary.json") + payload = json.loads(output_path.read_text(encoding="utf-8")) + provider = { + "type": self.name, + "model": payload.get("model", self.model or "copilot-cli"), + "agent": self.agent or None, + "promptVersion": "summary-v1", + "validated": True, + } try: - run_copilot_cli(command, run_env, "Copilot CLI") - if not output_path.exists(): - raise RuntimeError("Copilot CLI did not write summary.json") - payload = json.loads(output_path.read_text(encoding="utf-8")) - provider = { - "type": self.name, - "model": payload.get("model", self.model or "copilot-cli"), - "agent": self.agent or None, - "promptVersion": "summary-v1", - "validated": True, - } return response_from_payload(payload, provider=provider) except ValueError as exc: fallback = FakeSummaryProvider().generate_structured_summary(request) diff --git a/tests/test_ai.py b/tests/test_ai.py index 85938af..dfbfbea 100644 --- a/tests/test_ai.py +++ b/tests/test_ai.py @@ -419,7 +419,7 @@ def test_copilot_summary_retries_then_fails_on_runtime_failure(self, _which, run scored = score_items([item], [source], load_app_config("config/interests.yml"), datetime(2026, 5, 6, tzinfo=UTC)) run_mock.return_value = Mock(returncode=1, stdout="failed", stderr="upstream error") try: - with self.assertRaisesRegex(RuntimeError, f"after {COPILOT_CLI_MAX_ATTEMPTS} attempts"): + with self.assertRaisesRegex(RuntimeError, f"after {COPILOT_CLI_MAX_ATTEMPTS} attempts") as context: CopilotCliSummaryProvider().generate_structured_summary( SummaryRequest( kind="hourly", @@ -438,6 +438,8 @@ def test_copilot_summary_retries_then_fails_on_runtime_failure(self, _which, run os.environ["COPILOT_GITHUB_TOKEN"] = previous_token self.assertEqual(COPILOT_CLI_MAX_ATTEMPTS, run_mock.call_count) + self.assertIn("stdout: failed", str(context.exception)) + self.assertIn("stderr: upstream error", str(context.exception)) class AiCurationProviderTests(unittest.TestCase): @@ -646,7 +648,7 @@ def test_copilot_cli_curation_retries_then_fails_on_runtime_failure(self, _which scored = score_items([item], [source], load_app_config("config/interests.yml"), datetime(2026, 5, 6, tzinfo=UTC)) run_mock.return_value = Mock(returncode=1, stdout="failed", stderr="upstream error") try: - with self.assertRaisesRegex(RuntimeError, f"after {COPILOT_CLI_MAX_ATTEMPTS} attempts"): + with self.assertRaisesRegex(RuntimeError, f"after {COPILOT_CLI_MAX_ATTEMPTS} attempts") as context: CopilotCliCurationProvider().curate_items( CurationRequest( kind="hourly", @@ -665,6 +667,8 @@ def test_copilot_cli_curation_retries_then_fails_on_runtime_failure(self, _which os.environ["COPILOT_GITHUB_TOKEN"] = previous_token self.assertEqual(COPILOT_CLI_MAX_ATTEMPTS, run_mock.call_count) + self.assertIn("stdout: failed", str(context.exception)) + self.assertIn("stderr: upstream error", str(context.exception)) class AiTransparencyReportProviderTests(unittest.TestCase): From b4166a659b33f8e01ac538076a11a2daa02d40a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 14:29:34 +0000 Subject: [PATCH 5/6] style: clarify unused retry loop variable Agent-Logs-Url: https://github.com/DevSecNinja/wazzup/sessions/8cf0349d-d3f8-4484-9586-ce6d58b41e81 Co-authored-by: DevSecNinja <14926452+DevSecNinja@users.noreply.github.com> --- src/wazzup/ai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wazzup/ai.py b/src/wazzup/ai.py index a545c35..a6c6897 100644 --- a/src/wazzup/ai.py +++ b/src/wazzup/ai.py @@ -234,7 +234,7 @@ def generate_transparency_report(self, request: TransparencyReportRequest) -> Tr def run_copilot_cli(command: list[str], run_env: dict[str, str], failure_label: str) -> subprocess.CompletedProcess[str]: result: subprocess.CompletedProcess[str] | None = None - for _attempt in range(COPILOT_CLI_MAX_ATTEMPTS): + for _ in range(COPILOT_CLI_MAX_ATTEMPTS): result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) if result.returncode == 0: return result From 6ddd3c2cef3b8999989ec4cd565dc879d7cb8dbf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 21 May 2026 14:31:31 +0000 Subject: [PATCH 6/6] fix: fail malformed copilot cli outputs Agent-Logs-Url: https://github.com/DevSecNinja/wazzup/sessions/8cf0349d-d3f8-4484-9586-ce6d58b41e81 Co-authored-by: DevSecNinja <14926452+DevSecNinja@users.noreply.github.com> --- docs/adr/0002-ai-execution-strategy.md | 4 +- docs/github-actions.md | 2 +- src/wazzup/ai.py | 35 +++-------------- tests/test_ai.py | 54 +++++++++++++------------- 4 files changed, 34 insertions(+), 61 deletions(-) diff --git a/docs/adr/0002-ai-execution-strategy.md b/docs/adr/0002-ai-execution-strategy.md index 3b87dd0..7ec1a27 100644 --- a/docs/adr/0002-ai-execution-strategy.md +++ b/docs/adr/0002-ai-execution-strategy.md @@ -39,7 +39,7 @@ Implemented safety behavior: - If `copilot-cli` is requested without either token secret, the workflow logs a warning and uses `AI_PROVIDER=fake`. - [../../src/wazzup/ai.py](../../src/wazzup/ai.py) checks for `COPILOT_GITHUB_TOKEN` in GitHub Actions and raises an actionable error if the workflow guard is bypassed. - The provider defaults to model `claude-sonnet-4.6` and the repo-local `wazzup-writer` custom agent, both overridable through environment variables. -- Copilot CLI stdout/stderr is captured and included in sanitized failure diagnostics when the CLI exits non-zero. +- Copilot CLI runtime failures are retried, and stdout/stderr is captured and included in sanitized failure diagnostics when all attempts exit non-zero. ## Consequences @@ -57,7 +57,7 @@ Implemented safety behavior: - CLI behavior can be less predictable than a direct structured-output API and must be validated defensively. - Usage accounting may be less precise than direct API token accounting. - Ollama on GitHub-hosted runners can be slow and model downloads can dominate runtime; Foundry or other provider paths will require separate evaluation. -- Fake-provider fallback keeps automation green but produces deterministic placeholder-style summaries instead of true AI summaries until Copilot token setup is complete. +- Fake-provider fallback is limited to missing-token setup. Once `copilot-cli` is selected, runtime or structured-output failures fail the scheduled run so a later run can retry without publishing deterministic placeholder summaries. ## Alternatives considered diff --git a/docs/github-actions.md b/docs/github-actions.md index c154323..f53afe1 100644 --- a/docs/github-actions.md +++ b/docs/github-actions.md @@ -161,7 +161,7 @@ The workflow triggers hourly because GitHub cron is UTC-only and does not unders Operational learning: the first live News hourly run failed because Copilot CLI was requested but the token secret was empty. The workflow now selects an effective provider before installing Node/Copilot. If `copilot-cli` is requested without `COPILOT_REQUESTS_PAT` or `COPILOT_GITHUB_TOKEN`, it logs a warning and uses `AI_PROVIDER=fake` so the release state and Pages deployment path can still be validated end to end. -After enabling the Copilot PAT, one live run failed because Copilot CLI wrote JSON without the required `sections` array. The provider now treats invalid structured Copilot output as an AI-provider failure and falls back to the deterministic summary shape, recording `provider.type: copilot-cli-fallback` and the validation reason instead of failing the whole state/deploy pipeline. +After enabling the Copilot PAT, live runs showed that Copilot CLI can fail transiently or return malformed structured output. The provider retries Copilot CLI runtime failures and then fails with captured stdout/stderr diagnostics, while invalid structured output fails validation before publication so a later scheduled run can try again without publishing deterministic fake content. The Copilot CLI provider pins the briefing writer to `COPILOT_MODEL`, defaulting to Claude Sonnet 4.6 via the CLI model ID `claude-sonnet-4.6`, and invokes the repo-local `wazzup-writer` custom agent. This avoids the CLI's higher-cost default model while keeping the model overridable for manual canaries. diff --git a/src/wazzup/ai.py b/src/wazzup/ai.py index a6c6897..f9df68d 100644 --- a/src/wazzup/ai.py +++ b/src/wazzup/ai.py @@ -233,14 +233,14 @@ def generate_transparency_report(self, request: TransparencyReportRequest) -> Tr def run_copilot_cli(command: list[str], run_env: dict[str, str], failure_label: str) -> subprocess.CompletedProcess[str]: - result: subprocess.CompletedProcess[str] | None = None + failed_results: list[subprocess.CompletedProcess[str]] = [] for _ in range(COPILOT_CLI_MAX_ATTEMPTS): result = subprocess.run(command, capture_output=True, cwd=Path.cwd(), env=run_env, text=True) if result.returncode == 0: return result - if result is None: - raise RuntimeError(f"{failure_label} was not attempted") + failed_results.append(result) + result = failed_results[-1] details = [] if result.stdout.strip(): details.append(f"stdout: {result.stdout.strip()}") @@ -318,18 +318,7 @@ def curate_items(self, request: CurationRequest) -> CurationResponse: payload = json.loads(output_path.read_text(encoding="utf-8")) selected_ids = payload.get("selectedIds") if not isinstance(selected_ids, list) or not all(isinstance(item_id, str) for item_id in selected_ids): - fallback = FakeCurationProvider() - fallback_response = fallback.curate_items(request) - return CurationResponse( - selected_ids=fallback_response.selected_ids, - provider={ - **fallback_response.provider, - "type": "copilot-cli-fallback", - "fallbackFrom": self.name, - "fallbackReason": "Curator returned invalid selectedIds", - "validated": True, - }, - ) + raise ValueError("Copilot CLI curation returned invalid selectedIds") provider = { "type": self.name, "model": payload.get("model", self.model or "copilot-cli"), @@ -410,21 +399,7 @@ def generate_structured_summary(self, request: SummaryRequest) -> SummaryRespons "promptVersion": "summary-v1", "validated": True, } - try: - return response_from_payload(payload, provider=provider) - except ValueError as exc: - fallback = FakeSummaryProvider().generate_structured_summary(request) - return SummaryResponse( - headline=fallback.headline, - sections=fallback.sections, - provider={ - **fallback.provider, - "type": "copilot-cli-fallback", - "fallbackFrom": self.name, - "fallbackReason": str(exc), - "validated": True, - }, - ) + return response_from_payload(payload, provider=provider) class CopilotCliTransparencyReportProvider: diff --git a/tests/test_ai.py b/tests/test_ai.py index dfbfbea..0afc776 100644 --- a/tests/test_ai.py +++ b/tests/test_ai.py @@ -373,7 +373,7 @@ def test_copilot_requires_token_in_github_actions(self, _which) -> None: # type @patch("wazzup.ai.subprocess.run") @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") - def test_copilot_invalid_payload_falls_back_to_deterministic_summary(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] + def test_copilot_invalid_payload_fails_summary_generation(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] previous_token = os.environ.get("COPILOT_GITHUB_TOKEN") os.environ["COPILOT_GITHUB_TOKEN"] = "test-token" source = load_sources("config/sources.yml")[0] @@ -388,26 +388,25 @@ def fake_run(_command, capture_output, cwd, env, text): # type: ignore[no-untyp run_mock.side_effect = fake_run try: - response = CopilotCliSummaryProvider().generate_structured_summary( - SummaryRequest( - kind="hourly", - window_start="2026-05-06T00:00:00Z", - window_end="2026-05-06T21:00:00Z", - generated_at="2026-05-06T21:00:00Z", - timezone="Europe/Amsterdam", - summary_language="en", - items=scored, + with self.assertRaisesRegex(ValueError, "missing sections"): + CopilotCliSummaryProvider().generate_structured_summary( + SummaryRequest( + kind="hourly", + window_start="2026-05-06T00:00:00Z", + window_end="2026-05-06T21:00:00Z", + generated_at="2026-05-06T21:00:00Z", + timezone="Europe/Amsterdam", + summary_language="en", + items=scored, + ) ) - ) finally: if previous_token is None: os.environ.pop("COPILOT_GITHUB_TOKEN", None) else: os.environ["COPILOT_GITHUB_TOKEN"] = previous_token - self.assertEqual("copilot-cli-fallback", response.provider["type"]) - self.assertIn("fallbackReason", response.provider) - self.assertTrue(response.sections[0]["bullets"]) + self.assertEqual(1, run_mock.call_count) @patch("wazzup.ai.subprocess.run") @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") @@ -602,7 +601,7 @@ def fake_run(command, capture_output, cwd, env, text): # type: ignore[no-untype @patch("wazzup.ai.subprocess.run") @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot") - def test_copilot_cli_curation_falls_back_on_invalid_response(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] + def test_copilot_cli_curation_fails_on_invalid_response(self, _which, run_mock) -> None: # type: ignore[no-untyped-def] previous_token = os.environ.get("COPILOT_GITHUB_TOKEN") os.environ["COPILOT_GITHUB_TOKEN"] = "test-token" source = load_sources("config/sources.yml")[0] @@ -617,26 +616,25 @@ def fake_run(_command, capture_output, cwd, env, text): # type: ignore[no-untyp run_mock.side_effect = fake_run try: - response = CopilotCliCurationProvider().curate_items( - CurationRequest( - kind="hourly", - window_start="2026-05-06T20:00:00Z", - window_end="2026-05-06T21:00:00Z", - generated_at="2026-05-06T21:00:00Z", - timezone="Europe/Amsterdam", - items=scored, - max_items=12, + with self.assertRaisesRegex(ValueError, "invalid selectedIds"): + CopilotCliCurationProvider().curate_items( + CurationRequest( + kind="hourly", + window_start="2026-05-06T20:00:00Z", + window_end="2026-05-06T21:00:00Z", + generated_at="2026-05-06T21:00:00Z", + timezone="Europe/Amsterdam", + items=scored, + max_items=12, + ) ) - ) finally: if previous_token is None: os.environ.pop("COPILOT_GITHUB_TOKEN", None) else: os.environ["COPILOT_GITHUB_TOKEN"] = previous_token - self.assertEqual("copilot-cli-fallback", response.provider["type"]) - self.assertIn("fallbackReason", response.provider) - self.assertTrue(response.selected_ids) + self.assertEqual(1, run_mock.call_count) @patch("wazzup.ai.subprocess.run") @patch("wazzup.ai.shutil.which", return_value="/usr/bin/copilot")