From 0e48a82beb7e03a99081128f73c2b47bf1685ca5 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Thu, 9 Apr 2026 00:27:57 +0300 Subject: [PATCH 01/28] docs(08): capture phase context --- .../08-github-action-readme/08-CONTEXT.md | 114 ++++++++++++ .../08-DISCUSSION-LOG.md | 164 ++++++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 .planning/phases/08-github-action-readme/08-CONTEXT.md create mode 100644 .planning/phases/08-github-action-readme/08-DISCUSSION-LOG.md diff --git a/.planning/phases/08-github-action-readme/08-CONTEXT.md b/.planning/phases/08-github-action-readme/08-CONTEXT.md new file mode 100644 index 0000000..41b1b3a --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-CONTEXT.md @@ -0,0 +1,114 @@ +# Phase 8: GitHub Action + README - Context + +**Gathered:** 2026-04-08 +**Status:** Ready for planning + + +## Phase Boundary + +Ship a reusable GitHub Actions workflow and an updated README so maintainers can automatically update the PR knowledge base after merged PRs, manually backfill when needed, avoid unnecessary API calls, and onboard from the docs alone. + + + + +## Implementation Decisions + +### Trigger Model +- **D-01:** The workflow runs primarily on post-merge PR events, not on a cron schedule. +- **D-02:** A `workflow_dispatch` entry point is also included for recovery and backfill runs. +- **D-03:** Normal automated runs process merged/closed PRs only, not open PRs. + +### Incremental Boundary & Recovery +- **D-04:** When a manual recovery/backfill run needs a cursor, it is based on PRs updated since the last successful backfill point. This intentionally aligns with the extractor's existing `updated_at` filtering semantics. +- **D-05:** The repository variable is a recovery/backfill cursor for manual runs, not the primary driver of normal post-merge execution. + +### Publication Mode +- **D-06:** KB updates are published through one rolling bot PR that is created if absent and updated in place if it already exists. +- **D-07:** The workflow commits only generated KB output to that PR. Extraction/classification working data stays out of git. + +### Cache & State Persistence +- **D-08:** The last-successful-run marker lives outside the tracked repo as a repository variable. +- **D-09:** Extraction/classification caches persist outside git, with GitHub Actions cache as the primary reuse mechanism. +- **D-10:** Workflow artifacts are optional and used only when helpful for debugging, not as the normal persistence channel. + +### README Shape +- **D-11:** The README is automation-first: GitHub Action setup comes before local CLI usage. +- **D-12:** Local setup, environment variables, command reference, and KB output examples are still included after the automation guidance so users can run and inspect the tool manually. + +### the agent's Discretion +- Exact event syntax and guard conditions for merged-only PR triggers +- Bot PR title/branch naming, commit message wording, and update mechanics +- Whether manual dispatch supports explicit PR targeting, date-range backfill, or both +- Exact README section naming and example formatting + + + + +## Canonical References + +**Downstream agents MUST read these before planning or implementing.** + +### Phase Requirements +- `.planning/ROADMAP.md` — Phase 8 goal, dependency chain, and success criteria +- `.planning/REQUIREMENTS.md` — ACTION-01, ACTION-02, ACTION-03, and INFRA-03 +- `.planning/PROJECT.md` — project vision, single-repo MVP scope, and CLI + GitHub Action product direction + +### Existing Pipeline +- `src/github_pr_kb/cli.py` — current CLI entry points (`extract`, `classify`, `generate`, `run`) and existing pipeline wiring +- `src/github_pr_kb/extractor.py` — extraction entry point, `updated_at` filtering semantics, and merge-on-rerun cache behavior +- `src/github_pr_kb/generator.py` — KB output, manifest behavior, and generation entry point +- `src/github_pr_kb/config.py` — environment/config surface the workflow and README must document +- `.env.example` — current documented environment variables and defaults + +### Existing Automation & Repo Conventions +- `.github/workflows/ci.yml` — established GitHub Actions setup pattern (`setup-uv`, dependency install, test invocation) +- `.github/workflows/claude.yml` — existing bot-oriented workflow pattern and permissions structure +- `.gitignore` — `.github-pr-kb/` is ignored, which constrains what automation should commit vs persist externally +- `README.md` — current documentation baseline to replace/restructure for Phase 8 + + + + +## Existing Code Insights + +### Reusable Assets +- `src/github_pr_kb/cli.py`: Existing commands already compose the full pipeline; the workflow can reuse CLI entry points rather than inventing parallel Python entrypoints. +- `src/github_pr_kb/extractor.py`: `extract()` already supports `since` / `until` filtering and merge-based cache reuse, which is useful for manual backfill behavior. +- `src/github_pr_kb/generator.py`: `generate_all()` already handles manifest-based incremental KB output and is the natural workflow generation step. +- `.github/workflows/ci.yml`: Existing Action setup shows the repo's preferred `uv` installation flow and Python execution pattern. + +### Established Patterns +- `.github-pr-kb/` is local working data and is currently gitignored. +- Generated KB content lives under `kb/` and is treated as repo-visible output. +- Existing GitHub workflows already use checkout + setup-uv conventions; Phase 8 should stay consistent with those patterns. +- The tool is single-repo and sync-first for v1; Phase 8 should not introduce multi-repo or async orchestration. + +### Integration Points +- New workflow logic will live alongside existing `.github/workflows/*.yml` automation. +- The workflow must bridge GitHub event context to the existing CLI pipeline and/or underlying Python entry points. +- Repository-variable reads/writes and Actions-cache restore/save become part of the workflow contract. +- README updates must align with the actual env vars, CLI flags, and workflow file shipped in the repo. + + + + +## Specific Ideas + +- The user explicitly wants the automation to run after PR merges rather than on a schedule. +- The bot should maintain a single rolling KB update PR instead of creating a new PR every run. +- Repo cleanliness matters: commit KB output, but keep extraction/classification working data outside git. +- Manual runs should exist for recovery/backfill, with repository-variable state supporting that path. + + + + +## Deferred Ideas + +None — discussion stayed within phase scope + + + +--- + +*Phase: 08-github-action-readme* +*Context gathered: 2026-04-08* diff --git a/.planning/phases/08-github-action-readme/08-DISCUSSION-LOG.md b/.planning/phases/08-github-action-readme/08-DISCUSSION-LOG.md new file mode 100644 index 0000000..eb4b5e8 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-DISCUSSION-LOG.md @@ -0,0 +1,164 @@ +# Phase 8: GitHub Action + README - Discussion Log + +> **Audit trail only.** Do not use as input to planning, research, or execution agents. +> Decisions are captured in CONTEXT.md — this log preserves the alternatives considered. + +**Date:** 2026-04-08T21:10:54Z +**Phase:** 08-GitHub Action + README +**Areas discussed:** Incremental boundary, PR state scope, Publication mode, Bot PR behavior, State persistence, External state mechanism, README structure, Cache persistence, Cache reuse mechanism, Trigger model, Recovery cursor role + +--- + +## Incremental boundary + +| Option | Description | Selected | +|--------|-------------|----------| +| Any PR updated since the last successful run | Reuses the extractor's current `updated_at` semantics and catches fresh discussion on older PRs | ✓ | +| Only PRs created since the last successful run | Lowest scope, but misses later activity on existing PRs | | +| Only PRs merged/closed since the last successful run | Stable lifecycle boundary, but narrower than current extractor behavior | | + +**User's choice:** Any PR updated since the last successful run. +**Notes:** This was selected because it matches the current extractor behavior and avoids missing fresh comments on older PRs. + +--- + +## PR state scope + +| Option | Description | Selected | +|--------|-------------|----------| +| Closed/merged only | Focus on stable review history and avoid churn from in-progress PRs | ✓ | +| Open and closed PRs | Capture review knowledge earlier, but includes unstable PRs | | +| Use the extractor default of all states | Minimal workflow opinion, but broader than needed | | + +**User's choice:** Closed/merged only. +**Notes:** This keeps automation focused on stable, merged-review knowledge. + +--- + +## Publication mode + +| Option | Description | Selected | +|--------|-------------|----------| +| Commit directly to the default branch | Most automatic, but least reviewable | | +| Open or update a bot PR with the KB changes | Keeps updates reviewable and visible | ✓ | +| Upload artifacts only | Avoids repo writes, but does not keep the KB in the repo | | + +**User's choice:** Open or update a bot PR with the KB changes. +**Notes:** Reviewability matters more than fully hands-off direct commits. + +--- + +## Bot PR behavior + +| Option | Description | Selected | +|--------|-------------|----------| +| Keep one rolling update PR and refresh it over time | One durable review surface for KB automation | ✓ | +| Open a fresh PR for each run with changes | Clear run boundaries, but creates more PR churn | | +| Skip when a KB PR already exists | Simple guard, but can leave the PR stale | | + +**User's choice:** Open a PR if absent; otherwise update the existing open PR. +**Notes:** Interpreted as one rolling bot PR that is kept current over time. + +--- + +## State persistence + +| Option | Description | Selected | +|--------|-------------|----------| +| Outside the tracked repo in GitHub-managed workflow state | Decouples progress tracking from KB PR merge timing | ✓ | +| In a state file inside the KB update PR | Transparent in git, but delayed until merge | | +| In a state file committed directly to default branch | Accurate cursor, but mixes publication modes | | + +**User's choice:** Outside the tracked repo in GitHub-managed workflow state. +**Notes:** This keeps recovery state independent from the bot PR lifecycle. + +--- + +## External state mechanism + +| Option | Description | Selected | +|--------|-------------|----------| +| Repository variable updated by the workflow | Durable repo-scoped state without committing files | ✓ | +| Artifact file attached to workflow runs | Inspectable, but retention-based and less direct | | +| Actions cache entry | Fast, but less clear as a canonical cursor | | + +**User's choice:** Repository variable updated by the workflow. +**Notes:** Chosen as the canonical recovery/backfill cursor. + +--- + +## README structure + +| Option | Description | Selected | +|--------|-------------|----------| +| Quickstart-first | Local setup and first run first, then automation | | +| Automation-first | GitHub Action setup first, local CLI usage after | ✓ | +| Reference-first | Concise overview, then full command/env/workflow reference | | + +**User's choice:** Automation-first. +**Notes:** The README should lead with automation setup, then cover local usage and reference details. + +--- + +## Cache persistence + +| Option | Description | Selected | +|--------|-------------|----------| +| Keep caches outside git using GitHub Actions cache/artifacts; commit only KB output | Matches current gitignore posture and keeps repo output focused | ✓ | +| Include cache files in the rolling KB bot PR too | Maximizes reproducibility, but adds noisy repo churn | | +| Do not persist caches across runs | Simplest, but wastes recomputation and API calls | | + +**User's choice:** Keep caches outside git and commit only KB output. +**Notes:** The repo should stay clean; working caches should remain non-git state. + +--- + +## Cache reuse mechanism + +| Option | Description | Selected | +|--------|-------------|----------| +| Actions cache primary; artifacts only when useful for debugging | Optimizes normal reuse without artifact noise | ✓ | +| Artifacts only | More inspectable, but clunkier for normal reuse | | +| Always write both cache and artifacts on every run | Maximum traceability, but noisy and redundant | | + +**User's choice:** Actions cache primary; artifacts only when useful for debugging. +**Notes:** Cache is the default persistence layer; artifacts are secondary. + +--- + +## Trigger model + +| Option | Description | Selected | +|--------|-------------|----------| +| Post-merge on merged PRs, plus manual workflow_dispatch | Event-driven automation tied to completed PRs | ✓ | +| Hybrid: post-merge primary, scheduled backstop, plus manual workflow_dispatch | More resilient, but keeps schedule complexity | | +| Keep the roadmap's schedule-based automation | Matches roadmap wording, but not the user's preferred trigger model | | + +**User's choice:** Post-merge on merged PRs, plus manual workflow_dispatch. +**Notes:** The user explicitly prefers post-merge execution over a schedule. + +--- + +## Recovery cursor role + +| Option | Description | Selected | +|--------|-------------|----------| +| Keep it as a recovery/backfill cursor for manual runs | Normal events use PR context directly; manual runs use the cursor | ✓ | +| Do not use a cursor at all | Simpler, but weaker recovery/backfill story | | +| Use it for every run anyway | Keeps one model, but is unnecessary for event-driven runs | | + +**User's choice:** Keep the repository variable as a recovery/backfill cursor for manual runs. +**Notes:** Post-merge events should use event context directly; the variable exists for manual recovery paths. + +--- + +## the agent's Discretion + +- Exact YAML event filters and condition expressions +- Exact PR title/branch naming for the rolling bot PR +- Exact manual-dispatch inputs for recovery mode +- Exact README section headings and example formatting + +## Deferred Ideas + +None From 85a1927d95556b8443455040a2c3a19198ec24f7 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Thu, 9 Apr 2026 00:27:59 +0300 Subject: [PATCH 02/28] docs(state): record phase 8 context session --- .planning/STATE.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.planning/STATE.md b/.planning/STATE.md index c912658..4aae2c8 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -3,15 +3,15 @@ gsd_state_version: 1.0 milestone: v1.0 milestone_name: milestone status: verifying -stopped_at: Phase 7 context gathered -last_updated: "2026-04-07T07:01:02.768Z" -last_activity: 2026-04-06 +stopped_at: Phase 8 context gathered +last_updated: "2026-04-08T21:27:59.165Z" +last_activity: 2026-04-08 progress: total_phases: 8 - completed_phases: 6 - total_plans: 9 - completed_plans: 9 - percent: 29 + completed_phases: 7 + total_plans: 12 + completed_plans: 12 + percent: 100 --- # Project State @@ -121,6 +121,6 @@ Recent decisions affecting current work: ## Session Continuity -Last session: 2026-04-07T07:01:02.757Z -Stopped at: Phase 7 context gathered -Resume file: .planning/phases/07-fix-article-generation-quality/07-CONTEXT.md +Last session: 2026-04-08T21:27:59.156Z +Stopped at: Phase 8 context gathered +Resume file: .planning\phases\08-github-action-readme\08-CONTEXT.md From da8d4efcae446f0525503827851aba86b063ae90 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 00:17:21 +0300 Subject: [PATCH 03/28] docs(08): create phase plan --- .planning/ROADMAP.md | 10 +- .../08-github-action-readme/08-01-PLAN.md | 170 ++++++++++++++ .../08-github-action-readme/08-02-PLAN.md | 208 ++++++++++++++++++ .../08-github-action-readme/08-03-PLAN.md | 174 +++++++++++++++ 4 files changed, 558 insertions(+), 4 deletions(-) create mode 100644 .planning/phases/08-github-action-readme/08-01-PLAN.md create mode 100644 .planning/phases/08-github-action-readme/08-02-PLAN.md create mode 100644 .planning/phases/08-github-action-readme/08-03-PLAN.md diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index b2baace..7ccb781 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -142,7 +142,7 @@ Plans: - [x] 07-03-PLAN.md — Fix CLI output accuracy, add --regenerate flag, API key validation for generate ### Phase 8: GitHub Action + README -**Goal**: A repository maintainer can add a provided workflow file and have PR comments automatically extracted and the KB updated on a schedule, with no wasted API calls when nothing is new, and a new user can get from zero to a generated KB by following the README alone. +**Goal**: A repository maintainer can add a provided workflow file and have PR comments automatically extracted after merged PRs or manual recovery runs, with no wasted API calls when nothing is new, and a new user can get from zero to a generated KB by following the README alone. **Depends on**: Phase 7 **Requirements**: ACTION-01, ACTION-02, ACTION-03, INFRA-03 **Success Criteria** (what must be TRUE): @@ -151,10 +151,12 @@ Plans: 3. Last-run state is persisted across Action runs so incremental executions only process PRs added since the previous successful run 4. README contains: setup steps (including uv installation), all required environment variables, CLI command reference, and at least one example of KB output 5. Tests covering this phase's components pass (mocked external APIs where applicable) -**Plans**: TBD +**Plans**: 3 plans Plans: -- [ ] TBD (run /gsd:plan-phase 8 to break down) +- [ ] 08-01-PLAN.md — testable action-state helper for no-new-PR guard and cursor decisions +- [ ] 08-02-PLAN.md — merged-PR workflow with dual-mode variable auth, cache reuse, and rolling KB PR publication +- [ ] 08-03-PLAN.md — automation-first README rewrite with PAT quickstart, GitHub App option, and local CLI docs ## Progress @@ -170,4 +172,4 @@ Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 | 5. KB Generator | 2/2 | Complete | 2026-04-06 | | 6. CLI Integration | 1/1 | Complete | 2026-04-06 | | 7. Fix Article Generation Quality | 3/3 | Ready for verification | - | -| 8. GitHub Action + README | 0/TBD | Not started | - | +| 8. GitHub Action + README | 0/3 | Not started | - | diff --git a/.planning/phases/08-github-action-readme/08-01-PLAN.md b/.planning/phases/08-github-action-readme/08-01-PLAN.md new file mode 100644 index 0000000..6b6dfe0 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-01-PLAN.md @@ -0,0 +1,170 @@ +--- +phase: 08-github-action-readme +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - src/github_pr_kb/action_state.py + - tests/test_action_state.py +autonomous: true +requirements: + - ACTION-02 + - ACTION-03 +must_haves: + truths: + - "Workflow support code decides before pipeline startup whether a merged-PR or manual run should proceed, per D-01, D-02, D-04, and D-05." + - "The automation cursor uses extractor updated_at semantics for manual recovery/backfill, per D-04." + - "A successful run can emit a deterministic next cursor, while failed runs can leave repository-variable writes to the workflow's post-success step, per D-08." + artifacts: + - path: "src/github_pr_kb/action_state.py" + provides: "Testable action skip/cursor decision helper and CLI entrypoint" + contains: "def decide_action_run" + - path: "tests/test_action_state.py" + provides: "Pytest coverage for skip/cursor/manual override logic" + contains: "test_manual_dispatch_skips_when_cursor_is_current" + key_links: + - from: "src/github_pr_kb/action_state.py" + to: "src/github_pr_kb/extractor.py" + via: "extract_since output aligned to updated_at filtering" + pattern: "updated_at|extract_since" + - from: "src/github_pr_kb/action_state.py" + to: ".github/workflows/github-pr-kb.yml" + via: "python -m github_pr_kb.action_state" + pattern: "main\\(" +--- + + +Create the testable automation decision contract the workflow will rely on for cost-aware skipping and cursor handling. + +Purpose: Keep state/cost-guard logic out of opaque YAML so ACTION-02 and ACTION-03 are enforced by pytest-covered Python, while preserving the locked merged-PR/manual-backfill behavior from D-01 through D-05 and D-08. +Output: `src/github_pr_kb/action_state.py` plus focused pytest coverage. + + + +@~/.copilot/get-shit-done/workflows/execute-plan.md +@~/.copilot/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/REQUIREMENTS.md +@.planning/phases/08-github-action-readme/08-CONTEXT.md +@.planning/phases/08-github-action-readme/08-RESEARCH.md +@.planning/phases/08-github-action-readme/08-VALIDATION.md +@src/github_pr_kb/cli.py +@src/github_pr_kb/extractor.py + + +From `src/github_pr_kb/extractor.py`: +```python +class GitHubExtractor: + def extract( + self, + state: str = "all", + since: datetime | None = None, + until: datetime | None = None, + ) -> list[Path]: +``` + +From `src/github_pr_kb/cli.py`: +```python +def _run_extract( + repo: str, + state: str, + since_dt: datetime | None, + until_dt: datetime | None, +) -> str: +``` + + + + + + + Task 1: Add workflow-state decision tests before implementation + tests/test_action_state.py + + - tests/test_cli.py + - src/github_pr_kb/extractor.py + - .planning/phases/08-github-action-readme/08-CONTEXT.md + - .planning/phases/08-github-action-readme/08-RESEARCH.md + - .planning/phases/08-github-action-readme/08-VALIDATION.md + + Create `tests/test_action_state.py` with pytest cases that lock the helper contract before code exists. Cover these exact scenarios: (1) merged PR event with `event_updated_at` newer than `stored_cursor` returns `should_run=True`, `extract_since=event_updated_at`, and `next_cursor=event_updated_at` per D-01/D-03; (2) `workflow_dispatch` with no `manual_since`, `force=False`, and `latest_merged_at <= stored_cursor` returns `should_run=False` with reason `no_new_closed_prs` per ACTION-02 and D-05; (3) `workflow_dispatch` with `manual_since` uses that value as `extract_since` and does not silently replace it with the stored cursor per D-04; (4) `force=True` bypasses the no-new-PR short-circuit but still emits the chosen `extract_since`; (5) the module's CLI mode prints JSON with keys `should_run`, `extract_since`, `next_cursor`, and `reason` so the workflow can consume it without YAML-only logic. Keep tests pure/unit-level; do not call the GitHub API. + + .venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q + + + - `tests/test_action_state.py` exists + - file contains `test_merged_pr_event_uses_event_updated_at` + - file contains `test_manual_dispatch_skips_when_cursor_is_current` + - file contains `test_manual_since_override_wins_over_stored_cursor` + - file contains an assertion on JSON keys `should_run`, `extract_since`, `next_cursor`, and `reason` + + Red tests define the exact skip/cursor contract for ACTION-02 and ACTION-03 before any helper logic is written. + + + + Task 2: Implement the action-state helper and JSON CLI contract + src/github_pr_kb/action_state.py, tests/test_action_state.py + + - src/github_pr_kb/action_state.py + - tests/test_action_state.py + - src/github_pr_kb/extractor.py + - src/github_pr_kb/cli.py + - .planning/phases/08-github-action-readme/08-CONTEXT.md + - .planning/phases/08-github-action-readme/08-RESEARCH.md + + + - Test 1: merged PR events emit the event PR `updated_at` timestamp as both `extract_since` and `next_cursor` + - Test 2: manual runs skip before extract/classify/generate when no merged PR is newer than the effective cursor + - Test 3: manual `since` override and `force` bypass behave deterministically + + Create `src/github_pr_kb/action_state.py` with a small typed contract such as `ActionRunDecision` plus a public `decide_action_run(...)` function. Normalize all incoming timestamps with timezone awareness, keep comparisons in UTC, and return ISO-8601 strings suitable for workflow env/JSON output. The function must implement these concrete rules: auto merged-PR runs use the event PR `updated_at` timestamp as the candidate cursor per D-01/D-03; manual runs use `manual_since` when supplied, otherwise the stored repository variable cursor per D-04/D-05; `should_run` is false only when `force` is false and the newest merged PR timestamp is not newer than the effective cursor; `next_cursor` is the newest successfully-targeted merged PR timestamp, but the workflow will persist it only after later steps succeed per D-08. Add a minimal `python -m github_pr_kb.action_state` entrypoint that accepts flags for `--event-name`, `--merged`, `--event-updated-at`, `--stored-cursor`, `--latest-merged-at`, `--manual-since`, and `--force`, then prints one JSON object to stdout for workflow consumption. Do not add Click or any new dependency. + + .venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q + + + - `src/github_pr_kb/action_state.py` contains `def decide_action_run` + - `src/github_pr_kb/action_state.py` contains `if __name__ == "__main__":` + - merged-event logic uses the event PR timestamp, not a cron/schedule path + - manual logic compares against the stored cursor or `manual_since` using `updated_at` semantics + - stdout JSON contains exact keys `should_run`, `extract_since`, `next_cursor`, and `reason` + + The repo has a reusable, pytest-covered Python helper the workflow can invoke for cost guard and cursor decisions instead of burying that logic in YAML. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| workflow inputs → action_state helper | GitHub event payload and manual dispatch strings are untrusted input | +| action_state helper → workflow outputs | Incorrect cursor/skip output can suppress processing or overrun API cost | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-08-01 | T | `src/github_pr_kb/action_state.py` timestamp parsing | mitigate | Normalize/validate ISO timestamps and reject malformed values with deterministic errors instead of silently coercing bad cursor data | +| T-08-02 | D | manual dispatch skip logic | mitigate | Unit-test the no-new-PR short-circuit and `force` bypass so malformed guard logic cannot trigger unnecessary extract/classify/generate runs | +| T-08-03 | E | workflow helper surface | accept | Helper emits decision JSON only; it does not write repo state or call external APIs, so privilege impact is low | + + + +Run `.venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q` and confirm the new helper emits the expected JSON contract for workflow consumption. + + + +- ACTION-02 guard logic is covered by targeted pytest cases +- ACTION-03 cursor selection is deterministic and uses updated-at semantics +- Workflow can import and execute `python -m github_pr_kb.action_state` without adding dependencies + + + +After completion, create `.planning/phases/08-github-action-readme/08-01-SUMMARY.md` + diff --git a/.planning/phases/08-github-action-readme/08-02-PLAN.md b/.planning/phases/08-github-action-readme/08-02-PLAN.md new file mode 100644 index 0000000..07bb45f --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-02-PLAN.md @@ -0,0 +1,208 @@ +--- +phase: 08-github-action-readme +plan: 02 +type: execute +wave: 2 +depends_on: + - "08-01" +files_modified: + - .github/workflows/github-pr-kb.yml + - tests/test_action_workflow.py +autonomous: true +requirements: + - ACTION-01 + - ACTION-02 + - ACTION-03 +user_setup: + - service: github + why: "Workflow needs repository-variable write auth and bot PR publication" + env_vars: + - name: ANTHROPIC_API_KEY + source: "Anthropic Console -> API Keys" + - name: KB_VARIABLES_TOKEN + source: "GitHub fine-grained PAT secret with repository Variables: write (default quickstart path)" + - name: KB_VARIABLES_APP_ID + source: "GitHub App settings -> App ID (optional advanced path)" + - name: KB_VARIABLES_APP_PRIVATE_KEY + source: "GitHub App settings -> Private keys (optional advanced path)" + dashboard_config: + - task: "Grant Variables: write to the PAT or GitHub App and store the chosen credentials as repository secrets" + location: "GitHub repository Settings -> Secrets and variables" +must_haves: + truths: + - "The repo ships a reusable workflow triggered by merged PRs and `workflow_dispatch`, not cron, per D-01 and D-02." + - "Automation restores `.github-pr-kb/cache/` from Actions cache and keeps that working data out of git, per D-07 and D-09." + - "The workflow updates exactly one rolling KB bot PR and stages `kb/.manifest.json` with generated markdown, per D-06 and D-07." + - "Repository-variable writes support PAT quickstart first and GitHub App as the advanced option with deterministic precedence, per D-13 and D-14." + artifacts: + - path: ".github/workflows/github-pr-kb.yml" + provides: "Merged-PR + workflow_dispatch automation" + contains: "workflow_dispatch" + - path: ".github/workflows/github-pr-kb.yml" + provides: "Cache/state/publication workflow" + contains: "KB_LAST_SUCCESSFUL_CURSOR" + - path: "tests/test_action_workflow.py" + provides: "Workflow contract assertions" + contains: "test_workflow_has_merged_pr_and_dispatch_triggers" + key_links: + - from: ".github/workflows/github-pr-kb.yml" + to: "src/github_pr_kb/action_state.py" + via: "python -m github_pr_kb.action_state" + pattern: "github_pr_kb\\.action_state" + - from: ".github/workflows/github-pr-kb.yml" + to: "src/github_pr_kb/cli.py" + via: "github-pr-kb extract/classify/generate" + pattern: "github-pr-kb extract|github-pr-kb classify|github-pr-kb generate" + - from: ".github/workflows/github-pr-kb.yml" + to: "kb/.manifest.json" + via: "git add kb/.manifest.json" + pattern: "kb/\\.manifest\\.json" +--- + + +Ship the actual GitHub Actions automation: merged-PR trigger, manual backfill entrypoint, dual-mode variable auth, cache reuse, rolling PR publication, and post-success cursor persistence. + +Purpose: Deliver ACTION-01 through ACTION-03 in the shipped workflow itself while honoring the locked trigger/auth/publication decisions D-01 through D-10 and D-13 through D-14. +Output: `.github/workflows/github-pr-kb.yml` plus workflow contract tests. + + + +@~/.copilot/get-shit-done/workflows/execute-plan.md +@~/.copilot/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/REQUIREMENTS.md +@.planning/phases/08-github-action-readme/08-CONTEXT.md +@.planning/phases/08-github-action-readme/08-RESEARCH.md +@.planning/phases/08-github-action-readme/08-VALIDATION.md +@.github/workflows/ci.yml +@.github/workflows/claude.yml +@.gitignore +@src/github_pr_kb/cli.py +@src/github_pr_kb/config.py +@src/github_pr_kb/action_state.py + + +From `src/github_pr_kb/cli.py`: +```python +github-pr-kb extract --repo owner/name --state closed --since 2024-01-01 +github-pr-kb classify +github-pr-kb generate +``` + +From `src/github_pr_kb/action_state.py` (Plan 01 contract): +```python +def decide_action_run(...) -> ActionRunDecision +# CLI mode prints: +# {"should_run": true, "extract_since": "...", "next_cursor": "...", "reason": "..."} +``` + + + + + + + Task 1: Add workflow contract tests that lock the shipped automation shape + tests/test_action_workflow.py + + - .github/workflows/ci.yml + - .github/workflows/claude.yml + - src/github_pr_kb/action_state.py + - src/github_pr_kb/cli.py + - .gitignore + - .planning/phases/08-github-action-readme/08-CONTEXT.md + - .planning/phases/08-github-action-readme/08-RESEARCH.md + - .planning/phases/08-github-action-readme/08-VALIDATION.md + + Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it uses `astral-sh/setup-uv` and `uv sync --all-groups --frozen` like `ci.yml`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; it runs `github-pr-kb extract --repo ${{ github.repository }} --state closed`, then `github-pr-kb classify`, then `github-pr-kb generate`; it stages `kb/.manifest.json` and KB markdown only, never `.github-pr-kb/cache`; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. + + .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q + + + - `tests/test_action_workflow.py` exists + - file contains `test_workflow_has_merged_pr_and_dispatch_triggers` + - file contains an assertion that `schedule:` is absent + - file contains `KB_LAST_SUCCESSFUL_CURSOR` + - file contains `automation/github-pr-kb` + + Workflow tests define the exact shipped automation contract before the YAML is written. + + + + Task 2: Implement the GitHub Actions workflow for cache-aware KB publication + .github/workflows/github-pr-kb.yml, tests/test_action_workflow.py + + - .github/workflows/github-pr-kb.yml + - tests/test_action_workflow.py + - .github/workflows/ci.yml + - .github/workflows/claude.yml + - src/github_pr_kb/action_state.py + - src/github_pr_kb/cli.py + - src/github_pr_kb/config.py + - src/github_pr_kb/generator.py + - .gitignore + - .planning/phases/08-github-action-readme/08-CONTEXT.md + - .planning/phases/08-github-action-readme/08-RESEARCH.md + + + - Test 1: merged PR and manual dispatch triggers are present, with no cron path + - Test 2: workflow skips before the expensive CLI pipeline when the helper reports no new merged PRs + - Test 3: successful runs stage KB markdown plus `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` + + Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty), determine the newest merged PR timestamp, and invoke `python -m github_pr_kb.action_state` to compute `should_run`, `extract_since`, and `next_cursor`. When `should_run` is false, exit before extract/classify/generate and print a no-new-PR reason per ACTION-02. When it is true, run `github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `github-pr-kb classify`, then `github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, commit only `kb/**/*.md` and `kb/.manifest.json`, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. + + .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q + + + - `.github/workflows/github-pr-kb.yml` contains `workflow_dispatch:` + - `.github/workflows/github-pr-kb.yml` contains `pull_request:` + - `.github/workflows/github-pr-kb.yml` contains `types: [closed]` + - `.github/workflows/github-pr-kb.yml` does not contain `schedule:` + - `.github/workflows/github-pr-kb.yml` contains `.github-pr-kb/cache/` + - `.github/workflows/github-pr-kb.yml` contains `kb/.manifest.json` + - `.github/workflows/github-pr-kb.yml` contains `KB_LAST_SUCCESSFUL_CURSOR` + - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_TOKEN` + - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_APP_ID` + - `.github/workflows/github-pr-kb.yml` contains `automation/github-pr-kb` + + The repo contains a copy-ready workflow that matches the locked merged-PR/manual-dispatch design, avoids cron, preserves cache/manifest rules, and updates repository state only after success. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| GitHub event + dispatch inputs → workflow job | Untrusted repo/event data and manual inputs enter the automation pipeline | +| workflow job → repository variable API | Workflow writes durable state outside git | +| workflow job → rolling PR branch | Workflow gains write access to tracked KB output | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-08-04 | S | variable-auth token selection | mitigate | Prefer GitHub App token when both app secrets are present; otherwise require explicit PAT secret, and fail closed when neither auth path is configured | +| T-08-05 | T | cursor persistence step | mitigate | Update `KB_LAST_SUCCESSFUL_CURSOR` only after extract/classify/generate and PR publication succeed, preventing state tampering that would skip unprocessed PRs | +| T-08-06 | I | git publication scope | mitigate | Stage only `kb/**/*.md` and `kb/.manifest.json`; never add `.github-pr-kb/cache/`, and rely on `.gitignore` plus explicit `git add` paths | +| T-08-07 | D | rerun/no-new-PR behavior | mitigate | Call the tested `action_state` helper before the CLI pipeline and short-circuit when no newer merged PR exists, limiting wasted API and Claude calls | + + + +Run `.venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q`. After that passes, inspect the workflow text to confirm there is no `schedule:` trigger and that `KB_LAST_SUCCESSFUL_CURSOR` is only written after the publication path. + + + +- ACTION-01 is implemented as a shipped workflow file +- ACTION-02 guard exists before extract/classify/generate and is covered by tests +- ACTION-03 persists the repository variable only on successful end-to-end publication +- `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` ships with KB markdown + + + +After completion, create `.planning/phases/08-github-action-readme/08-02-SUMMARY.md` + diff --git a/.planning/phases/08-github-action-readme/08-03-PLAN.md b/.planning/phases/08-github-action-readme/08-03-PLAN.md new file mode 100644 index 0000000..8b82065 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-03-PLAN.md @@ -0,0 +1,174 @@ +--- +phase: 08-github-action-readme +plan: 03 +type: execute +wave: 3 +depends_on: + - "08-02" +files_modified: + - README.md + - tests/test_readme.py +autonomous: true +requirements: + - INFRA-03 +must_haves: + truths: + - "README leads with automation setup for the shipped merged-PR workflow, per D-11." + - "README documents PAT quickstart first and GitHub App as the optional advanced path for repository-variable auth, per D-13 and D-14." + - "README still covers local setup, env vars, CLI commands, and KB output examples, per D-12 and INFRA-03." + - "README tells maintainers exactly what is committed (`kb/` + `kb/.manifest.json`) and what remains outside git (`.github-pr-kb/cache/`), per D-07 and D-09." + artifacts: + - path: "README.md" + provides: "Automation-first onboarding and local usage guide" + contains: "KB_VARIABLES_TOKEN" + - path: "tests/test_readme.py" + provides: "README contract checks" + contains: "test_readme_is_automation_first" + key_links: + - from: "README.md" + to: ".github/workflows/github-pr-kb.yml" + via: "documented setup steps and trigger behavior" + pattern: "github-pr-kb\\.yml|workflow_dispatch|pull_request" + - from: "README.md" + to: "src/github_pr_kb/config.py" + via: "documented env/config surface" + pattern: "ANTHROPIC_API_KEY|KB_OUTPUT_DIR|MIN_CONFIDENCE" +--- + + +Rewrite the README so a maintainer can install the shipped workflow correctly and a new user can still run the tool locally from the docs alone. + +Purpose: Close INFRA-03 with documentation that matches the implemented workflow exactly, honors automation-first ordering from D-11, and preserves local CLI onboarding from D-12. +Output: Updated `README.md` and README contract tests. + + + +@~/.copilot/get-shit-done/workflows/execute-plan.md +@~/.copilot/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/REQUIREMENTS.md +@.planning/phases/08-github-action-readme/08-CONTEXT.md +@.planning/phases/08-github-action-readme/08-RESEARCH.md +@.planning/phases/08-github-action-readme/08-VALIDATION.md +@README.md +@.env.example +@src/github_pr_kb/cli.py +@src/github_pr_kb/config.py +@.github/workflows/github-pr-kb.yml + + +From `src/github_pr_kb/config.py`: +```python +class Settings(BaseSettings): + github_token: str + anthropic_api_key: str | None = None + anthropic_model: str | None = None + kb_output_dir: str = "kb" + anthropic_generate_model: str | None = None + min_confidence: float = 0.5 +``` + +From `src/github_pr_kb/cli.py`: +```python +github-pr-kb extract --repo owner/name --state closed --since 2024-01-01 +github-pr-kb classify +github-pr-kb generate +github-pr-kb run --repo owner/name +``` + + + + + + + Task 1: Add README contract tests for workflow-first onboarding + tests/test_readme.py + + - README.md + - .github/workflows/github-pr-kb.yml + - .env.example + - src/github_pr_kb/cli.py + - src/github_pr_kb/config.py + - .planning/phases/08-github-action-readme/08-CONTEXT.md + - .planning/phases/08-github-action-readme/08-RESEARCH.md + - .planning/phases/08-github-action-readme/08-VALIDATION.md + + Create `tests/test_readme.py` with text-contract assertions for the final README. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; and it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown. + + .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q + + + - `tests/test_readme.py` exists + - file contains `test_readme_is_automation_first` + - file contains `KB_VARIABLES_TOKEN` + - file contains `KB_VARIABLES_APP_ID` + - file contains `.github-pr-kb/cache/` + + README tests pin the exact documentation promises before the README is rewritten. + + + + Task 2: Rewrite README to match the shipped workflow and local CLI reality + README.md, tests/test_readme.py + + - README.md + - tests/test_readme.py + - .github/workflows/github-pr-kb.yml + - .env.example + - src/github_pr_kb/cli.py + - src/github_pr_kb/config.py + - CLAUDE.md + - .planning/phases/08-github-action-readme/08-CONTEXT.md + - .planning/phases/08-github-action-readme/08-RESEARCH.md + + Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, required secret `ANTHROPIC_API_KEY`, PAT quickstart for repository-variable writes using `KB_VARIABLES_TOKEN`, and the optional advanced GitHub App path using `KB_VARIABLES_APP_ID` plus `KB_VARIABLES_APP_PRIVATE_KEY` per D-11/D-14. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, mention the repo's test command `.venv/Scripts/python.exe -m pytest tests/` from `CLAUDE.md`, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Explicitly state that `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. + + .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q + + + - `README.md` contains `.github/workflows/github-pr-kb.yml` + - `README.md` mentions `workflow_dispatch` + - `README.md` mentions `KB_VARIABLES_TOKEN` before `KB_VARIABLES_APP_ID` + - `README.md` contains `github-pr-kb extract --repo owner/name` + - `README.md` contains `.github-pr-kb/cache/` + - `README.md` contains `kb/.manifest.json` + + The README alone is sufficient to configure automation and run the tool locally, and every documented secret/command matches shipped code and workflow behavior. + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| README guidance → maintainer setup | Incorrect docs can cause over-privileged tokens, broken workflows, or accidental cache commits | +| README env docs → local CLI usage | Users rely on docs to populate secrets and run commands safely | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-08-08 | S | auth setup documentation | mitigate | Document PAT quickstart and GitHub App separately with exact secret names and precedence, reducing setup spoofing/confusion | +| T-08-09 | T | repo hygiene documentation | mitigate | Explicitly state that `.github-pr-kb/cache/` stays out of git and that only `kb/*.md` plus `kb/.manifest.json` are committed | +| T-08-10 | I | README examples | mitigate | Remove stale architecture claims and list only the actual env vars and commands present in `config.py`, `.env.example`, and `cli.py` | + + + +Run `.venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q`, then read the first major sections of `README.md` to confirm automation setup appears before local usage. + + + +- INFRA-03 is satisfied by an automation-first README that matches the shipped workflow +- Both auth paths are documented with PAT first and GitHub App second +- Local env vars and CLI commands in README match current code exactly + + + +After completion, create `.planning/phases/08-github-action-readme/08-03-SUMMARY.md` + From 3857a59485eb3c638b55bbcb3304a5a34ad5c782 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 00:23:52 +0300 Subject: [PATCH 04/28] fix(08): revise phase plans based on checker feedback --- .../08-github-action-readme/08-02-PLAN.md | 19 +++++++++++++++++-- .../08-github-action-readme/08-03-PLAN.md | 8 +++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.planning/phases/08-github-action-readme/08-02-PLAN.md b/.planning/phases/08-github-action-readme/08-02-PLAN.md index 07bb45f..c1ecf25 100644 --- a/.planning/phases/08-github-action-readme/08-02-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-02-PLAN.md @@ -32,6 +32,7 @@ must_haves: truths: - "The repo ships a reusable workflow triggered by merged PRs and `workflow_dispatch`, not cron, per D-01 and D-02." - "Automation restores `.github-pr-kb/cache/` from Actions cache and keeps that working data out of git, per D-07 and D-09." + - "Workflow runtime env is wired explicitly so the CLI receives `GITHUB_TOKEN`, classification receives `ANTHROPIC_API_KEY`, and `gh` commands receive authenticated `GH_TOKEN`, making the shipped automation executable." - "The workflow updates exactly one rolling KB bot PR and stages `kb/.manifest.json` with generated markdown, per D-06 and D-07." - "Repository-variable writes support PAT quickstart first and GitHub App as the advanced option with deterministic precedence, per D-13 and D-14." artifacts: @@ -53,6 +54,14 @@ must_haves: to: "src/github_pr_kb/cli.py" via: "github-pr-kb extract/classify/generate" pattern: "github-pr-kb extract|github-pr-kb classify|github-pr-kb generate" + - from: ".github/workflows/github-pr-kb.yml" + to: "src/github_pr_kb/config.py" + via: "workflow env mappings for extractor/classifier runtime" + pattern: "GITHUB_TOKEN|ANTHROPIC_API_KEY" + - from: ".github/workflows/github-pr-kb.yml" + to: "GitHub repository variables + PR APIs" + via: "gh api / gh pr authenticated with resolved GH_TOKEN" + pattern: "GH_TOKEN|gh api|gh pr" - from: ".github/workflows/github-pr-kb.yml" to: "kb/.manifest.json" via: "git add kb/.manifest.json" @@ -117,7 +126,7 @@ def decide_action_run(...) -> ActionRunDecision - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it uses `astral-sh/setup-uv` and `uv sync --all-groups --frozen` like `ci.yml`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; it runs `github-pr-kb extract --repo ${{ github.repository }} --state closed`, then `github-pr-kb classify`, then `github-pr-kb generate`; it stages `kb/.manifest.json` and KB markdown only, never `.github-pr-kb/cache`; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. + Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it uses `astral-sh/setup-uv` and `uv sync --all-groups --frozen` like `ci.yml`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; it runs `github-pr-kb extract --repo ${{ github.repository }} --state closed`, then `github-pr-kb classify`, then `github-pr-kb generate`; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it stages `kb/.manifest.json` and KB markdown only, never `.github-pr-kb/cache`; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -125,7 +134,9 @@ def decide_action_run(...) -> ActionRunDecision - `tests/test_action_workflow.py` exists - file contains `test_workflow_has_merged_pr_and_dispatch_triggers` - file contains an assertion that `schedule:` is absent + - file contains an assertion for `github.event.pull_request.merged == true` - file contains `KB_LAST_SUCCESSFUL_CURSOR` + - file contains `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, and `GH_TOKEN` - file contains `automation/github-pr-kb` Workflow tests define the exact shipped automation contract before the YAML is written. @@ -152,7 +163,7 @@ def decide_action_run(...) -> ActionRunDecision - Test 2: workflow skips before the expensive CLI pipeline when the helper reports no new merged PRs - Test 3: successful runs stage KB markdown plus `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` - Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty), determine the newest merged PR timestamp, and invoke `python -m github_pr_kb.action_state` to compute `should_run`, `extract_since`, and `next_cursor`. When `should_run` is false, exit before extract/classify/generate and print a no-new-PR reason per ACTION-02. When it is true, run `github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `github-pr-kb classify`, then `github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, commit only `kb/**/*.md` and `kb/.manifest.json`, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. + Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty), determine the newest merged PR timestamp, and invoke `python -m github_pr_kb.action_state` to compute `should_run`, `extract_since`, and `next_cursor`. When `should_run` is false, exit before extract/classify/generate and print a no-new-PR reason per ACTION-02. When it is true, run `github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `github-pr-kb classify`, then `github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, commit only `kb/**/*.md` and `kb/.manifest.json`, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -160,10 +171,14 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `workflow_dispatch:` - `.github/workflows/github-pr-kb.yml` contains `pull_request:` - `.github/workflows/github-pr-kb.yml` contains `types: [closed]` + - `.github/workflows/github-pr-kb.yml` contains `github.event.pull_request.merged == true` - `.github/workflows/github-pr-kb.yml` does not contain `schedule:` - `.github/workflows/github-pr-kb.yml` contains `.github-pr-kb/cache/` - `.github/workflows/github-pr-kb.yml` contains `kb/.manifest.json` - `.github/workflows/github-pr-kb.yml` contains `KB_LAST_SUCCESSFUL_CURSOR` + - `.github/workflows/github-pr-kb.yml` contains `GITHUB_TOKEN` + - `.github/workflows/github-pr-kb.yml` contains `ANTHROPIC_API_KEY` + - `.github/workflows/github-pr-kb.yml` contains `GH_TOKEN` - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_TOKEN` - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_APP_ID` - `.github/workflows/github-pr-kb.yml` contains `automation/github-pr-kb` diff --git a/.planning/phases/08-github-action-readme/08-03-PLAN.md b/.planning/phases/08-github-action-readme/08-03-PLAN.md index 8b82065..eb69d68 100644 --- a/.planning/phases/08-github-action-readme/08-03-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-03-PLAN.md @@ -15,7 +15,7 @@ must_haves: truths: - "README leads with automation setup for the shipped merged-PR workflow, per D-11." - "README documents PAT quickstart first and GitHub App as the optional advanced path for repository-variable auth, per D-13 and D-14." - - "README still covers local setup, env vars, CLI commands, and KB output examples, per D-12 and INFRA-03." + - "README still covers uv installation, local setup, env vars, CLI commands, and KB output examples, per D-12 and INFRA-03." - "README tells maintainers exactly what is committed (`kb/` + `kb/.manifest.json`) and what remains outside git (`.github-pr-kb/cache/`), per D-07 and D-09." artifacts: - path: "README.md" @@ -97,7 +97,7 @@ github-pr-kb run --repo owner/name - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_readme.py` with text-contract assertions for the final README. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; and it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown. + Create `tests/test_readme.py` with text-contract assertions for the final README. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; it includes an explicit `Install uv` subsection before any `uv sync --all-groups --frozen` command, includes the official docs URL `https://docs.astral.sh/uv/getting-started/installation/`, and includes at least one copy/pasteable install command so a brand-new user can start from zero per INFRA-03; it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; and it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown. .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q @@ -106,6 +106,7 @@ github-pr-kb run --repo owner/name - file contains `test_readme_is_automation_first` - file contains `KB_VARIABLES_TOKEN` - file contains `KB_VARIABLES_APP_ID` + - file contains `https://docs.astral.sh/uv/getting-started/installation/` - file contains `.github-pr-kb/cache/` README tests pin the exact documentation promises before the README is rewritten. @@ -125,7 +126,7 @@ github-pr-kb run --repo owner/name - .planning/phases/08-github-action-readme/08-CONTEXT.md - .planning/phases/08-github-action-readme/08-RESEARCH.md - Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, required secret `ANTHROPIC_API_KEY`, PAT quickstart for repository-variable writes using `KB_VARIABLES_TOKEN`, and the optional advanced GitHub App path using `KB_VARIABLES_APP_ID` plus `KB_VARIABLES_APP_PRIVATE_KEY` per D-11/D-14. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, mention the repo's test command `.venv/Scripts/python.exe -m pytest tests/` from `CLAUDE.md`, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Explicitly state that `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. + Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, required secret `ANTHROPIC_API_KEY`, PAT quickstart for repository-variable writes using `KB_VARIABLES_TOKEN`, and the optional advanced GitHub App path using `KB_VARIABLES_APP_ID` plus `KB_VARIABLES_APP_PRIVATE_KEY` per D-11/D-14. In the local setup section, add an explicit `Install uv` subsection before any sync command: include the official docs URL `https://docs.astral.sh/uv/getting-started/installation/` plus concrete install commands for Windows PowerShell (`powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`) and macOS/Linux (`curl -LsSf https://astral.sh/uv/install.sh | sh`) so a brand-new user can go from zero to a working environment using the README alone. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, mention the repo's test command `.venv/Scripts/python.exe -m pytest tests/` from `CLAUDE.md`, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Explicitly state that `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q @@ -133,6 +134,7 @@ github-pr-kb run --repo owner/name - `README.md` contains `.github/workflows/github-pr-kb.yml` - `README.md` mentions `workflow_dispatch` - `README.md` mentions `KB_VARIABLES_TOKEN` before `KB_VARIABLES_APP_ID` + - `README.md` contains `https://docs.astral.sh/uv/getting-started/installation/` - `README.md` contains `github-pr-kb extract --repo owner/name` - `README.md` contains `.github-pr-kb/cache/` - `README.md` contains `kb/.manifest.json` From 6dac5c35625d28c7999c5486e7fbb58de18e8e05 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 00:32:27 +0300 Subject: [PATCH 05/28] fix(08): revise plans based on checker feedback --- .../08-github-action-readme/08-01-PLAN.md | 33 +++++++++-------- .../08-github-action-readme/08-02-PLAN.md | 35 +++++++++++++------ .../08-github-action-readme/08-03-PLAN.md | 16 ++++++--- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/.planning/phases/08-github-action-readme/08-01-PLAN.md b/.planning/phases/08-github-action-readme/08-01-PLAN.md index 6b6dfe0..937c45e 100644 --- a/.planning/phases/08-github-action-readme/08-01-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-01-PLAN.md @@ -13,16 +13,17 @@ requirements: - ACTION-03 must_haves: truths: - - "Workflow support code decides before pipeline startup whether a merged-PR or manual run should proceed, per D-01, D-02, D-04, and D-05." - - "The automation cursor uses extractor updated_at semantics for manual recovery/backfill, per D-04." - - "A successful run can emit a deterministic next cursor, while failed runs can leave repository-variable writes to the workflow's post-success step, per D-08." + - "Workflow support code decides before pipeline startup whether a merged-PR or manual run should proceed, using separate auto/manual formulas per D-01, D-02, D-04, and D-05." + - "Auto merged-event reruns become a deterministic no-op when the event PR `updated_at` is not newer than the stored cursor, preventing self-skip/self-repeat behavior per ACTION-02." + - "The automation cursor uses extractor updated_at semantics for manual recovery/backfill, with `manual_since` overriding the stored cursor only for the manual path per D-04 and D-05." + - "A successful run can emit a deterministic next cursor without writing repository state itself, leaving repository-variable writes to the workflow's post-success step per D-08." artifacts: - path: "src/github_pr_kb/action_state.py" provides: "Testable action skip/cursor decision helper and CLI entrypoint" contains: "def decide_action_run" - path: "tests/test_action_state.py" provides: "Pytest coverage for skip/cursor/manual override logic" - contains: "test_manual_dispatch_skips_when_cursor_is_current" + contains: "test_repeated_auto_event_is_no_op_when_cursor_matches" key_links: - from: "src/github_pr_kb/action_state.py" to: "src/github_pr_kb/extractor.py" @@ -92,18 +93,19 @@ def _run_extract( - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_action_state.py` with pytest cases that lock the helper contract before code exists. Cover these exact scenarios: (1) merged PR event with `event_updated_at` newer than `stored_cursor` returns `should_run=True`, `extract_since=event_updated_at`, and `next_cursor=event_updated_at` per D-01/D-03; (2) `workflow_dispatch` with no `manual_since`, `force=False`, and `latest_merged_at <= stored_cursor` returns `should_run=False` with reason `no_new_closed_prs` per ACTION-02 and D-05; (3) `workflow_dispatch` with `manual_since` uses that value as `extract_since` and does not silently replace it with the stored cursor per D-04; (4) `force=True` bypasses the no-new-PR short-circuit but still emits the chosen `extract_since`; (5) the module's CLI mode prints JSON with keys `should_run`, `extract_since`, `next_cursor`, and `reason` so the workflow can consume it without YAML-only logic. Keep tests pure/unit-level; do not call the GitHub API. + Create `tests/test_action_state.py` with pytest cases that lock the helper contract before code exists. Cover these exact scenarios: (1) merged PR event with `event_updated_at` newer than `stored_cursor` returns `should_run=True`, `extract_since=stored_cursor` (or null when no cursor exists), and `next_cursor=event_updated_at` per D-01/D-03 so the triggering PR cannot self-skip; (2) repeated merged PR event with `event_updated_at <= stored_cursor` returns `should_run=False` with reason `already_processed_auto_event`, proving reruns/no-op behavior for ACTION-02; (3) `workflow_dispatch` with no `manual_since`, `force=False`, and `latest_merged_at <= stored_cursor` returns `should_run=False` with reason `no_new_closed_prs` per ACTION-02 and D-05; (4) `workflow_dispatch` with `manual_since` uses that value as the manual effective cursor and `extract_since`, while `next_cursor` resolves to the newer of `manual_since` and `latest_merged_at` so manual recovery never regresses state per D-04; (5) `force=True` bypasses the no-new-PR short-circuit but still emits the chosen `extract_since`; (6) the module's CLI mode prints JSON with keys `should_run`, `extract_since`, `next_cursor`, and `reason` so the workflow can consume it without YAML-only logic. Keep tests pure/unit-level; do not call the GitHub API. .venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q - `tests/test_action_state.py` exists - file contains `test_merged_pr_event_uses_event_updated_at` + - file contains `test_repeated_auto_event_is_no_op_when_cursor_matches` - file contains `test_manual_dispatch_skips_when_cursor_is_current` - file contains `test_manual_since_override_wins_over_stored_cursor` - file contains an assertion on JSON keys `should_run`, `extract_since`, `next_cursor`, and `reason` - Red tests define the exact skip/cursor contract for ACTION-02 and ACTION-03 before any helper logic is written. + Red tests define the exact auto/manual skip-cursor contract for ACTION-02 and ACTION-03 before any helper logic is written. @@ -118,22 +120,24 @@ def _run_extract( - .planning/phases/08-github-action-readme/08-RESEARCH.md - - Test 1: merged PR events emit the event PR `updated_at` timestamp as both `extract_since` and `next_cursor` - - Test 2: manual runs skip before extract/classify/generate when no merged PR is newer than the effective cursor - - Test 3: manual `since` override and `force` bypass behave deterministically + - Test 1: merged PR events compare `event_updated_at` to `stored_cursor`, use the stored cursor as `extract_since`, and emit `event_updated_at` as `next_cursor` + - Test 2: repeated auto events where `event_updated_at <= stored_cursor` become a deterministic no-op + - Test 3: manual runs skip before extract/classify/generate when no merged PR is newer than the effective manual cursor + - Test 4: manual `since` override and `force` bypass behave deterministically - Create `src/github_pr_kb/action_state.py` with a small typed contract such as `ActionRunDecision` plus a public `decide_action_run(...)` function. Normalize all incoming timestamps with timezone awareness, keep comparisons in UTC, and return ISO-8601 strings suitable for workflow env/JSON output. The function must implement these concrete rules: auto merged-PR runs use the event PR `updated_at` timestamp as the candidate cursor per D-01/D-03; manual runs use `manual_since` when supplied, otherwise the stored repository variable cursor per D-04/D-05; `should_run` is false only when `force` is false and the newest merged PR timestamp is not newer than the effective cursor; `next_cursor` is the newest successfully-targeted merged PR timestamp, but the workflow will persist it only after later steps succeed per D-08. Add a minimal `python -m github_pr_kb.action_state` entrypoint that accepts flags for `--event-name`, `--merged`, `--event-updated-at`, `--stored-cursor`, `--latest-merged-at`, `--manual-since`, and `--force`, then prints one JSON object to stdout for workflow consumption. Do not add Click or any new dependency. + Create `src/github_pr_kb/action_state.py` with a small typed contract such as `ActionRunDecision` plus a public `decide_action_run(...)` function. Normalize all incoming timestamps with timezone awareness, keep comparisons in UTC, and return ISO-8601 strings suitable for workflow env/JSON output. Make the helper mathematically explicit with two separate paths. **Auto merged-event path (per D-01/D-03):** compare only `event_updated_at` against `stored_cursor` to decide work (`should_run = merged and (stored_cursor is None or event_updated_at > stored_cursor)`); set `extract_since = stored_cursor` (or `None` if no stored cursor exists) so the triggering PR remains in extractor scope; set `next_cursor = event_updated_at` after a successful run. **Manual recovery/backfill path (per D-02/D-04/D-05):** set `effective_cursor = manual_since if manual_since is not None else stored_cursor`; compare `latest_merged_at` against that effective cursor when `force` is false; set `extract_since = effective_cursor`; set `next_cursor = max(effective_cursor, latest_merged_at)` when either value exists so manual runs never regress the saved boundary. When an auto rerun or manual no-new-PR run is a no-op, return `should_run=False` with a specific reason and leave cursor persistence to the workflow. Add a minimal `python -m github_pr_kb.action_state` entrypoint that accepts flags for `--event-name`, `--merged`, `--event-updated-at`, `--stored-cursor`, `--latest-merged-at`, `--manual-since`, and `--force`, then prints one JSON object to stdout for workflow consumption. Do not add Click or any new dependency. .venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q - `src/github_pr_kb/action_state.py` contains `def decide_action_run` - `src/github_pr_kb/action_state.py` contains `if __name__ == "__main__":` - - merged-event logic uses the event PR timestamp, not a cron/schedule path - - manual logic compares against the stored cursor or `manual_since` using `updated_at` semantics + - merged-event logic compares `event_updated_at` to `stored_cursor`, then uses `stored_cursor` as `extract_since` + - file contains explicit no-op handling for repeated auto events + - manual logic compares against the stored cursor or `manual_since` using `updated_at` semantics and computes a non-regressing `next_cursor` - stdout JSON contains exact keys `should_run`, `extract_since`, `next_cursor`, and `reason` - The repo has a reusable, pytest-covered Python helper the workflow can invoke for cost guard and cursor decisions instead of burying that logic in YAML. + The repo has a reusable, pytest-covered Python helper the workflow can invoke for cost guard and cursor decisions instead of burying that logic in YAML, and repeated auto events are guaranteed to no-op rather than self-skip or reprocess ambiguously. @@ -161,7 +165,8 @@ Run `.venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q` and co - ACTION-02 guard logic is covered by targeted pytest cases -- ACTION-03 cursor selection is deterministic and uses updated-at semantics +- ACTION-03 cursor selection is deterministic, separated for auto vs manual runs, and uses updated-at semantics +- Repeated merged-event reruns no-op cleanly when the stored cursor already matches or exceeds the event timestamp - Workflow can import and execute `python -m github_pr_kb.action_state` without adding dependencies diff --git a/.planning/phases/08-github-action-readme/08-02-PLAN.md b/.planning/phases/08-github-action-readme/08-02-PLAN.md index c1ecf25..d4327fd 100644 --- a/.planning/phases/08-github-action-readme/08-02-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-02-PLAN.md @@ -31,21 +31,27 @@ user_setup: must_haves: truths: - "The repo ships a reusable workflow triggered by merged PRs and `workflow_dispatch`, not cron, per D-01 and D-02." + - "The workflow is copy-ready for target repositories that do not contain this tool's source tree because it bootstraps the CLI from an explicit checkout of this tool repo, satisfying ACTION-01." - "Automation restores `.github-pr-kb/cache/` from Actions cache and keeps that working data out of git, per D-07 and D-09." - "Workflow runtime env is wired explicitly so the CLI receives `GITHUB_TOKEN`, classification receives `ANTHROPIC_API_KEY`, and `gh` commands receive authenticated `GH_TOKEN`, making the shipped automation executable." - "The workflow updates exactly one rolling KB bot PR and stages `kb/.manifest.json` with generated markdown, per D-06 and D-07." + - "Auto merged-event runs compare only the event PR `updated_at` against the stored cursor, while manual recovery runs use the separate manual/latest-merged formula from Plan 08-01, eliminating self-skip ambiguity for ACTION-02 and ACTION-03." - "Repository-variable writes support PAT quickstart first and GitHub App as the advanced option with deterministic precedence, per D-13 and D-14." artifacts: - path: ".github/workflows/github-pr-kb.yml" - provides: "Merged-PR + workflow_dispatch automation" - contains: "workflow_dispatch" + provides: "Merged-PR + workflow_dispatch automation with explicit tool bootstrap" + contains: "KB_TOOL_REPOSITORY" - path: ".github/workflows/github-pr-kb.yml" provides: "Cache/state/publication workflow" contains: "KB_LAST_SUCCESSFUL_CURSOR" - path: "tests/test_action_workflow.py" provides: "Workflow contract assertions" - contains: "test_workflow_has_merged_pr_and_dispatch_triggers" + contains: "test_workflow_bootstraps_tool_checkout_for_copyable_repos" key_links: + - from: ".github/workflows/github-pr-kb.yml" + to: "galzi/github-pr-kb" + via: "secondary checkout + uv project bootstrap" + pattern: "KB_TOOL_REPOSITORY|KB_TOOL_REF|actions/checkout" - from: ".github/workflows/github-pr-kb.yml" to: "src/github_pr_kb/action_state.py" via: "python -m github_pr_kb.action_state" @@ -126,16 +132,18 @@ def decide_action_run(...) -> ActionRunDecision - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it uses `astral-sh/setup-uv` and `uv sync --all-groups --frozen` like `ci.yml`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; it runs `github-pr-kb extract --repo ${{ github.repository }} --state closed`, then `github-pr-kb classify`, then `github-pr-kb generate`; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it stages `kb/.manifest.json` and KB markdown only, never `.github-pr-kb/cache`; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. + Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it bootstraps the CLI for copyable consumer repos by defining `KB_TOOL_REPOSITORY` (defaulting to this repo) and `KB_TOOL_REF`, checking out that tool repo into a dedicated directory, and installing it with `uv sync --project ... --all-groups --frozen` instead of assuming the target repo already contains this project's `pyproject.toml`; it invokes the CLI through that checkout via `uv run --project ... github-pr-kb ...`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; for auto merged events it passes `--event-updated-at` and `--stored-cursor` to the helper and does not require `latest_merged_at` to decide the auto path; for manual dispatch it passes `--latest-merged-at`, optional `--manual-since`, and `--force`; it includes an explicit repeated-auto-event/no-op guard assertion (for example matching the helper reason `already_processed_auto_event` or an equivalent skip branch) so ACTION-02 is locked; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it stages `kb/.manifest.json` and KB markdown only, never `.github-pr-kb/cache`; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q - `tests/test_action_workflow.py` exists - file contains `test_workflow_has_merged_pr_and_dispatch_triggers` + - file contains `test_workflow_bootstraps_tool_checkout_for_copyable_repos` - file contains an assertion that `schedule:` is absent - file contains an assertion for `github.event.pull_request.merged == true` - file contains `KB_LAST_SUCCESSFUL_CURSOR` + - file contains `KB_TOOL_REPOSITORY` - file contains `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, and `GH_TOKEN` - file contains `automation/github-pr-kb` @@ -160,10 +168,11 @@ def decide_action_run(...) -> ActionRunDecision - Test 1: merged PR and manual dispatch triggers are present, with no cron path - - Test 2: workflow skips before the expensive CLI pipeline when the helper reports no new merged PRs - - Test 3: successful runs stage KB markdown plus `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` + - Test 2: workflow bootstraps the CLI from a checked-out copy of this tool repo so copied workflows do not depend on the target repo containing the package + - Test 3: workflow skips before the expensive CLI pipeline when the helper reports no new merged PRs, including repeated auto events + - Test 4: successful runs stage KB markdown plus `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` - Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty), determine the newest merged PR timestamp, and invoke `python -m github_pr_kb.action_state` to compute `should_run`, `extract_since`, and `next_cursor`. When `should_run` is false, exit before extract/classify/generate and print a no-new-PR reason per ACTION-02. When it is true, run `github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `github-pr-kb classify`, then `github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, commit only `kb/**/*.md` and `kb/.manifest.json`, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. + Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Make the workflow copyable into consumer repos that do not contain this project by using one explicit bootstrap strategy: first checkout the target repository as normal, then checkout this tool repo (default `galzi/github-pr-kb`) into a dedicated directory such as `.github-pr-kb-tool` using workflow env/config keys `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`; install that checkout with `uv sync --project .github-pr-kb-tool --all-groups --frozen`; and invoke the CLI via `uv run --project .github-pr-kb-tool github-pr-kb ...` so commands run from the shipped tool checkout rather than from the consumer repo. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty). For auto merged-event runs, pass only the stored cursor plus `${{ github.event.pull_request.updated_at }}` into `python -m github_pr_kb.action_state`, and rely on the Plan 08-01 contract that compares `event_updated_at` to `stored_cursor`, emits `extract_since=stored_cursor`, and returns `next_cursor=event_updated_at`; do not let `latest_merged_at` participate in the auto decision path. For manual recovery/backfill runs, query the newest merged PR `updated_at`, pass that as `--latest-merged-at` together with optional `--manual-since` and `--force`, and rely on the separate manual formula from Plan 08-01. When `should_run` is false, exit before extract/classify/generate and print the helper reason, including repeated-auto-event no-op cases, per ACTION-02. When it is true, run `uv run --project .github-pr-kb-tool github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `uv run --project .github-pr-kb-tool github-pr-kb classify`, then `uv run --project .github-pr-kb-tool github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, commit only `kb/**/*.md` and `kb/.manifest.json`, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -173,9 +182,15 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `types: [closed]` - `.github/workflows/github-pr-kb.yml` contains `github.event.pull_request.merged == true` - `.github/workflows/github-pr-kb.yml` does not contain `schedule:` + - `.github/workflows/github-pr-kb.yml` contains `KB_TOOL_REPOSITORY` + - `.github/workflows/github-pr-kb.yml` contains `KB_TOOL_REF` + - `.github/workflows/github-pr-kb.yml` contains `uv sync --project .github-pr-kb-tool --all-groups --frozen` + - `.github/workflows/github-pr-kb.yml` contains `uv run --project .github-pr-kb-tool github-pr-kb extract` - `.github/workflows/github-pr-kb.yml` contains `.github-pr-kb/cache/` - `.github/workflows/github-pr-kb.yml` contains `kb/.manifest.json` - `.github/workflows/github-pr-kb.yml` contains `KB_LAST_SUCCESSFUL_CURSOR` + - `.github/workflows/github-pr-kb.yml` contains `--event-updated-at` + - `.github/workflows/github-pr-kb.yml` contains `already_processed_auto_event` or an equivalent explicit auto no-op branch - `.github/workflows/github-pr-kb.yml` contains `GITHUB_TOKEN` - `.github/workflows/github-pr-kb.yml` contains `ANTHROPIC_API_KEY` - `.github/workflows/github-pr-kb.yml` contains `GH_TOKEN` @@ -183,7 +198,7 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_APP_ID` - `.github/workflows/github-pr-kb.yml` contains `automation/github-pr-kb` - The repo contains a copy-ready workflow that matches the locked merged-PR/manual-dispatch design, avoids cron, preserves cache/manifest rules, and updates repository state only after success. + The repo contains a copy-ready workflow that bootstraps this tool explicitly for consumer repos, matches the locked merged-PR/manual-dispatch design, avoids cron, preserves cache/manifest rules, and updates repository state only after success. @@ -212,8 +227,8 @@ Run `.venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q`. Af -- ACTION-01 is implemented as a shipped workflow file -- ACTION-02 guard exists before extract/classify/generate and is covered by tests +- ACTION-01 is implemented as a shipped workflow file that can be copied into repos without this tool's source tree +- ACTION-02 guard exists before extract/classify/generate, uses the explicit auto/manual helper contract, and is covered by tests - ACTION-03 persists the repository variable only on successful end-to-end publication - `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` ships with KB markdown diff --git a/.planning/phases/08-github-action-readme/08-03-PLAN.md b/.planning/phases/08-github-action-readme/08-03-PLAN.md index eb69d68..f16e14c 100644 --- a/.planning/phases/08-github-action-readme/08-03-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-03-PLAN.md @@ -14,21 +14,22 @@ requirements: must_haves: truths: - "README leads with automation setup for the shipped merged-PR workflow, per D-11." + - "README explains the explicit copyable-workflow bootstrap path: the target repo checks out this tool repo and installs/runs the CLI from that checkout, satisfying ACTION-01." - "README documents PAT quickstart first and GitHub App as the optional advanced path for repository-variable auth, per D-13 and D-14." - "README still covers uv installation, local setup, env vars, CLI commands, and KB output examples, per D-12 and INFRA-03." - "README tells maintainers exactly what is committed (`kb/` + `kb/.manifest.json`) and what remains outside git (`.github-pr-kb/cache/`), per D-07 and D-09." artifacts: - path: "README.md" provides: "Automation-first onboarding and local usage guide" - contains: "KB_VARIABLES_TOKEN" + contains: "KB_TOOL_REPOSITORY" - path: "tests/test_readme.py" provides: "README contract checks" contains: "test_readme_is_automation_first" key_links: - from: "README.md" to: ".github/workflows/github-pr-kb.yml" - via: "documented setup steps and trigger behavior" - pattern: "github-pr-kb\\.yml|workflow_dispatch|pull_request" + via: "documented setup steps, trigger behavior, and tool bootstrap checkout" + pattern: "github-pr-kb\\.yml|workflow_dispatch|pull_request|KB_TOOL_REPOSITORY|KB_TOOL_REF" - from: "README.md" to: "src/github_pr_kb/config.py" via: "documented env/config surface" @@ -97,13 +98,15 @@ github-pr-kb run --repo owner/name - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_readme.py` with text-contract assertions for the final README. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; it includes an explicit `Install uv` subsection before any `uv sync --all-groups --frozen` command, includes the official docs URL `https://docs.astral.sh/uv/getting-started/installation/`, and includes at least one copy/pasteable install command so a brand-new user can start from zero per INFRA-03; it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; and it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown. + Create `tests/test_readme.py` with text-contract assertions for the final README. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; the README explains the copyable bootstrap path for consumer repositories by naming `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`, describing the second checkout of this tool repo, and making clear that the target repo does not need to contain this project's `pyproject.toml` or package source to use the workflow; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; it includes an explicit `Install uv` subsection before any `uv sync --all-groups --frozen` command, includes the official docs URL `https://docs.astral.sh/uv/getting-started/installation/`, and includes at least one copy/pasteable install command so a brand-new user can start from zero per INFRA-03; it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; and it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown. .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q - `tests/test_readme.py` exists - file contains `test_readme_is_automation_first` + - file contains `KB_TOOL_REPOSITORY` + - file contains `KB_TOOL_REF` - file contains `KB_VARIABLES_TOKEN` - file contains `KB_VARIABLES_APP_ID` - file contains `https://docs.astral.sh/uv/getting-started/installation/` @@ -126,13 +129,16 @@ github-pr-kb run --repo owner/name - .planning/phases/08-github-action-readme/08-CONTEXT.md - .planning/phases/08-github-action-readme/08-RESEARCH.md - Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, required secret `ANTHROPIC_API_KEY`, PAT quickstart for repository-variable writes using `KB_VARIABLES_TOKEN`, and the optional advanced GitHub App path using `KB_VARIABLES_APP_ID` plus `KB_VARIABLES_APP_PRIVATE_KEY` per D-11/D-14. In the local setup section, add an explicit `Install uv` subsection before any sync command: include the official docs URL `https://docs.astral.sh/uv/getting-started/installation/` plus concrete install commands for Windows PowerShell (`powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`) and macOS/Linux (`curl -LsSf https://astral.sh/uv/install.sh | sh`) so a brand-new user can go from zero to a working environment using the README alone. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, mention the repo's test command `.venv/Scripts/python.exe -m pytest tests/` from `CLAUDE.md`, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Explicitly state that `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. + Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, and the copyable bootstrap strategy from Plan 08-02: the workflow checks out this tool repo (default `galzi/github-pr-kb`) via `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`, installs it in `.github-pr-kb-tool`, and runs the CLI from that checkout so consumer repos do not need to vendor this project's source tree. Document required secret `ANTHROPIC_API_KEY`, PAT quickstart for repository-variable writes using `KB_VARIABLES_TOKEN`, and the optional advanced GitHub App path using `KB_VARIABLES_APP_ID` plus `KB_VARIABLES_APP_PRIVATE_KEY` per D-11/D-14. In the local setup section, add an explicit `Install uv` subsection before any sync command: include the official docs URL `https://docs.astral.sh/uv/getting-started/installation/` plus concrete install commands for Windows PowerShell (`powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`) and macOS/Linux (`curl -LsSf https://astral.sh/uv/install.sh | sh`) so a brand-new user can go from zero to a working environment using the README alone. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, mention the repo's test command `.venv/Scripts/python.exe -m pytest tests/` from `CLAUDE.md`, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Explicitly state that `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q - `README.md` contains `.github/workflows/github-pr-kb.yml` - `README.md` mentions `workflow_dispatch` + - `README.md` contains `KB_TOOL_REPOSITORY` + - `README.md` contains `KB_TOOL_REF` + - `README.md` explains that consumer repos do not need this tool's source tree checked in - `README.md` mentions `KB_VARIABLES_TOKEN` before `KB_VARIABLES_APP_ID` - `README.md` contains `https://docs.astral.sh/uv/getting-started/installation/` - `README.md` contains `github-pr-kb extract --repo owner/name` From 9731c785f58bd069ef5e0842ccecbb9075c43ef3 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 00:37:21 +0300 Subject: [PATCH 06/28] fix(08): revise plan 08-02 publication staging contract --- .../08-github-action-readme/08-02-PLAN.md | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.planning/phases/08-github-action-readme/08-02-PLAN.md b/.planning/phases/08-github-action-readme/08-02-PLAN.md index d4327fd..dbe4960 100644 --- a/.planning/phases/08-github-action-readme/08-02-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-02-PLAN.md @@ -34,7 +34,7 @@ must_haves: - "The workflow is copy-ready for target repositories that do not contain this tool's source tree because it bootstraps the CLI from an explicit checkout of this tool repo, satisfying ACTION-01." - "Automation restores `.github-pr-kb/cache/` from Actions cache and keeps that working data out of git, per D-07 and D-09." - "Workflow runtime env is wired explicitly so the CLI receives `GITHUB_TOKEN`, classification receives `ANTHROPIC_API_KEY`, and `gh` commands receive authenticated `GH_TOKEN`, making the shipped automation executable." - - "The workflow updates exactly one rolling KB bot PR and stages `kb/.manifest.json` with generated markdown, per D-06 and D-07." + - "The workflow updates exactly one rolling KB bot PR and stages the full generated KB publication set — `kb/INDEX.md`, category/article markdown under `kb/`, and `kb/.manifest.json` — per D-06 and D-07." - "Auto merged-event runs compare only the event PR `updated_at` against the stored cursor, while manual recovery runs use the separate manual/latest-merged formula from Plan 08-01, eliminating self-skip ambiguity for ACTION-02 and ACTION-03." - "Repository-variable writes support PAT quickstart first and GitHub App as the advanced option with deterministic precedence, per D-13 and D-14." artifacts: @@ -44,6 +44,9 @@ must_haves: - path: ".github/workflows/github-pr-kb.yml" provides: "Cache/state/publication workflow" contains: "KB_LAST_SUCCESSFUL_CURSOR" + - path: ".github/workflows/github-pr-kb.yml" + provides: "Explicit KB publication staging contract" + contains: "kb/INDEX.md" - path: "tests/test_action_workflow.py" provides: "Workflow contract assertions" contains: "test_workflow_bootstraps_tool_checkout_for_copyable_repos" @@ -72,6 +75,10 @@ must_haves: to: "kb/.manifest.json" via: "git add kb/.manifest.json" pattern: "kb/\\.manifest\\.json" + - from: ".github/workflows/github-pr-kb.yml" + to: "kb/INDEX.md" + via: "explicit git add for top-level KB index publication" + pattern: "kb/INDEX\\.md" --- @@ -132,7 +139,7 @@ def decide_action_run(...) -> ActionRunDecision - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it bootstraps the CLI for copyable consumer repos by defining `KB_TOOL_REPOSITORY` (defaulting to this repo) and `KB_TOOL_REF`, checking out that tool repo into a dedicated directory, and installing it with `uv sync --project ... --all-groups --frozen` instead of assuming the target repo already contains this project's `pyproject.toml`; it invokes the CLI through that checkout via `uv run --project ... github-pr-kb ...`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; for auto merged events it passes `--event-updated-at` and `--stored-cursor` to the helper and does not require `latest_merged_at` to decide the auto path; for manual dispatch it passes `--latest-merged-at`, optional `--manual-since`, and `--force`; it includes an explicit repeated-auto-event/no-op guard assertion (for example matching the helper reason `already_processed_auto_event` or an equivalent skip branch) so ACTION-02 is locked; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it stages `kb/.manifest.json` and KB markdown only, never `.github-pr-kb/cache`; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. + Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it bootstraps the CLI for copyable consumer repos by defining `KB_TOOL_REPOSITORY` (defaulting to this repo) and `KB_TOOL_REF`, checking out that tool repo into a dedicated directory, and installing it with `uv sync --project ... --all-groups --frozen` instead of assuming the target repo already contains this project's `pyproject.toml`; it invokes the CLI through that checkout via `uv run --project ... github-pr-kb ...`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; for auto merged events it passes `--event-updated-at` and `--stored-cursor` to the helper and does not require `latest_merged_at` to decide the auto path; for manual dispatch it passes `--latest-merged-at`, optional `--manual-since`, and `--force`; it includes an explicit repeated-auto-event/no-op guard assertion (for example matching the helper reason `already_processed_auto_event` or an equivalent skip branch) so ACTION-02 is locked; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it stages the entire generated KB publication set explicitly — `kb/INDEX.md`, `kb/**/*.md`, and `kb/.manifest.json` — and never stages `.github-pr-kb/cache`, so top-level index publication cannot depend on ambiguous glob behavior; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -145,9 +152,10 @@ def decide_action_run(...) -> ActionRunDecision - file contains `KB_LAST_SUCCESSFUL_CURSOR` - file contains `KB_TOOL_REPOSITORY` - file contains `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, and `GH_TOKEN` + - file contains `kb/INDEX.md` - file contains `automation/github-pr-kb` - Workflow tests define the exact shipped automation contract before the YAML is written. + Workflow tests define the exact shipped automation contract before the YAML is written, including explicit staging of `kb/INDEX.md` with the rest of the generated KB output. @@ -170,9 +178,9 @@ def decide_action_run(...) -> ActionRunDecision - Test 1: merged PR and manual dispatch triggers are present, with no cron path - Test 2: workflow bootstraps the CLI from a checked-out copy of this tool repo so copied workflows do not depend on the target repo containing the package - Test 3: workflow skips before the expensive CLI pipeline when the helper reports no new merged PRs, including repeated auto events - - Test 4: successful runs stage KB markdown plus `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` + - Test 4: successful runs explicitly stage `kb/INDEX.md`, KB markdown, and `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` - Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Make the workflow copyable into consumer repos that do not contain this project by using one explicit bootstrap strategy: first checkout the target repository as normal, then checkout this tool repo (default `galzi/github-pr-kb`) into a dedicated directory such as `.github-pr-kb-tool` using workflow env/config keys `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`; install that checkout with `uv sync --project .github-pr-kb-tool --all-groups --frozen`; and invoke the CLI via `uv run --project .github-pr-kb-tool github-pr-kb ...` so commands run from the shipped tool checkout rather than from the consumer repo. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty). For auto merged-event runs, pass only the stored cursor plus `${{ github.event.pull_request.updated_at }}` into `python -m github_pr_kb.action_state`, and rely on the Plan 08-01 contract that compares `event_updated_at` to `stored_cursor`, emits `extract_since=stored_cursor`, and returns `next_cursor=event_updated_at`; do not let `latest_merged_at` participate in the auto decision path. For manual recovery/backfill runs, query the newest merged PR `updated_at`, pass that as `--latest-merged-at` together with optional `--manual-since` and `--force`, and rely on the separate manual formula from Plan 08-01. When `should_run` is false, exit before extract/classify/generate and print the helper reason, including repeated-auto-event no-op cases, per ACTION-02. When it is true, run `uv run --project .github-pr-kb-tool github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `uv run --project .github-pr-kb-tool github-pr-kb classify`, then `uv run --project .github-pr-kb-tool github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, commit only `kb/**/*.md` and `kb/.manifest.json`, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. + Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Make the workflow copyable into consumer repos that do not contain this project by using one explicit bootstrap strategy: first checkout the target repository as normal, then checkout this tool repo (default `galzi/github-pr-kb`) into a dedicated directory such as `.github-pr-kb-tool` using workflow env/config keys `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`; install that checkout with `uv sync --project .github-pr-kb-tool --all-groups --frozen`; and invoke the CLI via `uv run --project .github-pr-kb-tool github-pr-kb ...` so commands run from the shipped tool checkout rather than from the consumer repo. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty). For auto merged-event runs, pass only the stored cursor plus `${{ github.event.pull_request.updated_at }}` into `python -m github_pr_kb.action_state`, and rely on the Plan 08-01 contract that compares `event_updated_at` to `stored_cursor`, emits `extract_since=stored_cursor`, and returns `next_cursor=event_updated_at`; do not let `latest_merged_at` participate in the auto decision path. For manual recovery/backfill runs, query the newest merged PR `updated_at`, pass that as `--latest-merged-at` together with optional `--manual-since` and `--force`, and rely on the separate manual formula from Plan 08-01. When `should_run` is false, exit before extract/classify/generate and print the helper reason, including repeated-auto-event no-op cases, per ACTION-02. When it is true, run `uv run --project .github-pr-kb-tool github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `uv run --project .github-pr-kb-tool github-pr-kb classify`, then `uv run --project .github-pr-kb-tool github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, stage the generated KB output explicitly with pathspecs or commands that include `kb/INDEX.md`, `kb/**/*.md`, and `kb/.manifest.json` (not just ambiguous recursive globs), verify no `.github-pr-kb/cache` files are added, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -187,6 +195,7 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `uv sync --project .github-pr-kb-tool --all-groups --frozen` - `.github/workflows/github-pr-kb.yml` contains `uv run --project .github-pr-kb-tool github-pr-kb extract` - `.github/workflows/github-pr-kb.yml` contains `.github-pr-kb/cache/` + - `.github/workflows/github-pr-kb.yml` contains `kb/INDEX.md` - `.github/workflows/github-pr-kb.yml` contains `kb/.manifest.json` - `.github/workflows/github-pr-kb.yml` contains `KB_LAST_SUCCESSFUL_CURSOR` - `.github/workflows/github-pr-kb.yml` contains `--event-updated-at` @@ -198,7 +207,7 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_APP_ID` - `.github/workflows/github-pr-kb.yml` contains `automation/github-pr-kb` - The repo contains a copy-ready workflow that bootstraps this tool explicitly for consumer repos, matches the locked merged-PR/manual-dispatch design, avoids cron, preserves cache/manifest rules, and updates repository state only after success. + The repo contains a copy-ready workflow that bootstraps this tool explicitly for consumer repos, matches the locked merged-PR/manual-dispatch design, avoids cron, preserves cache/manifest rules, explicitly stages `kb/INDEX.md` with the rest of the generated KB output, and updates repository state only after success. @@ -218,19 +227,19 @@ def decide_action_run(...) -> ActionRunDecision |-----------|----------|-----------|-------------|-----------------| | T-08-04 | S | variable-auth token selection | mitigate | Prefer GitHub App token when both app secrets are present; otherwise require explicit PAT secret, and fail closed when neither auth path is configured | | T-08-05 | T | cursor persistence step | mitigate | Update `KB_LAST_SUCCESSFUL_CURSOR` only after extract/classify/generate and PR publication succeed, preventing state tampering that would skip unprocessed PRs | -| T-08-06 | I | git publication scope | mitigate | Stage only `kb/**/*.md` and `kb/.manifest.json`; never add `.github-pr-kb/cache/`, and rely on `.gitignore` plus explicit `git add` paths | +| T-08-06 | I | git publication scope | mitigate | Stage only the explicit generated KB publication set: `kb/INDEX.md`, article markdown under `kb/`, and `kb/.manifest.json`; never add `.github-pr-kb/cache/`, and rely on `.gitignore` plus explicit `git add` paths | | T-08-07 | D | rerun/no-new-PR behavior | mitigate | Call the tested `action_state` helper before the CLI pipeline and short-circuit when no newer merged PR exists, limiting wasted API and Claude calls | -Run `.venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q`. After that passes, inspect the workflow text to confirm there is no `schedule:` trigger and that `KB_LAST_SUCCESSFUL_CURSOR` is only written after the publication path. +Run `.venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q`. After that passes, inspect the workflow text to confirm there is no `schedule:` trigger, that `kb/INDEX.md` and `kb/.manifest.json` are staged explicitly with the generated KB markdown, and that `KB_LAST_SUCCESSFUL_CURSOR` is only written after the publication path. - ACTION-01 is implemented as a shipped workflow file that can be copied into repos without this tool's source tree - ACTION-02 guard exists before extract/classify/generate, uses the explicit auto/manual helper contract, and is covered by tests - ACTION-03 persists the repository variable only on successful end-to-end publication -- `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` ships with KB markdown +- `.github-pr-kb/cache/` stays out of git while `kb/INDEX.md` and `kb/.manifest.json` ship with the generated KB markdown From c62005fdbfc7a261f966e9fbc0e4251e1ce07fd5 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 00:42:07 +0300 Subject: [PATCH 07/28] docs(08): add planning research artifacts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../08-github-action-readme/08-CONTEXT.md | 5 + .../08-github-action-readme/08-RESEARCH.md | 304 ++++++++++++++++++ .../08-github-action-readme/08-VALIDATION.md | 79 +++++ 3 files changed, 388 insertions(+) create mode 100644 .planning/phases/08-github-action-readme/08-RESEARCH.md create mode 100644 .planning/phases/08-github-action-readme/08-VALIDATION.md diff --git a/.planning/phases/08-github-action-readme/08-CONTEXT.md b/.planning/phases/08-github-action-readme/08-CONTEXT.md index 41b1b3a..b267f9b 100644 --- a/.planning/phases/08-github-action-readme/08-CONTEXT.md +++ b/.planning/phases/08-github-action-readme/08-CONTEXT.md @@ -35,10 +35,15 @@ Ship a reusable GitHub Actions workflow and an updated README so maintainers can - **D-11:** The README is automation-first: GitHub Action setup comes before local CLI usage. - **D-12:** Local setup, environment variables, command reference, and KB output examples are still included after the automation guidance so users can run and inspect the tool manually. +### State Auth Setup +- **D-13:** Repository-variable auth is dual-mode: the README documents a fine-grained PAT path first as the default quickstart, and a GitHub App path as the optional advanced setup. +- **D-14:** The workflow supports either auth path for repository-variable writes so users can choose between lower setup burden (PAT) and stronger service-account style isolation (GitHub App). + ### the agent's Discretion - Exact event syntax and guard conditions for merged-only PR triggers - Bot PR title/branch naming, commit message wording, and update mechanics - Whether manual dispatch supports explicit PR targeting, date-range backfill, or both +- Exact precedence rules when both PAT and GitHub App credentials are configured - Exact README section naming and example formatting diff --git a/.planning/phases/08-github-action-readme/08-RESEARCH.md b/.planning/phases/08-github-action-readme/08-RESEARCH.md new file mode 100644 index 0000000..8e9983d --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-RESEARCH.md @@ -0,0 +1,304 @@ +# Phase 8: GitHub Action + README - Research + +**Researched:** 2026-04-09 +**Domain:** GitHub Actions / GitHub REST API / gh CLI / uv / existing github-pr-kb CLI +**Confidence:** MEDIUM-HIGH + +## Summary + +Phase 8 should reuse the existing CLI pipeline, but **not** the `github-pr-kb run` command as-is. +`run` currently hardcodes the full pipeline, while `extract` already exposes the exact incremental +controls this phase needs: `--state`, `--since`, and `--until`. For automated runs after merge, the +workflow should drive `extract --state closed --since ` followed by `classify` and +`generate`, then publish `kb/` changes through one rolling bot PR. + +The biggest implementation constraint is state persistence. The phase context already locks the +last-successful-run cursor to a repository variable, and GitHub's current REST docs show that both +fine-grained PATs and GitHub App installation tokens can read/write repository variables when they +have **Variables** repository permission. That makes the safest product shape a **dual-mode auth** +workflow: PAT-first quickstart in the README, GitHub App as the advanced path. + +The workflow must also preserve two existing incremental mechanisms already built into the tool: +`.github-pr-kb/cache/` for extractor/classifier reuse and `kb/.manifest.json` for generator-side +dedup. The latter is especially important: if the automation commits `kb/*.md` but skips +`kb/.manifest.json`, fresh runners can generate duplicate `-2` / `-3` articles even when the KB +already contains the prior content. + +**Primary recommendation:** plan the phase around three deliverables: +1. workflow/auth/state publication mechanics, +2. automation-supporting code/tests for incremental state decisions, +3. a README rewrite that documents both automation auth paths and local usage accurately. + +--- + + +## User Constraints (from CONTEXT.md + follow-up clarification) + +### Locked Decisions + +#### Trigger Model +- **D-01:** Workflow triggers on merged PRs, not on a schedule. +- **D-02:** `workflow_dispatch` exists for manual recovery/backfill. +- **D-03:** Normal automated runs process merged/closed PRs only. + +#### Incremental Boundary & Recovery +- **D-04:** Recovery/backfill uses the extractor's `updated_at` semantics. +- **D-05:** Repository variable is a recovery/backfill cursor, not the normal trigger driver. + +#### Publication Mode +- **D-06:** Publish KB updates through one rolling bot PR. +- **D-07:** Commit generated KB output only; keep extraction/classification working data out of git. + +#### Cache & State Persistence +- **D-08:** Store the last-successful-run marker in a repository variable. +- **D-09:** Persist `.github-pr-kb/cache` outside git via Actions cache. +- **D-10:** Use workflow artifacts only for debugging, not normal persistence. + +#### README Shape +- **D-11:** README is automation-first. +- **D-12:** README still documents local setup, env vars, CLI usage, and KB examples. + +#### State Auth Setup +- **D-13:** Support both a fine-grained PAT path and a GitHub App path for repository-variable auth. +- **D-14:** PAT is the default quickstart; GitHub App is the optional advanced path. + +### The Agent's Discretion +- Exact helper surface for state/cost-guard logic (YAML-only vs small Python helper vs both) +- PR branch name, PR title, commit message, and idempotent update mechanics +- Manual dispatch inputs (date cursor, force run, target PR range, or simplified recovery toggle) +- Exact README section names and example snippets + +### Deferred Ideas (OUT OF SCOPE) +- Scheduled polling runs +- Multi-repo automation +- Publishing working cache files into git + + +--- + +## Standard Stack + +### Existing tools to reuse + +| Tool / Surface | Status | Why it matters | +|----------------|--------|----------------| +| `src/github_pr_kb/cli.py` | Existing | Reuse `extract`, `classify`, and `generate` entry points instead of inventing new automation-only commands | +| `src/github_pr_kb/extractor.py` | Existing | Already supports `state`, `since`, `until`, cache merge, and `updated_at` boundaries | +| `src/github_pr_kb/generator.py` | Existing | Already performs KB generation and manifest-based dedup | +| `.github/workflows/ci.yml` | Existing | Canonical repo pattern for checkout + setup-uv + dependency install | +| `gh` CLI / GitHub REST API | Available in Actions | Practical way to update a rolling PR and repository variable | + +### No new libraries are required + +The repo already contains the Python surfaces needed for the pipeline. The likely implementation +needs are workflow YAML, small Python support code and tests if the planner chooses that route, and +README updates. Avoid adding action-specific Python dependencies unless the final plan proves a real +need. + +--- + +## Architecture Patterns + +### Pattern 1: Use subcommands, not `github-pr-kb run` + +`github-pr-kb run` is convenient for humans, but automation needs a cost-aware boundary before the +pipeline starts. The workflow should: + +1. compute whether new merged PRs exist since the stored cursor, +2. skip the pipeline entirely if none exist, +3. otherwise run: + - `github-pr-kb extract --repo --state closed --since ` + - `github-pr-kb classify` + - `github-pr-kb generate` + +This keeps the workflow aligned with the extractor's existing `updated_at` behavior and avoids +smuggling event-specific logic into the `run` command. + +### Pattern 2: Commit `kb/.manifest.json` with `kb/*.md` + +`KBGenerator` uses `kb/.manifest.json` as its dedup source of truth. If the automation publishes the +markdown articles but not the manifest, the next fresh runner will rebuild state from disk +imperfectly and may create duplicate slug variants. Phase 8 plans should treat the manifest as part +of the published KB output. + +### Pattern 3: Repository variable stores the cursor; cache stores working data + +Use the repository variable only for the **last successful incremental boundary**. Keep +`.github-pr-kb/cache/` in Actions cache so the extractor/classifier can reuse previous work without +putting transient data in git. This cleanly separates: + +- durable workflow state: repository variable +- transient pipeline cache: Actions cache +- published artifact: `kb/` plus `kb/.manifest.json` + +### Pattern 4: Dual-mode auth for repository-variable writes + +GitHub's current REST docs for repository variables allow both: +- fine-grained PATs with **Variables: write** +- GitHub App installation tokens with **Variables: write** + +That supports a dual-mode workflow: + +1. **PAT quickstart path** + - simplest README onboarding + - one secret such as `KB_VARIABLES_TOKEN` +2. **GitHub App path** + - workflow mints an installation token from app credentials + - better long-term service-account model + +The workflow should define deterministic precedence if both are configured (for example: prefer app +credentials, otherwise fall back to PAT), and fail clearly if neither path is configured. + +### Pattern 5: Rolling PR update should be idempotent + +The publication path needs one stable branch/PR pair. Typical flow: + +1. create/update branch +2. commit KB output if files changed +3. create PR if none exists +4. update existing PR if it already exists + +This is a better fit for `gh` CLI or a purpose-built PR action than for ad hoc git-only shell +snippets, because the workflow must detect existing open PRs and avoid PR spam. + +### Pattern 6: Keep decision logic testable + +The riskiest business logic in this phase is not YAML syntax; it is the incremental decision logic: + +- has anything new arrived since the stored cursor? +- what cursor should be persisted after success? +- how do manual backfill overrides interact with the saved cursor? + +If that logic lives only in YAML/github-script, it becomes harder to test in the repo's normal +pytest suite. A small helper module or script can make Phase 8 easier to verify and less brittle. + +--- + +## Don't Hand-Roll + +| Problem | Don't build | Use instead | Why | +|---------|-------------|-------------|-----| +| Python environment bootstrapping | Custom install shell | `astral-sh/setup-uv` + `uv sync --all-groups --frozen` | Matches existing CI | +| Incremental PR filtering | Custom event-only heuristics | Existing extractor `--since` / `updated_at` behavior | Keeps semantics consistent with cache layer | +| KB dedup reconstruction | Markdown-only dedup guesswork | `kb/.manifest.json` | Existing generator contract | +| Repo variable CRUD | Hand-maintained state file in git | GitHub Actions repository variables API | Matches locked phase decision | + +--- + +## Runtime State Inventory + +### Existing mutable state + +| State | Location | Owner | Notes | +|-------|----------|-------|-------| +| Extract/classify cache | `.github-pr-kb/cache/` | local runner / Actions cache | ignored by git; safe to persist via cache | +| KB output | `kb/` | git-tracked output | published result | +| KB manifest | `kb/.manifest.json` | generator | must be published with KB files | +| Workflow cursor | repository variable | GitHub repo settings | new Phase 8 automation state | + +### State transitions Phase 8 must preserve + +1. no-new-PR run -> do not call extract/classify/generate -> do not advance cursor incorrectly +2. successful incremental run -> update KB + persist new cursor +3. failed run -> do not advance success cursor +4. manual backfill/recovery -> allow explicit override without corrupting normal automation state + +--- + +## Environment Availability + +| Dependency | Required By | Available | Notes | +|------------|------------|-----------|-------| +| `astral-sh/setup-uv` | workflow setup | Yes | already used in `.github/workflows/ci.yml` | +| GitHub-hosted Ubuntu runner | workflow | Yes | repo already uses `ubuntu-latest` | +| `gh` CLI | PR / API automation | Likely available on hosted runners | still validate in plan execution | +| `.venv/Scripts/python.exe -m pytest` | local validation | Yes | project instruction requires venv Python directly | + +No missing local dependencies were found in the repo itself. The external setup burden will be in +GitHub credentials and repo settings, which README must explain clearly. + +--- + +## Validation Architecture + +> `workflow.nyquist_validation` is enabled in `.planning/config.json`, so this section is required. + +### Test Framework + +| Property | Value | +|----------|-------| +| Framework | pytest (existing) | +| Config file | `pyproject.toml` | +| Quick run command | `.venv/Scripts/python.exe -m pytest tests/test_cli.py tests/test_extractor.py tests/test_generator.py -x -q` | +| Full suite command | `.venv/Scripts/python.exe -m pytest tests/ -q` | + +### Phase Requirements -> Verification Map + +| Req ID | Behavior | Test Type | Automated Command | File Exists? | +|--------|----------|-----------|-------------------|-------------| +| ACTION-01 | Workflow file exists, is wired for merged PR + manual dispatch, and documents required inputs/secrets | unit / file assertion | `.venv/Scripts/python.exe -m pytest tests/ -k "workflow or action" -q` | Likely Wave 0 | +| ACTION-02 | No-new-PR runs short-circuit before extract/classify/generate | unit | `.venv/Scripts/python.exe -m pytest tests/ -k "no_new_pr or cost_guard" -q` | Likely Wave 0 | +| ACTION-03 | Successful run persists repository-variable cursor correctly; failed run does not advance it | unit | `.venv/Scripts/python.exe -m pytest tests/ -k "cursor or state" -q` | Likely Wave 0 | +| INFRA-03 | README documents setup, env vars, CLI usage, and KB example | unit / doc assertion | `.venv/Scripts/python.exe -m pytest tests/ -k "readme" -q` | Likely Wave 0 | + +### Sampling Rate + +- **Per task commit:** `.venv/Scripts/python.exe -m pytest tests/ -k "workflow or action or readme or cursor" -q` +- **Per wave merge:** `.venv/Scripts/python.exe -m pytest tests/ -q` +- **Phase gate:** full suite green before verification + +### Wave 0 Gaps + +- [ ] Workflow-focused tests for trigger/auth/state decision logic +- [ ] README contract tests if docs assertions are not already covered + +Existing pytest infrastructure is sufficient; Phase 8 should add targeted tests rather than new test +tooling. + +--- + +## Common Pitfalls + +### Pitfall 1: Advancing the cursor on failure +If the workflow updates the repository variable before the pipeline fully succeeds, failed runs can +permanently skip PRs. Persist the cursor only after extract/classify/generate and PR publication +finish successfully. + +### Pitfall 2: Treating post-merge event time as the extraction boundary +The extractor already uses `pr.updated_at`. If the workflow instead uses merge event timestamps or +raw PR numbers as the incremental boundary, manual recovery and normal automation can drift apart. + +### Pitfall 3: Publishing KB markdown without the manifest +This silently breaks dedup expectations on fresh runners and leads to duplicate article slugging on +later runs. + +### Pitfall 4: Supporting both PAT and GitHub App without deterministic precedence +If both auth methods are configured and the workflow does not define which wins, troubleshooting +becomes unpredictable. The plan should choose a single precedence rule and document it. + +### Pitfall 5: Hiding too much logic in YAML +YAML is fine for orchestration, but state/cost-guard logic becomes hard to test and reason about if +it is embedded only in long shell/github-script blocks. + +--- + +## Likely File Touch Points + +- `.github/workflows/` — add the reusable KB automation workflow +- `README.md` — rewrite for automation-first onboarding and dual auth setup +- `src/github_pr_kb/cli.py` or a new helper module — only if planner chooses to extract reusable automation logic from YAML +- `src/github_pr_kb/extractor.py` — only if small supporting hooks are needed for cost/state alignment +- `tests/` — add workflow/state/doc contract coverage + +--- + +## Recommendation to Planner + +Split Phase 8 into at least two plans: + +1. **Automation mechanics plan** — workflow trigger/auth/cursor/cache/bot-PR behavior, plus any + helper code and tests needed to make the decision logic reliable. +2. **Documentation plan** — README rewrite that reflects the final shipped workflow, PAT quickstart, + GitHub App alternative, local CLI usage, env vars, and KB example output. + +If the workflow logic becomes too dense, a third plan for state/cost-guard helper code is warranted. diff --git a/.planning/phases/08-github-action-readme/08-VALIDATION.md b/.planning/phases/08-github-action-readme/08-VALIDATION.md new file mode 100644 index 0000000..6508083 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-VALIDATION.md @@ -0,0 +1,79 @@ +--- +phase: 8 +slug: github-action-readme +status: draft +nyquist_compliant: false +wave_0_complete: false +created: 2026-04-09 +--- + +# Phase 8 — Validation Strategy + +> Per-phase validation contract for feedback sampling during execution. + +--- + +## Test Infrastructure + +| Property | Value | +|----------|-------| +| **Framework** | pytest (existing) | +| **Config file** | `pyproject.toml` | +| **Quick run command** | `.venv/Scripts/python.exe -m pytest tests/ -k "workflow or action or readme or cursor" -q` | +| **Full suite command** | `.venv/Scripts/python.exe -m pytest tests/ -q` | +| **Estimated runtime** | ~10 seconds | + +--- + +## Sampling Rate + +- **After every task commit:** Run `.venv/Scripts/python.exe -m pytest tests/ -k "workflow or action or readme or cursor" -q` +- **After every plan wave:** Run `.venv/Scripts/python.exe -m pytest tests/ -q` +- **Before `/gsd-verify-work`:** Full suite must be green +- **Max feedback latency:** 10 seconds + +--- + +## Per-Task Verification Map + +| Task ID | Plan | Wave | Requirement | Test Type | Automated Command | File Exists | Status | +|---------|------|------|-------------|-----------|-------------------|-------------|--------| +| 08-01-01 | 01 | 0 | ACTION-01 | unit / file assertion | `.venv/Scripts/python.exe -m pytest tests/ -k "workflow and trigger" -q` | Likely W0 | pending | +| 08-01-02 | 01 | 0 | ACTION-02 | unit | `.venv/Scripts/python.exe -m pytest tests/ -k "no_new_pr or cost_guard" -q` | Likely W0 | pending | +| 08-01-03 | 01 | 0 | ACTION-03 | unit | `.venv/Scripts/python.exe -m pytest tests/ -k "cursor or state" -q` | Likely W0 | pending | +| 08-01-04 | 01 | 0 | ACTION-01/ACTION-03 | integration-lite | `.venv/Scripts/python.exe -m pytest tests/ -k "cache or manifest" -q` | Likely W0 | pending | +| 08-02-01 | 02 | 1 | INFRA-03 | doc contract | `.venv/Scripts/python.exe -m pytest tests/ -k "readme" -q` | Likely W0 | pending | + +*Status: pending / green / red / flaky* + +--- + +## Wave 0 Requirements + +- [ ] Workflow/state decision tests for merged-only trigger, cursor handling, and cost guard +- [ ] README contract coverage if docs expectations are not already asserted elsewhere + +*Existing infrastructure covers framework and fixtures. Phase 8 should add tests, not new tooling.* + +--- + +## Manual-Only Verifications + +| Behavior | Requirement | Why Manual | Test Instructions | +|----------|-------------|------------|-------------------| +| Rolling bot PR is created once and updated on subsequent successful runs | ACTION-01 | Requires live GitHub PR behavior | Run `workflow_dispatch` in a test repo twice and confirm one open KB PR is updated rather than duplicated | +| No-new-PR run skips extract/classify/generate work in a real repository | ACTION-02 | Requires live workflow/event context | Trigger the workflow with the saved cursor newer than all merged PRs and confirm the pipeline exits without KB changes | +| Dual auth setup docs are understandable for maintainers | INFRA-03 | Human readability / setup burden | Follow the README once with PAT path and once with GitHub App path in a test repo | + +--- + +## Validation Sign-Off + +- [ ] All tasks have `` verify or Wave 0 dependencies +- [ ] Sampling continuity: no 3 consecutive tasks without automated verify +- [ ] Wave 0 covers all MISSING references +- [ ] No watch-mode flags +- [ ] Feedback latency < 10s +- [ ] `nyquist_compliant: true` set in frontmatter + +**Approval:** pending From 61276f03d9c952c8f9e5e16288178b92ea430b2d Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 09:49:29 +0300 Subject: [PATCH 08/28] Tighten phase 8 plan docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../08-github-action-readme/08-01-PLAN.md | 54 ++++++--- .../08-01-RISK-REVIEW.md | 70 ++++++++++++ .../08-github-action-readme/08-02-PLAN.md | 53 +++++++-- .../08-02-RISK-REVIEW.md | 105 ++++++++++++++++++ .../08-github-action-readme/08-03-PLAN.md | 51 ++++++--- .../08-03-RISK-REVIEW.md | 99 +++++++++++++++++ 6 files changed, 390 insertions(+), 42 deletions(-) create mode 100644 .planning/phases/08-github-action-readme/08-01-RISK-REVIEW.md create mode 100644 .planning/phases/08-github-action-readme/08-02-RISK-REVIEW.md create mode 100644 .planning/phases/08-github-action-readme/08-03-RISK-REVIEW.md diff --git a/.planning/phases/08-github-action-readme/08-01-PLAN.md b/.planning/phases/08-github-action-readme/08-01-PLAN.md index 937c45e..1bfc04d 100644 --- a/.planning/phases/08-github-action-readme/08-01-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-01-PLAN.md @@ -15,8 +15,9 @@ must_haves: truths: - "Workflow support code decides before pipeline startup whether a merged-PR or manual run should proceed, using separate auto/manual formulas per D-01, D-02, D-04, and D-05." - "Auto merged-event reruns become a deterministic no-op when the event PR `updated_at` is not newer than the stored cursor, preventing self-skip/self-repeat behavior per ACTION-02." - - "The automation cursor uses extractor updated_at semantics for manual recovery/backfill, with `manual_since` overriding the stored cursor only for the manual path per D-04 and D-05." - - "A successful run can emit a deterministic next cursor without writing repository state itself, leaving repository-variable writes to the workflow's post-success step per D-08." + - "The automation cursor uses extractor `updated_at` semantics end-to-end: manual recovery/backfill may override the extract boundary with `manual_since`, but the helper's manual `next_cursor` must be `max(stored_cursor, manual_since, latest_merged_at)` so successful backfills never regress durable state per D-04 and D-05." + - "The helper remains import-safe and config-free so `python -m github_pr_kb.action_state` works without repository/API secrets or unrelated settings initialization." + - "A successful run emits a deterministic candidate next cursor without writing repository state itself; Plan 08-02 must persist that value monotonically against the current repository variable in the workflow's post-success step per D-08." artifacts: - path: "src/github_pr_kb/action_state.py" provides: "Testable action skip/cursor decision helper and CLI entrypoint" @@ -38,7 +39,7 @@ must_haves: Create the testable automation decision contract the workflow will rely on for cost-aware skipping and cursor handling. -Purpose: Keep state/cost-guard logic out of opaque YAML so ACTION-02 and ACTION-03 are enforced by pytest-covered Python, while preserving the locked merged-PR/manual-backfill behavior from D-01 through D-05 and D-08. +Purpose: Keep state/cost-guard logic out of opaque YAML so ACTION-02 and ACTION-03 are enforced by pytest-covered Python, while preserving the locked merged-PR/manual-backfill behavior from D-01 through D-05 and D-08, preventing manual cursor regression, and making the downstream workflow contract explicit for timestamp sourcing and monotonic persistence. Output: `src/github_pr_kb/action_state.py` plus focused pytest coverage. @@ -78,6 +79,11 @@ def _run_extract( until_dt: datetime | None, ) -> str: ``` + +Workflow contract for Plan 08-02: +- `event_updated_at` must come from `github.event.pull_request.updated_at` +- `latest_merged_at` must come from the newest merged PR `updated_at`, not merge-event timing metadata +- all incoming timestamps must be timezone-aware ISO-8601 values that match extractor `updated_at` semantics @@ -93,7 +99,7 @@ def _run_extract( - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_action_state.py` with pytest cases that lock the helper contract before code exists. Cover these exact scenarios: (1) merged PR event with `event_updated_at` newer than `stored_cursor` returns `should_run=True`, `extract_since=stored_cursor` (or null when no cursor exists), and `next_cursor=event_updated_at` per D-01/D-03 so the triggering PR cannot self-skip; (2) repeated merged PR event with `event_updated_at <= stored_cursor` returns `should_run=False` with reason `already_processed_auto_event`, proving reruns/no-op behavior for ACTION-02; (3) `workflow_dispatch` with no `manual_since`, `force=False`, and `latest_merged_at <= stored_cursor` returns `should_run=False` with reason `no_new_closed_prs` per ACTION-02 and D-05; (4) `workflow_dispatch` with `manual_since` uses that value as the manual effective cursor and `extract_since`, while `next_cursor` resolves to the newer of `manual_since` and `latest_merged_at` so manual recovery never regresses state per D-04; (5) `force=True` bypasses the no-new-PR short-circuit but still emits the chosen `extract_since`; (6) the module's CLI mode prints JSON with keys `should_run`, `extract_since`, `next_cursor`, and `reason` so the workflow can consume it without YAML-only logic. Keep tests pure/unit-level; do not call the GitHub API. + Create `tests/test_action_state.py` with pytest cases that lock the helper contract before code exists. Cover these exact scenarios: (1) merged PR event with `event_updated_at` newer than `stored_cursor` returns `should_run=True`, `extract_since=stored_cursor` (or JSON `null` when no cursor exists), and `next_cursor=event_updated_at` per D-01/D-03 so the triggering PR cannot self-skip; (2) repeated merged PR event with `event_updated_at <= stored_cursor` returns `should_run=False` with reason `already_processed_auto_event`, proving reruns/no-op behavior for ACTION-02; (3) `workflow_dispatch` with no `manual_since`, `force=False`, and `latest_merged_at <= stored_cursor` returns `should_run=False` with reason `no_new_closed_prs` per ACTION-02 and D-05; (4) `workflow_dispatch` with `manual_since` uses that value as the manual effective cursor and `extract_since`, while `next_cursor` resolves to `max(stored_cursor, manual_since, latest_merged_at)` so manual recovery/backfill can never emit a regressing cursor; (5) when `stored_cursor` is newer than both `manual_since` and `latest_merged_at`, the manual path may still use `manual_since` as the extract boundary but must keep `next_cursor=stored_cursor`; (6) `force=True` bypasses the no-new-PR short-circuit but still emits the chosen `extract_since` and non-regressing `next_cursor`; (7) the module's CLI mode prints JSON with keys `should_run`, `extract_since`, `next_cursor`, and `reason`, encodes absent timestamps as JSON `null`, and runs successfully in an isolated environment without repo/API secrets; (8) malformed or timezone-naive timestamp input causes a deterministic non-zero CLI exit with a stable error message on stderr rather than a success-shaped JSON payload. Keep tests pure/unit-level; do not call the GitHub API. .venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q @@ -103,9 +109,12 @@ def _run_extract( - file contains `test_repeated_auto_event_is_no_op_when_cursor_matches` - file contains `test_manual_dispatch_skips_when_cursor_is_current` - file contains `test_manual_since_override_wins_over_stored_cursor` + - file contains `test_manual_next_cursor_never_regresses_past_stored_cursor` + - file contains `test_action_state_cli_runs_without_repo_or_api_secrets` + - file contains `test_action_state_cli_rejects_malformed_timestamp_input` - file contains an assertion on JSON keys `should_run`, `extract_since`, `next_cursor`, and `reason` - Red tests define the exact auto/manual skip-cursor contract for ACTION-02 and ACTION-03 before any helper logic is written. + Red tests define the exact auto/manual skip-cursor contract for ACTION-02 and ACTION-03 before any helper logic is written, including non-regressing manual state, import-safe CLI execution, and deterministic invalid-input behavior. @@ -123,9 +132,9 @@ def _run_extract( - Test 1: merged PR events compare `event_updated_at` to `stored_cursor`, use the stored cursor as `extract_since`, and emit `event_updated_at` as `next_cursor` - Test 2: repeated auto events where `event_updated_at <= stored_cursor` become a deterministic no-op - Test 3: manual runs skip before extract/classify/generate when no merged PR is newer than the effective manual cursor - - Test 4: manual `since` override and `force` bypass behave deterministically + - Test 4: manual `since` override, stored-cursor preservation, and `force` bypass behave deterministically - Create `src/github_pr_kb/action_state.py` with a small typed contract such as `ActionRunDecision` plus a public `decide_action_run(...)` function. Normalize all incoming timestamps with timezone awareness, keep comparisons in UTC, and return ISO-8601 strings suitable for workflow env/JSON output. Make the helper mathematically explicit with two separate paths. **Auto merged-event path (per D-01/D-03):** compare only `event_updated_at` against `stored_cursor` to decide work (`should_run = merged and (stored_cursor is None or event_updated_at > stored_cursor)`); set `extract_since = stored_cursor` (or `None` if no stored cursor exists) so the triggering PR remains in extractor scope; set `next_cursor = event_updated_at` after a successful run. **Manual recovery/backfill path (per D-02/D-04/D-05):** set `effective_cursor = manual_since if manual_since is not None else stored_cursor`; compare `latest_merged_at` against that effective cursor when `force` is false; set `extract_since = effective_cursor`; set `next_cursor = max(effective_cursor, latest_merged_at)` when either value exists so manual runs never regress the saved boundary. When an auto rerun or manual no-new-PR run is a no-op, return `should_run=False` with a specific reason and leave cursor persistence to the workflow. Add a minimal `python -m github_pr_kb.action_state` entrypoint that accepts flags for `--event-name`, `--merged`, `--event-updated-at`, `--stored-cursor`, `--latest-merged-at`, `--manual-since`, and `--force`, then prints one JSON object to stdout for workflow consumption. Do not add Click or any new dependency. + Create `src/github_pr_kb/action_state.py` with a small typed contract such as `ActionRunDecision` plus a public `decide_action_run(...)` function. Keep the module standalone and stdlib-only: do not import `config.settings`, `cli`, or other env-bound surfaces that can fail before arg parsing. Normalize all incoming timestamps with timezone awareness, reject malformed or timezone-naive values with deterministic validation errors, keep comparisons in UTC, and return ISO-8601 strings suitable for workflow env/JSON output. Make the helper mathematically explicit with two separate paths and an explicit workflow timestamp contract: `event_updated_at` is sourced from `github.event.pull_request.updated_at`, while `latest_merged_at` is sourced from the newest merged PR `updated_at`; the helper should not accept merge-event timestamps with different semantics. **Auto merged-event path (per D-01/D-03):** compare only `event_updated_at` against `stored_cursor` to decide work (`should_run = merged and (stored_cursor is None or event_updated_at > stored_cursor)`); set `extract_since = stored_cursor` (or `None` if no stored cursor exists) so the triggering PR remains in extractor scope; set `next_cursor = event_updated_at` after a successful run. **Manual recovery/backfill path (per D-02/D-04/D-05):** set `effective_cursor = manual_since if manual_since is not None else stored_cursor`; compare `latest_merged_at` against that effective cursor when `force` is false; set `extract_since = effective_cursor`; set `next_cursor = max(stored_cursor, manual_since, latest_merged_at)` across all present values so manual runs can widen extraction scope without ever lowering the durable cursor candidate. When an auto rerun or manual no-new-PR run is a no-op, return `should_run=False` with a specific reason and leave cursor persistence to the workflow. In the module docstring or public function docs, state clearly that `next_cursor` is a candidate value only and that Plan 08-02 must persist it monotonically against the latest repository-variable value while also using workflow concurrency to prevent out-of-order merged-event writes. Add a minimal `python -m github_pr_kb.action_state` entrypoint that accepts flags for `--event-name`, `--merged`, `--event-updated-at`, `--stored-cursor`, `--latest-merged-at`, `--manual-since`, and `--force`, prints one JSON object to stdout for workflow consumption on success, and exits non-zero with a deterministic stderr message on invalid input. Do not add Click or any new dependency. .venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q @@ -134,14 +143,26 @@ def _run_extract( - `src/github_pr_kb/action_state.py` contains `if __name__ == "__main__":` - merged-event logic compares `event_updated_at` to `stored_cursor`, then uses `stored_cursor` as `extract_since` - file contains explicit no-op handling for repeated auto events - - manual logic compares against the stored cursor or `manual_since` using `updated_at` semantics and computes a non-regressing `next_cursor` + - manual logic compares against the stored cursor or `manual_since` using `updated_at` semantics and computes `next_cursor = max(stored_cursor, manual_since, latest_merged_at)` across present values + - file documents that `next_cursor` is a candidate value and that workflow persistence must be monotonic in Plan 08-02 + - file stays import-safe without `config.settings` or other env-dependent imports - stdout JSON contains exact keys `should_run`, `extract_since`, `next_cursor`, and `reason` + - malformed or timezone-naive CLI input exits non-zero with a deterministic stderr error - The repo has a reusable, pytest-covered Python helper the workflow can invoke for cost guard and cursor decisions instead of burying that logic in YAML, and repeated auto events are guaranteed to no-op rather than self-skip or reprocess ambiguously. + The repo has a reusable, pytest-covered Python helper the workflow can invoke for cost guard and cursor decisions instead of burying that logic in YAML, repeated auto events are guaranteed to no-op rather than self-skip or reprocess ambiguously, and manual recovery can never emit a regressing durable cursor candidate. + +## Downstream workflow requirements for Plan 08-02 + +1. Source `event_updated_at` from `github.event.pull_request.updated_at`. +2. Source `latest_merged_at` from the newest merged PR `updated_at` field so helper inputs match extractor semantics. +3. Persist `next_cursor` monotonically against the freshest repository-variable value after success, not just against the value read at job start. +4. Add workflow-level concurrency and one workflow-consumer smoke test so overlapping merged-event runs and JSON/null handling are verified outside the pure unit layer. + + ## Trust Boundaries @@ -154,20 +175,21 @@ def _run_extract( | Threat ID | Category | Component | Disposition | Mitigation Plan | |-----------|----------|-----------|-------------|-----------------| -| T-08-01 | T | `src/github_pr_kb/action_state.py` timestamp parsing | mitigate | Normalize/validate ISO timestamps and reject malformed values with deterministic errors instead of silently coercing bad cursor data | -| T-08-02 | D | manual dispatch skip logic | mitigate | Unit-test the no-new-PR short-circuit and `force` bypass so malformed guard logic cannot trigger unnecessary extract/classify/generate runs | -| T-08-03 | E | workflow helper surface | accept | Helper emits decision JSON only; it does not write repo state or call external APIs, so privilege impact is low | +| T-08-01 | T | `src/github_pr_kb/action_state.py` timestamp parsing and input contract | mitigate | Normalize/validate timezone-aware ISO timestamps, reject malformed or naive values with deterministic CLI failures, and lock the exact `updated_at` source fields the workflow must pass | +| T-08-02 | D | manual dispatch skip/cursor logic | mitigate | Unit-test the no-new-PR short-circuit, `force` bypass, and stored-cursor-dominates `next_cursor` behavior so manual backfills can expand extraction scope without regressing durable state | +| T-08-03 | R | helper output → workflow persistence handoff | mitigate | Treat `next_cursor` as a candidate value only; require Plan 08-02 to add concurrency plus monotonic post-success writes against the latest stored cursor so overlapping runs cannot move state backward | -Run `.venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q` and confirm the new helper emits the expected JSON contract for workflow consumption. +Run `.venv/Scripts/python.exe -m pytest tests/test_action_state.py -x -q` and confirm the helper covers the success-path JSON contract, isolated-env CLI execution, and deterministic invalid-input failure behavior. -- ACTION-02 guard logic is covered by targeted pytest cases -- ACTION-03 cursor selection is deterministic, separated for auto vs manual runs, and uses updated-at semantics +- ACTION-02 guard logic is covered by targeted pytest cases, including repeated auto-event no-ops and deterministic invalid-input handling +- ACTION-03 cursor selection is deterministic, separated for auto vs manual runs, uses `updated_at` semantics, and manual runs cannot emit a cursor older than the current stored state - Repeated merged-event reruns no-op cleanly when the stored cursor already matches or exceeds the event timestamp -- Workflow can import and execute `python -m github_pr_kb.action_state` without adding dependencies +- Workflow can import and execute `python -m github_pr_kb.action_state` without adding dependencies or unrelated env/config requirements +- Plan 08-02 receives an explicit handoff contract for timestamp sourcing, monotonic persistence, concurrency, and a workflow-consumer smoke test diff --git a/.planning/phases/08-github-action-readme/08-01-RISK-REVIEW.md b/.planning/phases/08-github-action-readme/08-01-RISK-REVIEW.md new file mode 100644 index 0000000..c5145b4 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-01-RISK-REVIEW.md @@ -0,0 +1,70 @@ +# Risk Review: 08-01-PLAN.md + +## Verdict + +**Not ready as written.** The core idea is good, but the current manual cursor formula appears to contradict the plan's own non-regression goal, and concurrency plus timestamp-source risks are not specified tightly enough yet. + +**Overall risk level:** High + +## Plan Summary + +The plan proposes a small Python helper, `src\github_pr_kb\action_state.py`, plus focused unit tests in `tests\test_action_state.py`, to decide whether the GitHub Action should run, which `extract_since` value to use, and which `next_cursor` value to emit for auto merged-PR runs and manual recovery runs. + +The intended benefit is to move skip/cursor logic out of YAML and into pytest-covered Python while leaving repository-variable writes to a later workflow step. + +## Assumptions and Evidence + +| ID | Assumption | Explicit / Implicit | Justification status | Blast radius if wrong | Early validation | +|---|---|---|---|---|---| +| A1 | Workflow can supply `event_updated_at` and `latest_merged_at` with the same `updated_at` semantics used by the extractor. | Implicit | Partial | High | Lock exact source fields in the workflow contract. | +| A2 | `next_cursor = max(manual_since, latest_merged_at)` is non-regressing for manual runs. | Explicit | Unjustified / contradicted | Critical | Add a test where `stored_cursor` is newer than both values. | +| A3 | Auto `next_cursor = event_updated_at` is safe under burst merges, reruns, and out-of-order completion. | Implicit | Weak | High | Model concurrent merged events against the same stored cursor. | +| A4 | JSON keys alone are a sufficient workflow contract. | Explicit | Partial | Medium | Add malformed-input, null, and exit-code contract tests. | +| A5 | `python -m github_pr_kb.action_state` can run without config/env side effects. | Implicit | Weak | Medium | Run it in a minimal environment with only its own flags. | +| A6 | Unit tests alone are enough for this plan stage. | Explicit | Mostly reasonable but incomplete | Medium | Require a workflow-consumer smoke test in 08-02. | + +## Devil's Advocacy + +### Inversion + +1. Moving logic into Python may reduce YAML complexity while still increasing overall system risk if the real contract is split across Python decision code and underspecified workflow timestamp sourcing. +2. Using `event_updated_at` as the auto cursor may prevent self-skip for one run while still allowing cursor lag or regression when merged-event runs overlap. +3. Manual override may improve recovery ergonomics while also making it easier to corrupt durable state if the emitted cursor can move backward. + +### Outsider questions + +- Which timestamp is authoritative: event payload time, latest merged PR time, or stored cursor? +- What happens when two merge-triggered runs overlap and finish out of order? +- What prevents a bad `workflow_dispatch` input from regressing or freezing automation state? + +### Failure-of-imagination scenarios + +- A successful manual backfill with an old `manual_since` lowers the durable cursor and causes old PRs to be reprocessed. +- Two PRs merge close together, both runs read the same stored cursor, and the older event finishes last and writes the smaller cursor. +- The workflow computes `latest_merged_at` from a field that does not match extractor `updated_at` semantics, so helper decisions are internally consistent but operationally wrong. + +## Risk Register + +| Risk ID | Category | Description | Trigger | Probability | Severity | Priority | Detection | Mitigation | Contingency | Assumption | +|---|---|---|---|---|---|---|---|---|---|---| +| R1 | Operational | Manual backfill can regress the saved cursor because the proposed manual formula ignores a newer `stored_cursor`. | `manual_since` is older than `stored_cursor` and no newer `latest_merged_at` exists. | High | Critical | Critical | Stored cursor decreases after manual success. | Compute manual `next_cursor` as `max(stored_cursor, manual_since, latest_merged_at)` or enforce monotonic writes in workflow. | Repair the repo variable and rerun recovery. | A2 | +| R2 | Operational | Concurrent or out-of-order merged-event runs can regress or lag the cursor because auto `next_cursor` is only the event time. | Multiple merges occur close together and workflows overlap. | Medium | High | High | Cursor moves backward or repeated runs cover the same window. | Add workflow concurrency and monotonic cursor persistence. | Re-run with corrected cursor. | A3 | +| R3 | Technical | Helper decisions can drift from extractor behavior if workflow timestamps use different semantics than extractor `updated_at`. | Workflow uses a different field or timezone treatment. | Medium | High | High | No-op decisions when new PRs exist, or empty extracts after `should_run=true`. | Define exact upstream source fields and add a downstream contract test in 08-02. | Manual backfill while fixing the mapping. | A1 | +| R4 | Technical | Malformed, naive, or future timestamps may produce brittle behavior because tests do not yet lock error/output handling. | Bad workflow input or bad manual dispatch value. | Medium | Medium | Medium | Traceback-like failures or implausible cursor output. | Add explicit malformed-input and normalization tests. | Fail closed and avoid cursor persistence on parse errors. | A4 | +| R5 | Organizational | The helper may accidentally import env-bound config surfaces and fail in contexts that do not need repo/API secrets. | `action_state.py` imports modules that instantiate settings at import time. | Medium | Medium | Medium | `python -m github_pr_kb.action_state` fails before arg parsing. | Keep the module standalone and config-free. | Refactor helper imports. | A5 | +| R6 | Technical | Unit tests can pass while the workflow consumer still breaks on JSON parsing, null handling, or shell quoting. | Later YAML consumer interprets helper output differently. | Medium | Medium | Medium | Workflow step fails despite green unit tests. | Add one workflow-consumer smoke test in 08-02. | Patch the workflow consumer. | A6 | + +## Recommended Changes Before Implementation + +1. Fix the manual cursor formula so successful manual runs cannot move durable state backward. +2. Add a test where `stored_cursor` is newer than both `manual_since` and `latest_merged_at`. +3. Treat cursor persistence as its own safety contract: workflow writes should be monotonic relative to current stored state. +4. Specify the exact source fields for `event_updated_at` and `latest_merged_at`, including timezone expectations. +5. Lock malformed-input behavior: deterministic error surface, null encoding, and exit behavior. +6. State explicitly that `action_state.py` must remain import-safe without unrelated env vars. + +## What the Plan Does Well + +- It isolates the hardest decision logic from opaque YAML into a testable Python surface. +- It keeps repository-variable writes out of the helper, which is the right separation of concerns. +- It correctly identifies timestamp parsing and skip logic as the threat-bearing surfaces. diff --git a/.planning/phases/08-github-action-readme/08-02-PLAN.md b/.planning/phases/08-github-action-readme/08-02-PLAN.md index dbe4960..d920a93 100644 --- a/.planning/phases/08-github-action-readme/08-02-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-02-PLAN.md @@ -31,12 +31,14 @@ user_setup: must_haves: truths: - "The repo ships a reusable workflow triggered by merged PRs and `workflow_dispatch`, not cron, per D-01 and D-02." - - "The workflow is copy-ready for target repositories that do not contain this tool's source tree because it bootstraps the CLI from an explicit checkout of this tool repo, satisfying ACTION-01." + - "The workflow is copy-ready for target repositories that do not contain this tool's source tree because it bootstraps the CLI and the 08-01 helper from an explicit checkout of this tool repo, and every project-code invocation runs through that same checked-out project environment, satisfying ACTION-01." - "Automation restores `.github-pr-kb/cache/` from Actions cache and keeps that working data out of git, per D-07 and D-09." - "Workflow runtime env is wired explicitly so the CLI receives `GITHUB_TOKEN`, classification receives `ANTHROPIC_API_KEY`, and `gh` commands receive authenticated `GH_TOKEN`, making the shipped automation executable." - "The workflow updates exactly one rolling KB bot PR and stages the full generated KB publication set — `kb/INDEX.md`, category/article markdown under `kb/`, and `kb/.manifest.json` — per D-06 and D-07." - "Auto merged-event runs compare only the event PR `updated_at` against the stored cursor, while manual recovery runs use the separate manual/latest-merged formula from Plan 08-01, eliminating self-skip ambiguity for ACTION-02 and ACTION-03." - "Repository-variable writes support PAT quickstart first and GitHub App as the advanced option with deterministic precedence, per D-13 and D-14." + - "Workflow-level concurrency serializes all automation/manual runs for this repo, and post-success cursor persistence re-reads the freshest repository variable and writes the monotonic max of that value and the helper's candidate `next_cursor`, per the 08-01 handoff contract." + - "External GitHub Actions are pinned by full commit SHA, and `KB_TOOL_REF` is an immutable release tag or full commit SHA rather than a floating branch, so copied workflows do not inherit silent supply-chain drift." artifacts: - path: ".github/workflows/github-pr-kb.yml" provides: "Merged-PR + workflow_dispatch automation with explicit tool bootstrap" @@ -50,6 +52,9 @@ must_haves: - path: "tests/test_action_workflow.py" provides: "Workflow contract assertions" contains: "test_workflow_bootstraps_tool_checkout_for_copyable_repos" + - path: "tests/test_action_workflow.py" + provides: "Execution-oriented smoke coverage for helper invocation" + contains: "test_action_state_helper_command_runs_from_project_checkout" key_links: - from: ".github/workflows/github-pr-kb.yml" to: "galzi/github-pr-kb" @@ -57,8 +62,8 @@ must_haves: pattern: "KB_TOOL_REPOSITORY|KB_TOOL_REF|actions/checkout" - from: ".github/workflows/github-pr-kb.yml" to: "src/github_pr_kb/action_state.py" - via: "python -m github_pr_kb.action_state" - pattern: "github_pr_kb\\.action_state" + via: "uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state" + pattern: "uv run --project \\.github-pr-kb-tool python -m github_pr_kb\\.action_state" - from: ".github/workflows/github-pr-kb.yml" to: "src/github_pr_kb/cli.py" via: "github-pr-kb extract/classify/generate" @@ -121,13 +126,18 @@ def decide_action_run(...) -> ActionRunDecision # CLI mode prints: # {"should_run": true, "extract_since": "...", "next_cursor": "...", "reason": "..."} ``` + +Workflow consumption contract: +```text +uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state ... +``` - Task 1: Add workflow contract tests that lock the shipped automation shape + Task 1: Add workflow contract and smoke tests that lock the shipped automation shape tests/test_action_workflow.py - .github/workflows/ci.yml @@ -139,7 +149,7 @@ def decide_action_run(...) -> ActionRunDecision - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_action_workflow.py` with text-based contract tests for `.github/workflows/github-pr-kb.yml`. Assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it bootstraps the CLI for copyable consumer repos by defining `KB_TOOL_REPOSITORY` (defaulting to this repo) and `KB_TOOL_REF`, checking out that tool repo into a dedicated directory, and installing it with `uv sync --project ... --all-groups --frozen` instead of assuming the target repo already contains this project's `pyproject.toml`; it invokes the CLI through that checkout via `uv run --project ... github-pr-kb ...`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; it invokes `python -m github_pr_kb.action_state`; for auto merged events it passes `--event-updated-at` and `--stored-cursor` to the helper and does not require `latest_merged_at` to decide the auto path; for manual dispatch it passes `--latest-merged-at`, optional `--manual-since`, and `--force`; it includes an explicit repeated-auto-event/no-op guard assertion (for example matching the helper reason `already_processed_auto_event` or an equivalent skip branch) so ACTION-02 is locked; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it stages the entire generated KB publication set explicitly — `kb/INDEX.md`, `kb/**/*.md`, and `kb/.manifest.json` — and never stages `.github-pr-kb/cache`, so top-level index publication cannot depend on ambiguous glob behavior; it references repository variable `KB_LAST_SUCCESSFUL_CURSOR`; it supports PAT secret `KB_VARIABLES_TOKEN` and GitHub App secrets `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` with app-first precedence; and it uses stable publication branch `automation/github-pr-kb` plus PR title `chore: update PR knowledge base`. + Create `tests/test_action_workflow.py` with two layers of coverage for `.github/workflows/github-pr-kb.yml`. First, add text-based contract tests that assert these exact workflow requirements: it has `pull_request` trigger with `types: [closed]`; it has `workflow_dispatch` with inputs named `since` and `force`; it does not contain `schedule:` per D-01; it contains a top-level or job-level `concurrency` block that serializes all runs for the repo with `cancel-in-progress: false`; it contains the job-level merged-only guard `github.event.pull_request.merged == true` per D-03; it bootstraps the CLI for copyable consumer repos by defining `KB_TOOL_REPOSITORY` (defaulting to this repo) and `KB_TOOL_REF`, where `KB_TOOL_REF` is an immutable release tag or full commit SHA rather than a floating branch, checking out that tool repo into a dedicated directory, and installing it with `uv sync --project ... --all-groups --frozen` instead of assuming the target repo already contains this project's `pyproject.toml`; it invokes both the helper and the CLI through that checkout via `uv run --project ... python -m github_pr_kb.action_state` and `uv run --project ... github-pr-kb ...`; it restores `.github-pr-kb/cache/` with Actions cache per D-09; for auto merged events it passes `--event-updated-at` and `--stored-cursor` to the helper and does not require `latest_merged_at` to decide the auto path; for manual dispatch it passes `--latest-merged-at`, optional `--manual-since`, and `--force`; it includes an explicit repeated-auto-event/no-op guard assertion (for example matching the helper reason `already_processed_auto_event` or an equivalent skip branch) so ACTION-02 is locked; the CLI steps explicitly map `GITHUB_TOKEN: ${{ github.token }}` and `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` so extractor/classifier auth is wired; every `gh api` / `gh pr` step explicitly maps `GH_TOKEN` from the resolved PAT-or-App token so repository-variable reads/writes and PR publication are authenticated; it uses a pinned GitHub App token-minting action by full commit SHA when `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are present, and falls back to `KB_VARIABLES_TOKEN` otherwise; it stages the entire generated KB publication set explicitly using git pathspec semantics — `kb/INDEX.md`, `:(glob)kb/**/*.md`, and `kb/.manifest.json` or an equivalent non-shell-glob enumeration — and never stages `.github-pr-kb/cache`, so top-level index publication cannot depend on ambiguous shell glob behavior; it re-reads repository variable `KB_LAST_SUCCESSFUL_CURSOR` immediately before persistence and writes the monotonic max of that fresh value and the helper's `next_cursor`; it treats `automation/github-pr-kb` as the dedicated bot branch and either updates it safely or fails clearly before cursor persistence; and it uses PR title `chore: update PR knowledge base`. Second, add one execution-oriented smoke test that runs the exact helper command shape through a project checkout environment (for this repo, `uv run --project python -m github_pr_kb.action_state ...`) and asserts it emits the expected JSON keys, so the plan is not relying on string assertions alone for the most fragile command form. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -147,15 +157,21 @@ def decide_action_run(...) -> ActionRunDecision - `tests/test_action_workflow.py` exists - file contains `test_workflow_has_merged_pr_and_dispatch_triggers` - file contains `test_workflow_bootstraps_tool_checkout_for_copyable_repos` + - file contains `test_action_state_helper_command_runs_from_project_checkout` - file contains an assertion that `schedule:` is absent + - file contains an assertion for workflow `concurrency` - file contains an assertion for `github.event.pull_request.merged == true` - file contains `KB_LAST_SUCCESSFUL_CURSOR` - file contains `KB_TOOL_REPOSITORY` + - file contains `uv run --project` and `github_pr_kb.action_state` - file contains `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, and `GH_TOKEN` + - file contains `actions/create-github-app-token` + - file contains a check that pinned actions use full commit SHA refs - file contains `kb/INDEX.md` + - file contains `:(glob)kb/**/*.md` or an equivalent explicit non-shell-glob staging assertion - file contains `automation/github-pr-kb` - Workflow tests define the exact shipped automation contract before the YAML is written, including explicit staging of `kb/INDEX.md` with the rest of the generated KB output. + Workflow tests define the exact shipped automation contract before the YAML is written, including helper execution through the checked-out tool environment, concurrency/monotonic cursor protections, pinned action/tool refs, and explicit staging of `kb/INDEX.md` with the rest of the generated KB output. @@ -178,9 +194,10 @@ def decide_action_run(...) -> ActionRunDecision - Test 1: merged PR and manual dispatch triggers are present, with no cron path - Test 2: workflow bootstraps the CLI from a checked-out copy of this tool repo so copied workflows do not depend on the target repo containing the package - Test 3: workflow skips before the expensive CLI pipeline when the helper reports no new merged PRs, including repeated auto events - - Test 4: successful runs explicitly stage `kb/INDEX.md`, KB markdown, and `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` + - Test 4: workflow serializes overlapping runs and persists `KB_LAST_SUCCESSFUL_CURSOR` monotonically against a fresh post-publication read + - Test 5: successful runs explicitly stage `kb/INDEX.md`, KB markdown, and `kb/.manifest.json`, update one rolling PR, and only then persist `KB_LAST_SUCCESSFUL_CURSOR` - Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Make the workflow copyable into consumer repos that do not contain this project by using one explicit bootstrap strategy: first checkout the target repository as normal, then checkout this tool repo (default `galzi/github-pr-kb`) into a dedicated directory such as `.github-pr-kb-tool` using workflow env/config keys `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`; install that checkout with `uv sync --project .github-pr-kb-tool --all-groups --frozen`; and invoke the CLI via `uv run --project .github-pr-kb-tool github-pr-kb ...` so commands run from the shipped tool checkout rather than from the consumer repo. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint an installation token first; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty). For auto merged-event runs, pass only the stored cursor plus `${{ github.event.pull_request.updated_at }}` into `python -m github_pr_kb.action_state`, and rely on the Plan 08-01 contract that compares `event_updated_at` to `stored_cursor`, emits `extract_since=stored_cursor`, and returns `next_cursor=event_updated_at`; do not let `latest_merged_at` participate in the auto decision path. For manual recovery/backfill runs, query the newest merged PR `updated_at`, pass that as `--latest-merged-at` together with optional `--manual-since` and `--force`, and rely on the separate manual formula from Plan 08-01. When `should_run` is false, exit before extract/classify/generate and print the helper reason, including repeated-auto-event no-op cases, per ACTION-02. When it is true, run `uv run --project .github-pr-kb-tool github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `uv run --project .github-pr-kb-tool github-pr-kb classify`, then `uv run --project .github-pr-kb-tool github-pr-kb generate`. Publish via one stable branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, stage the generated KB output explicitly with pathspecs or commands that include `kb/INDEX.md`, `kb/**/*.md`, and `kb/.manifest.json` (not just ambiguous recursive globs), verify no `.github-pr-kb/cache` files are added, push/update the branch, create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research. Add optional `actions/upload-artifact` on failure only for debugging per D-10. + Create `.github/workflows/github-pr-kb.yml` with one automation job that follows the repo's existing checkout/setup-uv/install pattern and implements these concrete steps. Trigger on `pull_request` `types: [closed]` with a job-level `if: github.event.pull_request.merged == true` guard per D-01/D-03, plus `workflow_dispatch` with inputs `since` (optional ISO timestamp) and `force` (boolean) per D-02/D-04. Add workflow-level `concurrency` that serializes all runs for the repo (for example `group: github-pr-kb-${{ github.repository }}` with `cancel-in-progress: false`) so merged-event and manual runs cannot overlap and write state out of order. Make the workflow copyable into consumer repos that do not contain this project by using one explicit bootstrap strategy: first checkout the target repository as normal, then checkout this tool repo (default `galzi/github-pr-kb`) into a dedicated directory such as `.github-pr-kb-tool` using workflow env/config keys `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`; set `KB_TOOL_REF` in the shipped workflow to an immutable release tag or full commit SHA, not a floating branch; install that checkout with `uv sync --project .github-pr-kb-tool --all-groups --frozen`; and invoke both the helper and the CLI via that checkout (`uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state ...` and `uv run --project .github-pr-kb-tool github-pr-kb ...`) so every project-code invocation runs from the shipped tool checkout rather than from the consumer repo. Pin every third-party action in the workflow by full commit SHA to match repo convention, including checkout/setup/cache/upload-artifact and the GitHub App token action. Use `permissions` at least `contents: write`, `pull-requests: write`, and `actions: read`. Restore `.github-pr-kb/cache/` via `actions/cache` and never add that path to git per D-07/D-09. Resolve repository-variable auth with deterministic precedence: if both `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY` are set, mint a GitHub App installation token first using a pinned token action such as `actions/create-github-app-token` targeted explicitly at the current repository owner/repo; fail clearly if that token cannot be minted for this repo; otherwise fall back to `KB_VARIABLES_TOKEN`; fail clearly if neither path exists per D-13/D-14. Export that resolved credential as `GH_TOKEN` on every `gh api` / `gh pr` step so repository-variable reads/writes and rolling-PR operations are actually authenticated. Separately wire the CLI/runtime env explicitly: set `GITHUB_TOKEN: ${{ github.token }}` on the extract/classify/generate pipeline steps so the existing config/extractor auth works, and set `ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}` on classification-generating steps so Claude-backed work can run. Read `KB_LAST_SUCCESSFUL_CURSOR` through `gh api` (404 => empty). For auto merged-event runs, pass only the stored cursor plus `${{ github.event.pull_request.updated_at }}` into `uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state`, and rely on the Plan 08-01 contract that compares `event_updated_at` to `stored_cursor`, emits `extract_since=stored_cursor`, and returns `next_cursor=event_updated_at`; do not let `latest_merged_at` participate in the auto decision path. For manual recovery/backfill runs, query the newest merged PR `updated_at`, pass that as `--latest-merged-at` together with optional `--manual-since` and `--force`, and rely on the separate manual formula from Plan 08-01. When `should_run` is false, exit before extract/classify/generate and print the helper reason, including repeated-auto-event no-op cases, per ACTION-02. When it is true, run `uv run --project .github-pr-kb-tool github-pr-kb extract --repo ${{ github.repository }} --state closed` with `--since "$EXTRACT_SINCE"` when provided, then `uv run --project .github-pr-kb-tool github-pr-kb classify`, then `uv run --project .github-pr-kb-tool github-pr-kb generate`. Publish via one stable bot-owned branch `automation/github-pr-kb`: configure git identity as `github-actions[bot]`, fetch the remote automation branch if it exists, recreate or refresh the local publication branch deterministically from the current base branch, stage the generated KB output explicitly with git pathspecs such as `git add -- 'kb/INDEX.md' 'kb/.manifest.json' ':(glob)kb/**/*.md'` (or an equivalent explicit non-shell-glob enumeration), verify no `.github-pr-kb/cache` files are added, and push the dedicated automation branch with the selected safe update strategy. Use `--force-with-lease` only for that dedicated bot branch if needed to prevent stale branch history or prior failed runs from deadlocking publication; if push/update still fails because of branch protection, manual edits, or ownership drift, fail clearly and do not persist the cursor. Create the PR with title `chore: update PR knowledge base` only if absent, otherwise reuse the existing open PR per D-06. Persist `KB_LAST_SUCCESSFUL_CURSOR` only after the pipeline and PR publication succeed per ACTION-03 and Pitfall 1 from research, and before writing it, re-read the current repository variable again and persist the monotonic max of that fresh value and the helper's candidate `next_cursor` so out-of-order completions cannot move state backward. Add optional `actions/upload-artifact` on failure only for debugging per D-10. .venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q @@ -190,14 +207,20 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `types: [closed]` - `.github/workflows/github-pr-kb.yml` contains `github.event.pull_request.merged == true` - `.github/workflows/github-pr-kb.yml` does not contain `schedule:` + - `.github/workflows/github-pr-kb.yml` contains workflow `concurrency` with `cancel-in-progress: false` - `.github/workflows/github-pr-kb.yml` contains `KB_TOOL_REPOSITORY` - `.github/workflows/github-pr-kb.yml` contains `KB_TOOL_REF` + - `.github/workflows/github-pr-kb.yml` does not use a floating branch for `KB_TOOL_REF` - `.github/workflows/github-pr-kb.yml` contains `uv sync --project .github-pr-kb-tool --all-groups --frozen` + - `.github/workflows/github-pr-kb.yml` contains `uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state` - `.github/workflows/github-pr-kb.yml` contains `uv run --project .github-pr-kb-tool github-pr-kb extract` - `.github/workflows/github-pr-kb.yml` contains `.github-pr-kb/cache/` - `.github/workflows/github-pr-kb.yml` contains `kb/INDEX.md` - `.github/workflows/github-pr-kb.yml` contains `kb/.manifest.json` + - `.github/workflows/github-pr-kb.yml` contains `:(glob)kb/**/*.md` or an equivalent explicit non-shell-glob staging strategy - `.github/workflows/github-pr-kb.yml` contains `KB_LAST_SUCCESSFUL_CURSOR` + - `.github/workflows/github-pr-kb.yml` contains a second post-publication read of `KB_LAST_SUCCESSFUL_CURSOR` + - `.github/workflows/github-pr-kb.yml` persists the monotonic max of the fresh stored cursor and helper `next_cursor` - `.github/workflows/github-pr-kb.yml` contains `--event-updated-at` - `.github/workflows/github-pr-kb.yml` contains `already_processed_auto_event` or an equivalent explicit auto no-op branch - `.github/workflows/github-pr-kb.yml` contains `GITHUB_TOKEN` @@ -205,9 +228,12 @@ def decide_action_run(...) -> ActionRunDecision - `.github/workflows/github-pr-kb.yml` contains `GH_TOKEN` - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_TOKEN` - `.github/workflows/github-pr-kb.yml` contains `KB_VARIABLES_APP_ID` + - `.github/workflows/github-pr-kb.yml` contains a pinned `actions/create-github-app-token` step or an equivalent pinned GitHub App token minting action + - `.github/workflows/github-pr-kb.yml` pins third-party actions by full commit SHA + - `.github/workflows/github-pr-kb.yml` defines the failure behavior for publication-branch push/update errors before cursor persistence - `.github/workflows/github-pr-kb.yml` contains `automation/github-pr-kb` - The repo contains a copy-ready workflow that bootstraps this tool explicitly for consumer repos, matches the locked merged-PR/manual-dispatch design, avoids cron, preserves cache/manifest rules, explicitly stages `kb/INDEX.md` with the rest of the generated KB output, and updates repository state only after success. + The repo contains a copy-ready workflow that bootstraps this tool explicitly for consumer repos, runs both helper and CLI from the same checked-out project environment, matches the locked merged-PR/manual-dispatch design, avoids cron, serializes runs, preserves cache/manifest rules, explicitly stages `kb/INDEX.md` with the rest of the generated KB output via git pathspecs, pins external refs, and updates repository state only after successful publication with monotonic cursor persistence. @@ -229,17 +255,22 @@ def decide_action_run(...) -> ActionRunDecision | T-08-05 | T | cursor persistence step | mitigate | Update `KB_LAST_SUCCESSFUL_CURSOR` only after extract/classify/generate and PR publication succeed, preventing state tampering that would skip unprocessed PRs | | T-08-06 | I | git publication scope | mitigate | Stage only the explicit generated KB publication set: `kb/INDEX.md`, article markdown under `kb/`, and `kb/.manifest.json`; never add `.github-pr-kb/cache/`, and rely on `.gitignore` plus explicit `git add` paths | | T-08-07 | D | rerun/no-new-PR behavior | mitigate | Call the tested `action_state` helper before the CLI pipeline and short-circuit when no newer merged PR exists, limiting wasted API and Claude calls | +| T-08-08 | T | out-of-order workflow completion | mitigate | Add workflow concurrency and a post-publication re-read/max-write of `KB_LAST_SUCCESSFUL_CURSOR` so overlapping runs cannot move durable state backward | +| T-08-09 | S | external workflow/tool refs | mitigate | Pin third-party actions by full commit SHA and keep `KB_TOOL_REF` immutable so copied workflows do not silently drift to unexpected code | +| T-08-10 | D | rolling PR branch drift/conflict | mitigate | Treat `automation/github-pr-kb` as a dedicated bot branch, allow only the chosen safe update strategy for that branch, and fail clearly before cursor persistence if publication cannot be updated | -Run `.venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q`. After that passes, inspect the workflow text to confirm there is no `schedule:` trigger, that `kb/INDEX.md` and `kb/.manifest.json` are staged explicitly with the generated KB markdown, and that `KB_LAST_SUCCESSFUL_CURSOR` is only written after the publication path. +Run `.venv/Scripts/python.exe -m pytest tests/test_action_workflow.py -x -q`. After that passes, inspect the workflow text to confirm there is no `schedule:` trigger, that helper execution uses `uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state`, that workflow concurrency is present, that third-party actions and `KB_TOOL_REF` are pinned immutably, that `kb/INDEX.md` and `kb/.manifest.json` are staged explicitly with the generated KB markdown via git pathspecs or an equivalent non-shell-glob strategy, and that `KB_LAST_SUCCESSFUL_CURSOR` is re-read and only written after the publication path using monotonic max semantics. - ACTION-01 is implemented as a shipped workflow file that can be copied into repos without this tool's source tree - ACTION-02 guard exists before extract/classify/generate, uses the explicit auto/manual helper contract, and is covered by tests -- ACTION-03 persists the repository variable only on successful end-to-end publication +- ACTION-03 persists the repository variable only on successful end-to-end publication, and does so monotonically against the freshest stored value - `.github-pr-kb/cache/` stays out of git while `kb/INDEX.md` and `kb/.manifest.json` ship with the generated KB markdown +- Both helper and CLI execution paths remain valid in copied consumer repos because they run from the same checked-out tool environment +- Workflow refs are pinned and publication-branch failure modes do not silently corrupt cursor state diff --git a/.planning/phases/08-github-action-readme/08-02-RISK-REVIEW.md b/.planning/phases/08-github-action-readme/08-02-RISK-REVIEW.md new file mode 100644 index 0000000..b2dd6c0 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-02-RISK-REVIEW.md @@ -0,0 +1,105 @@ +# Risk Review: 08-02-PLAN.md + +## Plan Summary + +The plan aims to ship a reusable, copy-ready GitHub Actions workflow that runs after merged PRs or on manual dispatch, bootstraps this tool from a dedicated checkout, uses the 08-01 `action_state` helper to decide whether to run, publishes generated `kb/` output through one rolling bot PR, reuses `.github-pr-kb/cache/`, and persists `KB_LAST_SUCCESSFUL_CURSOR` only after successful publication. + +The key moving parts are: + +- the consumer repository checkout plus a second checkout of the tool repository +- `uv`-based bootstrap and command execution from the tool checkout +- the 08-01 `github_pr_kb.action_state` helper +- repository-variable reads/writes through `gh api` +- dual-mode auth (`KB_VARIABLES_TOKEN` or GitHub App credentials) +- Actions cache for `.github-pr-kb/cache/` +- explicit staging and publication of `kb/INDEX.md`, `kb/**/*.md`, and `kb/.manifest.json` +- one stable publication branch and one rolling PR + +The plan's theory of success is sound in outline: if the workflow can bootstrap the tool deterministically, invoke the helper in the correct environment, skip no-op runs early, publish only the generated KB outputs, and persist cursor state monotonically after success, then merged PRs should keep the KB current without cron, duplicate PRs, or cache files leaking into git. + +## Assumptions & Evidence + +| ID | Assumption | Explicit / Implicit | Justification status | Blast radius if wrong | Early validation | +|---|---|---|---|---|---| +| A1 | A copy-ready workflow can bootstrap from `.github-pr-kb-tool` while still invoking `python -m github_pr_kb.action_state` successfully in consumer repos. | Implicit | Weak | Critical | Require the helper to run through the same tool environment as the CLI (`uv run --project ... python -m ...`) or add a workflow smoke test in a repo without this source tree. | +| A2 | "Persist after success" is enough to prevent cursor corruption under overlapping merged-event runs. | Implicit | Weak | Critical | Add workflow-level concurrency and a final re-read/max-write of the repository variable just before persistence. | +| A3 | The GitHub App auth path is fully specified by `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`. | Implicit | Weak | High | Specify the exact token-minting action/API and how installation selection works for the current repo. | +| A4 | Defaulting `KB_TOOL_REPOSITORY` / `KB_TOOL_REF` is safe for consumer repos from both reliability and supply-chain perspectives. | Explicit | Partial | High | Pin the default ref to an immutable release tag or SHA and document the upgrade path. | +| A5 | Text-based workflow contract tests are sufficient to catch real execution failures in shell quoting, pathspecs, auth wiring, and JSON/null handling. | Explicit | Partial | High | Add at least one execution-oriented smoke test for the helper invocation and publication path. | +| A6 | The workflow can stage `kb/INDEX.md`, nested KB markdown, and `kb/.manifest.json` deterministically without accidentally depending on shell-specific glob behavior. | Implicit | Partial | High | Use explicit git pathspecs or enumerated file lists and assert the exact command shape. | +| A7 | One stable branch and one rolling PR will remain operable in repos with branch protection, merge conflicts, or pre-existing automation branches. | Implicit | Weak | Medium-High | Define expected behavior when the branch already exists, cannot be pushed, or has conflicting changes. | +| A8 | Introducing additional GitHub Actions for cache, artifact upload, or app token minting without pinning is acceptable in this repo. | Implicit | Weak | High | Follow the existing repo convention of pinning third-party actions by commit SHA. | + +## Ipcha Mistabra — Devil's Advocacy + +### Inversion Test + +1. **The copy-ready bootstrap may reduce portability rather than improve it.** A second checkout plus project-scoped `uv` commands makes the workflow more reusable in principle, but it also creates a two-repo runtime model with ref management, environment separation, and failure modes that do not exist when the code is local or packaged. If the helper invocation is not routed through that same environment, copied workflows will fail immediately in the exact repos they were meant to support. + +2. **The cursor design may still skip or replay work even though 08-01 is correct.** The helper can emit a correct candidate cursor and still lose the race operationally if two merged-event runs read the same stored cursor, do work in parallel, and the older event writes last. In other words, moving logic out of YAML does not remove the need for concurrency control in YAML. + +3. **A rolling PR may reduce PR spam while increasing operational brittleness.** One long-lived branch is tidy, but it is also a single choke point for merge conflicts, stale branch state, protection rules, and manual tampering. The cleaner UX for reviewers can come at the cost of harder automation recovery. + +4. **Dual auth may increase setup burden more than it increases robustness.** Supporting PAT and App paths sounds flexible, but it also doubles the number of credentials, precedence branches, and failure modes. If the App path is underspecified, the documented "advanced" option becomes a support trap. + +### The Little Boy from Copenhagen + +- A maintainer copying this workflow into another repo will ask a simple question: "Why does `uv run --project .github-pr-kb-tool github-pr-kb ...` work, but `python -m github_pr_kb.action_state` also somehow work?" If the answer is "because the environment happens to be activated," the workflow is not actually copy-ready. +- A security reviewer will ask why a consumer repo is executing code from an external repository ref and possibly unpinned third-party actions. "It is our own repo" is not a sufficient supply-chain argument if the ref floats. +- An on-call engineer will ask what prevents two close-together merges from racing each other and writing cursor state out of order. +- A new maintainer will ask what happens when `automation/github-pr-kb` already exists, has conflicts, or is protected. + +### Failure-of-Imagination Check + +- A copied workflow defaults to a floating `KB_TOOL_REF`, the upstream tool changes behavior, and many consumer repos silently start generating different KB output without any repo-local code change. +- The App credentials are present but the app is not installed on the current repo, or the token action resolves the wrong installation; the workflow treats a failed variable read as "no cursor" and performs an unexpectedly broad extraction. +- Workflow text tests pass, but `git add kb/**/*.md` behaves differently than expected on the runner shell, so nested articles are not staged while `kb/INDEX.md` is, creating a publication state that looks valid in the PR but breaks dedup on later runs. +- A merged-event run and a manual backfill both succeed; the later finisher writes the smaller cursor, and the system appears healthy until future runs repeat or skip windows. + +## Risk Register + +| Risk ID | Category | Description | Trigger | Probability | Severity | Priority | Detection | Mitigation | Contingency | Assumption link | +|---|---|---|---|---|---|---|---|---|---|---| +| R1 | Technical | **Known Unknown:** the workflow may fail in copied consumer repos because the helper is described as `python -m github_pr_kb.action_state` even though the plan's portability model depends on a separate tool checkout and project-scoped environment. | The workflow runs in a repo that does not contain this package source tree. | High — the plan text currently points in two different execution directions. | Critical — the workflow fails before skip logic, extraction, or publication. | Critical | Early workflow failure with `ModuleNotFoundError` or import-path errors. | Make helper invocation explicit and environment-consistent: `uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state` or an equivalent tool-scoped command. Add a consumer-repo smoke test. | Patch the workflow to route the helper through the tool checkout and rerun manually. | A1 | +| R2 | Operational | **Known Unknown:** overlapping merged-event or manual runs can still regress or stall durable state because the plan does not yet require workflow concurrency plus a final monotonic write against the freshest stored cursor. | Two runs overlap and finish out of order. | Medium-High — merged PR workflows naturally overlap in active repos. | Critical — skipped or replayed PR windows undermine the whole incremental contract. | Critical | Cursor value moves backward, repeated "already processed" skips appear unexpectedly, or later runs repeat older windows. | Add workflow/job concurrency, re-read `KB_LAST_SUCCESSFUL_CURSOR` immediately before writing, and persist `max(current_stored, candidate_next_cursor)`. | Repair the variable manually, run backfill, and temporarily serialize the workflow. | A2 | +| R3 | Security / Operational | **Known Unknown:** the GitHub App path is underspecified; minting or selecting the installation token may fail or target the wrong installation. | App credentials are configured, but installation resolution is ambiguous or incorrect. | Medium | High — repository-variable reads/writes and PR operations fail or act on the wrong auth path. | High | `gh api` returns auth/permission errors, or App path behaves differently across repos/orgs. | Specify the exact action/API for App token minting, pin it, and define installation lookup for the current repository. Consider requiring an installation identifier if automatic lookup is unreliable. | Fall back to PAT quickstart path and document the limitation until App flow is proven. | A3 | +| R4 | Security | **Unknown Unknown surfaced by inversion:** consumer repos may inherit supply-chain drift because the plan allows external tool checkout and likely new third-party actions without requiring immutable pins. | `KB_TOOL_REF` or an added action references a mutable branch/tag that later changes. | Medium | High — behavior can change across repos without local review, and compromised upstream refs expand blast radius. | High | Workflow behavior changes without repo-local code changes; diffs show unexpected action/tool versions. | Pin `KB_TOOL_REF` to a release tag or SHA by default, pin all third-party actions by commit SHA, and document version upgrade steps explicitly. | Freeze to a known-good ref and publish a hotfix README/workflow update. | A4, A8 | +| R5 | Technical | **Known Unknown:** text-only workflow tests can pass while shell quoting, JSON parsing, null handling, or publication commands still fail at runtime. | YAML contains expected strings but actual step execution differs from the intended contract. | Medium | High — false confidence leads to broken shipped automation. | High | Green contract tests but failed Actions runs in real repos; helper output not parsed as expected. | Add at least one execution-oriented smoke test around helper invocation and publication-step command construction. Keep string tests, but do not rely on them alone. | Fix the workflow step wiring and rerun without changing product behavior. | A5 | +| R6 | Operational | Explicit staging of generated KB output may still be brittle if the implementation relies on shell globbing rather than git pathspec semantics. | The workflow uses `kb/**/*.md` in a shell-sensitive way or forgets to include `kb/.manifest.json` and `kb/INDEX.md` together. | Medium | High — publication can become silently incomplete and later cause duplicate generation. | High | PR branch is missing nested KB files or manifest/index drift after a "successful" run. | Use git pathspecs like `:(glob)kb/**/*.md` or explicit file enumeration, and assert the exact staging command shape. | Repair the branch contents and rerun generation from the last good cursor. | A6 | +| R7 | Operational | A single rolling branch/PR may deadlock on merge conflicts, branch protection, or stale manual edits. | `automation/github-pr-kb` already exists with conflicting history or protected push rules. | Medium | Medium-High — updates stall and repeated runs keep failing. | High | Push rejection, repeated PR-update failures, or a permanently stale open KB PR. | Define branch recovery behavior, expected bot ownership, and whether force-with-lease is permitted. Document how maintainers reset the branch safely. | Close/reset the automation branch and recreate the rolling PR. | A7 | + +## Verdict & Recommendations + +**Overall Risk Level:** High + +**Verdict:** The plan is strong on product shape and much better grounded than 08-01, but it is **not ready as written** because several operationally load-bearing details are still implicit. The biggest issue is that the workflow's portability story and the helper invocation story are currently inconsistent, and the 08-01 handoff requirement for concurrency-aware monotonic persistence is not yet turned into an explicit 08-02 implementation/test contract. + +**Top 3 Risks** + +1. R1 — helper execution path breaks in copied consumer repos +2. R2 — out-of-order runs corrupt or stall cursor state +3. R3 — GitHub App auth path is underspecified and may fail in real repos + +**Recommended Actions** + +1. Make the helper execution path explicit and single-source: every invocation of project code, including `action_state`, should run through the `.github-pr-kb-tool` environment. +2. Promote concurrency and monotonic persistence from "background expectation" to a first-class acceptance criterion and test target in 08-02. +3. Specify the GitHub App token implementation in concrete terms: which action/API, how installation is selected, and what exact scopes are required. +4. Pin every third-party action by commit SHA and default `KB_TOOL_REF` to an immutable versioned ref rather than a floating branch. +5. Replace shell-dependent staging language with exact git pathspec language and verify nested KB files, `kb/INDEX.md`, and `kb/.manifest.json` move together while `.github-pr-kb/cache/` never enters git. +6. Add one execution-level smoke test or fixture that proves the helper, env wiring, and publication command structure behave correctly outside pure string assertions. + +**Open Questions** + +- Will `KB_TOOL_REF` default to a floating branch, a release tag, or a full SHA? +- How exactly will the workflow mint and select a GitHub App installation token for the current repository? +- What concurrency group will serialize or supersede overlapping runs? +- What is the intended recovery path if `automation/github-pr-kb` cannot be pushed or is manually modified? +- Is the workflow expected to tolerate branch protection on the automation branch, or is that explicitly unsupported? + +**What the Plan Does Well** + +- It correctly assumes 08-01 exists and reuses that helper rather than re-embedding decision logic in YAML. +- It makes the workflow copy-ready as a first-class requirement instead of quietly depending on the source tree being present. +- It is explicit about auth separation (`GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `GH_TOKEN`) and about staging the manifest and top-level index with the KB content. +- It preserves the right high-level invariant: publish first, then persist the cursor. diff --git a/.planning/phases/08-github-action-readme/08-03-PLAN.md b/.planning/phases/08-github-action-readme/08-03-PLAN.md index f16e14c..7038a39 100644 --- a/.planning/phases/08-github-action-readme/08-03-PLAN.md +++ b/.planning/phases/08-github-action-readme/08-03-PLAN.md @@ -7,6 +7,7 @@ depends_on: - "08-02" files_modified: - README.md + - .env.example - tests/test_readme.py autonomous: true requirements: @@ -15,13 +16,18 @@ must_haves: truths: - "README leads with automation setup for the shipped merged-PR workflow, per D-11." - "README explains the explicit copyable-workflow bootstrap path: the target repo checks out this tool repo and installs/runs the CLI from that checkout, satisfying ACTION-01." - - "README documents PAT quickstart first and GitHub App as the optional advanced path for repository-variable auth, per D-13 and D-14." - - "README still covers uv installation, local setup, env vars, CLI commands, and KB output examples, per D-12 and INFRA-03." - - "README tells maintainers exactly what is committed (`kb/` + `kb/.manifest.json`) and what remains outside git (`.github-pr-kb/cache/`), per D-07 and D-09." + - "README documents PAT quickstart first and GitHub App as the optional advanced path for repository-variable auth, but also separates local CLI/runtime credentials from workflow-only repository-variable credentials so token roles are unambiguous, per D-13 and D-14." + - "README still covers uv installation, local setup, env vars, CLI commands, and KB output examples, per D-12 and INFRA-03, using platform-aware local setup/test guidance rather than a Windows-only command presented as universal." + - "README tells maintainers exactly what is committed (`kb/` + `kb/.manifest.json`) and what remains outside git (`.github-pr-kb/cache/`), per D-07 and D-09, ideally via an explicit committed-vs-not-committed table." + - "README explains that consumer repos copy the workflow file, not this tool's source tree, and includes safe upgrade guidance for immutable `KB_TOOL_REF` pins." + - "`.env.example` stays aligned with the shipped local config surface and explicitly does not masquerade as the workflow-secrets template, preventing README/template drift." artifacts: - path: "README.md" provides: "Automation-first onboarding and local usage guide" contains: "KB_TOOL_REPOSITORY" + - path: ".env.example" + provides: "Local CLI environment template aligned with README" + contains: "ANTHROPIC_GENERATE_MODEL" - path: "tests/test_readme.py" provides: "README contract checks" contains: "test_readme_is_automation_first" @@ -34,13 +40,17 @@ must_haves: to: "src/github_pr_kb/config.py" via: "documented env/config surface" pattern: "ANTHROPIC_API_KEY|KB_OUTPUT_DIR|MIN_CONFIDENCE" + - from: "README.md" + to: ".env.example" + via: "local env template and authority boundary" + pattern: "GITHUB_TOKEN|ANTHROPIC_API_KEY|workflow secrets" --- Rewrite the README so a maintainer can install the shipped workflow correctly and a new user can still run the tool locally from the docs alone. -Purpose: Close INFRA-03 with documentation that matches the implemented workflow exactly, honors automation-first ordering from D-11, and preserves local CLI onboarding from D-12. -Output: Updated `README.md` and README contract tests. +Purpose: Close INFRA-03 with documentation that matches the implemented workflow exactly, honors automation-first ordering from D-11, preserves local CLI onboarding from D-12, and keeps `.env.example` aligned with the documented local config surface. +Output: Updated `README.md`, `.env.example`, and README contract tests. @@ -87,7 +97,7 @@ github-pr-kb run --repo owner/name Task 1: Add README contract tests for workflow-first onboarding - tests/test_readme.py + tests/test_readme.py, .env.example - README.md - .github/workflows/github-pr-kb.yml @@ -98,26 +108,29 @@ github-pr-kb run --repo owner/name - .planning/phases/08-github-action-readme/08-RESEARCH.md - .planning/phases/08-github-action-readme/08-VALIDATION.md - Create `tests/test_readme.py` with text-contract assertions for the final README. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; the README explains the copyable bootstrap path for consumer repositories by naming `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`, describing the second checkout of this tool repo, and making clear that the target repo does not need to contain this project's `pyproject.toml` or package source to use the workflow; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; it includes an explicit `Install uv` subsection before any `uv sync --all-groups --frozen` command, includes the official docs URL `https://docs.astral.sh/uv/getting-started/installation/`, and includes at least one copy/pasteable install command so a brand-new user can start from zero per INFRA-03; it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; and it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown. + Create `tests/test_readme.py` with text-contract assertions for the final README and local env template. Lock these exact expectations: the automation section appears before local CLI usage per D-11; the workflow file `.github/workflows/github-pr-kb.yml` is named explicitly; the README explains the copyable bootstrap path for consumer repositories by naming `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`, describing the second checkout of this tool repo, making clear that the target repo copies only the workflow file and does not need to contain this project's `pyproject.toml` or package source to use the workflow, and explaining that `KB_TOOL_REF` should remain immutable and be updated intentionally; PAT quickstart secret `KB_VARIABLES_TOKEN` is documented before the GitHub App section; the README also names `KB_VARIABLES_APP_ID` and `KB_VARIABLES_APP_PRIVATE_KEY`; the README distinguishes local CLI/runtime auth (`GITHUB_TOKEN`, `ANTHROPIC_API_KEY`) from workflow repository-variable auth (`KB_VARIABLES_TOKEN` or the GitHub App path) and mentions that the workflow maps the resolved repository-variable credential to `GH_TOKEN` internally for `gh api` / `gh pr`; it includes an explicit `Install uv` subsection before any `uv sync --all-groups --frozen` command, includes the official docs URL `https://docs.astral.sh/uv/getting-started/installation/`, and includes at least one copy/pasteable install command so a brand-new user can start from zero per INFRA-03; it provides platform-aware local test guidance (Windows plus macOS/Linux, or a clearly scoped Windows-only note rather than a universal claim); it lists `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE`; it includes the commands `github-pr-kb extract --repo owner/name`, `github-pr-kb classify`, and `github-pr-kb generate`; it states that `.github-pr-kb/cache/` stays out of git while `kb/.manifest.json` is committed with KB markdown; and it includes a committed-vs-not-committed explanation or equivalent table. Also add assertions that `.env.example` stays aligned with the local config surface (`GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, `MIN_CONFIDENCE`) while making clear that workflow secrets such as `KB_VARIABLES_TOKEN` and `KB_VARIABLES_APP_*` belong in repository secrets rather than `.env`. .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q - `tests/test_readme.py` exists - file contains `test_readme_is_automation_first` + - file contains `test_readme_distinguishes_local_and_workflow_credentials` - file contains `KB_TOOL_REPOSITORY` - file contains `KB_TOOL_REF` - file contains `KB_VARIABLES_TOKEN` - file contains `KB_VARIABLES_APP_ID` + - file contains `GH_TOKEN` - file contains `https://docs.astral.sh/uv/getting-started/installation/` - file contains `.github-pr-kb/cache/` + - file contains `.env.example` - README tests pin the exact documentation promises before the README is rewritten. + README tests pin the exact documentation promises before the README is rewritten, including credential-role clarity, committed-vs-not-committed guidance, and README/`.env.example` alignment. Task 2: Rewrite README to match the shipped workflow and local CLI reality - README.md, tests/test_readme.py + README.md, .env.example, tests/test_readme.py - README.md - tests/test_readme.py @@ -129,7 +142,7 @@ github-pr-kb run --repo owner/name - .planning/phases/08-github-action-readme/08-CONTEXT.md - .planning/phases/08-github-action-readme/08-RESEARCH.md - Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, and the copyable bootstrap strategy from Plan 08-02: the workflow checks out this tool repo (default `galzi/github-pr-kb`) via `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`, installs it in `.github-pr-kb-tool`, and runs the CLI from that checkout so consumer repos do not need to vendor this project's source tree. Document required secret `ANTHROPIC_API_KEY`, PAT quickstart for repository-variable writes using `KB_VARIABLES_TOKEN`, and the optional advanced GitHub App path using `KB_VARIABLES_APP_ID` plus `KB_VARIABLES_APP_PRIVATE_KEY` per D-11/D-14. In the local setup section, add an explicit `Install uv` subsection before any sync command: include the official docs URL `https://docs.astral.sh/uv/getting-started/installation/` plus concrete install commands for Windows PowerShell (`powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`) and macOS/Linux (`curl -LsSf https://astral.sh/uv/install.sh | sh`) so a brand-new user can go from zero to a working environment using the README alone. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, mention the repo's test command `.venv/Scripts/python.exe -m pytest tests/` from `CLAUDE.md`, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Explicitly state that `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. + Rewrite `README.md` so it describes the shipped product, not the early brainstorm. Lead with an automation-first section that explains the merged-PR trigger plus `workflow_dispatch`, the exact workflow file path `.github/workflows/github-pr-kb.yml`, and the copyable bootstrap strategy from Plan 08-02: the workflow checks out this tool repo (default `galzi/github-pr-kb`) via `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`, installs it in `.github-pr-kb-tool`, and runs the CLI from that checkout so consumer repos copy the workflow file only and do not vendor this project's source tree. Document that `KB_TOOL_REF` should stay pinned to an immutable release tag or full SHA and explain how maintainers intentionally upgrade that pin. Add a compact credential-role matrix or equivalent explanation that distinguishes local/runtime auth (`GITHUB_TOKEN`, `ANTHROPIC_API_KEY`) from workflow repository-variable auth (`KB_VARIABLES_TOKEN` quickstart or `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` advanced path), and mention that the workflow maps the resolved repository-variable credential to `GH_TOKEN` internally for `gh api` / `gh pr` steps. Keep PAT quickstart first per D-13/D-14, but document the minimum fine-grained PAT permissions needed and frame the GitHub App path as the preferred long-term service-account option. In the local setup section, add an explicit `Install uv` subsection before any sync command: include the official docs URL `https://docs.astral.sh/uv/getting-started/installation/` plus concrete install commands for Windows PowerShell (`powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"`) and macOS/Linux (`curl -LsSf https://astral.sh/uv/install.sh | sh`) so a brand-new user can go from zero to a working environment using the README alone. After that, document local setup with `uv sync --all-groups --frozen` for automation parity, provide platform-aware test guidance (for example Windows `.venv/Scripts/python.exe -m pytest tests/` and macOS/Linux `.venv/bin/python -m pytest tests/`) rather than presenting a Windows-only command as universal, list the local env/config surface from `config.py` and `.env.example`, and provide concrete CLI examples for `extract`, `classify`, `generate`, and `run`. Include a small KB output example/tree that shows `kb/INDEX.md`, category markdown files, and `kb/.manifest.json`. Add a small committed-vs-not-committed table or equivalent explicit section that states `.github-pr-kb/cache/` is transient Actions/local cache data and must stay out of git, while the workflow commits KB markdown plus the manifest only. Also update `.env.example` so it remains consistent with the shipped local config surface, includes the current local env vars/defaults, and explicitly notes that workflow secrets such as `KB_VARIABLES_TOKEN` and `KB_VARIABLES_APP_*` belong in GitHub repository secrets rather than `.env`. Remove stale claims about OpenRouter, Chroma, TOON, schedules, or any other unshipped architecture. .venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q @@ -140,12 +153,17 @@ github-pr-kb run --repo owner/name - `README.md` contains `KB_TOOL_REF` - `README.md` explains that consumer repos do not need this tool's source tree checked in - `README.md` mentions `KB_VARIABLES_TOKEN` before `KB_VARIABLES_APP_ID` + - `README.md` distinguishes local/runtime credentials from workflow repository-variable credentials + - `README.md` mentions internal workflow `GH_TOKEN` mapping for `gh api` / `gh pr` - `README.md` contains `https://docs.astral.sh/uv/getting-started/installation/` + - `README.md` contains platform-aware local test guidance rather than a Windows-only universal command - `README.md` contains `github-pr-kb extract --repo owner/name` - `README.md` contains `.github-pr-kb/cache/` - `README.md` contains `kb/.manifest.json` + - `README.md` includes committed-vs-not-committed guidance for `kb/` and `.github-pr-kb/cache/` + - `.env.example` reflects the current local env surface and explicitly excludes workflow-only secrets from `.env` - The README alone is sufficient to configure automation and run the tool locally, and every documented secret/command matches shipped code and workflow behavior. + The README alone is sufficient to configure automation and run the tool locally, every documented secret/command matches shipped code and workflow behavior, and `.env.example` no longer drifts from the documented local config surface. @@ -162,19 +180,22 @@ github-pr-kb run --repo owner/name | Threat ID | Category | Component | Disposition | Mitigation Plan | |-----------|----------|-----------|-------------|-----------------| -| T-08-08 | S | auth setup documentation | mitigate | Document PAT quickstart and GitHub App separately with exact secret names and precedence, reducing setup spoofing/confusion | -| T-08-09 | T | repo hygiene documentation | mitigate | Explicitly state that `.github-pr-kb/cache/` stays out of git and that only `kb/*.md` plus `kb/.manifest.json` are committed | -| T-08-10 | I | README examples | mitigate | Remove stale architecture claims and list only the actual env vars and commands present in `config.py`, `.env.example`, and `cli.py` | +| T-08-08 | S | auth setup documentation | mitigate | Document PAT quickstart and GitHub App separately with exact secret names, minimum PAT permissions, credential-role separation, and explicit workflow `GH_TOKEN` mapping so local/runtime auth is not confused with repository-variable auth | +| T-08-09 | T | repo hygiene documentation | mitigate | Add explicit committed-vs-not-committed guidance so `.github-pr-kb/cache/` stays out of git while `kb/*.md` plus `kb/.manifest.json` are committed | +| T-08-10 | I | README and `.env.example` examples | mitigate | Remove stale architecture claims, keep `.env.example` aligned with the local config surface, and distinguish workflow-only secrets from `.env` to avoid contradictory onboarding docs | +| T-08-11 | D | platform-specific local guidance | mitigate | Provide platform-aware setup/test commands or clearly scope repo-specific Windows commands so the README's zero-to-working promise remains true for non-Windows readers | -Run `.venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q`, then read the first major sections of `README.md` to confirm automation setup appears before local usage. +Run `.venv/Scripts/python.exe -m pytest tests/test_readme.py -x -q`, then read the first major sections of `README.md` to confirm automation setup appears before local usage, credential roles are distinct, committed-vs-not-committed guidance is explicit, and `.env.example` aligns with the documented local env surface. - INFRA-03 is satisfied by an automation-first README that matches the shipped workflow - Both auth paths are documented with PAT first and GitHub App second - Local env vars and CLI commands in README match current code exactly +- README and `.env.example` do not contradict each other about local vs workflow-only configuration +- Local setup/test guidance is understandable for both Windows and macOS/Linux readers diff --git a/.planning/phases/08-github-action-readme/08-03-RISK-REVIEW.md b/.planning/phases/08-github-action-readme/08-03-RISK-REVIEW.md new file mode 100644 index 0000000..47c358b --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-03-RISK-REVIEW.md @@ -0,0 +1,99 @@ +# Risk Review: 08-03-PLAN.md + +## Plan Summary + +The plan aims to rewrite `README.md` and add `tests\test_readme.py` so the shipped merged-PR workflow from 08-02 is documented first, PAT and GitHub App setup are explained, local CLI usage remains viable, and maintainers understand exactly what the automation commits versus what stays out of git. + +The key moving parts are: + +- the README restructure itself, with automation setup before local usage per D-11 +- README contract tests that pin ordering, secret names, commands, and output-path guidance +- the 08-02 workflow contract, especially `.github\workflows\github-pr-kb.yml`, `KB_TOOL_REPOSITORY`, and `KB_TOOL_REF` +- the local config surface from `src\github_pr_kb\config.py` and `.env.example` +- the shipped CLI command surface from `src\github_pr_kb\cli.py` +- repo hygiene guidance for `kb\`, `kb\.manifest.json`, and `.github-pr-kb\cache\` + +The plan's theory of success is reasonable: if the README accurately distinguishes workflow auth from local runtime auth, explains the copyable bootstrap model without ambiguity, preserves zero-to-working local setup, and stays aligned with the actual workflow/config/code surfaces, then a maintainer should be able to enable automation correctly and a contributor should still be able to run the tool locally from the docs alone. + +## Assumptions & Evidence + +| ID | Assumption | Explicit / Implicit | Justification status | Blast radius if wrong | Early validation | +|---|---|---|---|---|---| +| A1 | 08-02 will leave a stable enough workflow contract that 08-03 can document it without chasing moving targets. | Explicit | Partial | High | Read the final `.github\workflows\github-pr-kb.yml` and derive README assertions from it rather than from the plan text alone. | +| A2 | Simply naming `GITHUB_TOKEN`, `KB_VARIABLES_TOKEN`, `KB_VARIABLES_APP_ID`, and `KB_VARIABLES_APP_PRIVATE_KEY` is enough for maintainers to understand which credential is used where. | Implicit | Weak | Critical | Add an explicit credential-role matrix and a test that README separates local CLI auth from repository-variable auth. | +| A3 | The repo-specific command `.venv\Scripts\python.exe -m pytest tests\` belongs in a cross-platform README exactly as written. | Explicit | Unjustified | High | Either scope it as a Windows/local-repo note or provide macOS/Linux equivalents alongside it. | +| A4 | `.env.example` is already consistent with the env surface that README will describe. | Implicit | Contradicted by current evidence | High | Update `.env.example` in the same plan or make it explicitly non-authoritative and test for consistency. | +| A5 | Text-presence README contract tests are sufficient to prove the docs are operationally clear, not just keyword-complete. | Explicit | Partial | High | Add assertions for meaning, not only tokens: role separation, copy-only-workflow guidance, and committed-vs-ignored outputs. | +| A6 | The copyable bootstrap story can be compressed into README prose without readers inferring they need to vendor this repo's source tree. | Explicit | Partial | High | Add a dedicated short section or table saying exactly what consumer repos copy and what stays external. | +| A7 | PAT-first quickstart will not encourage over-scoped or long-lived use of the weaker auth path. | Implicit | Partial | Medium-High | Document the minimum fine-grained PAT permissions and recommend the GitHub App path as the longer-term option. | +| A8 | Prose alone is enough to keep users from confusing published KB artifacts with transient cache data. | Implicit | Partial | Medium-High | Add a compact "Committed vs not committed" table and pin it with tests. | + +## Ipcha Mistabra - Devil's Advocacy + +### Inversion Test + +1. **Automation-first may make the product harder to adopt, not easier.** Front-loading workflow setup, dual auth, and cross-repo bootstrap details can overwhelm a user who just wants to run the CLI locally and inspect output before committing to automation. +2. **README contract tests may increase false confidence, not correctness.** If the tests mostly assert the presence of strings, the docs can satisfy the letter of the contract while still leaving readers confused about token roles, repo boundaries, or what gets committed. +3. **PAT-first quickstart may optimize initial setup while degrading long-term security posture.** Readers often stop at the first working path. If PAT comes first without crisp minimum-scope guidance and an explicit "App is better long-term" message, the quickstart becomes the de facto permanent setup. +4. **"README alone" may be less true than the plan assumes.** With workflow bootstrap, multiple credentials, and git publication rules, a reader may still need to inspect `.env.example` or the workflow file to resolve ambiguity unless the README is unusually explicit. + +### The Little Boy from Copenhagen + +- A new maintainer will ask: "Why are there multiple tokens here, and why doesn't `${{ github.token }}` cover repository-variable writes?" +- A macOS/Linux contributor will ask: "Why does the README show only `.venv\Scripts\python.exe` as the test command?" +- A security reviewer will ask: "What exact fine-grained PAT permissions are required, and why is PAT the default path instead of the App?" +- A consumer-repo maintainer will ask: "Do I copy only `.github\workflows\github-pr-kb.yml`, or do I also need this repo's source tree, lockfile, or `.env.example`?" + +### Failure-of-Imagination Check + +- The README becomes accurate, but `.env.example` remains stale, so users copy the wrong template and then distrust the docs when classify/generate behavior does not match what they expected. +- A maintainer stores a PAT in the wrong place because the README names both `GITHUB_TOKEN` and `KB_VARIABLES_TOKEN` without a crisp responsibility split; the workflow then fails only at `gh api` or rolling-PR publication time. +- The docs correctly say `.github-pr-kb\cache\` stays out of git, but the examples do not make the publication boundary concrete enough, so readers still misunderstand the role of `kb\.manifest.json`. +- `KB_TOOL_REF` is documented as a required knob but the README never explains how to upgrade it safely, so copied workflows either stay pinned forever or users switch to a floating branch manually. + +## Risk Register + +| Risk ID | Category | Description | Trigger | Probability | Severity | Priority | Detection | Mitigation | Contingency | Assumption link | +|---|---|---|---|---|---|---|---|---|---|---| +| R1 | Security / Operational | **Known Unknown:** maintainers may confuse local CLI auth with repository-variable auth because the plan requires secret names but not a crisp explanation of credential roles. | A user follows README setup and reuses `GITHUB_TOKEN` where `KB_VARIABLES_TOKEN` or App credentials were required, or vice versa. | High - the workflow spans several similarly named auth surfaces. | High - setup fails late and can lead to over-scoped credentials. | High | `gh api` variable writes fail, support questions cluster around token setup, or users put PATs in the wrong secret. | Add a README token-role matrix covering local `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `KB_VARIABLES_TOKEN`, App creds, and the workflow's internal `GH_TOKEN` mapping; pin this with tests. | Correct the secret mapping, rotate the credential if it was exposed or over-scoped, and rerun. | A2 | +| R2 | Technical / Documentation | **Known Unknown:** the plan can turn a machine-specific contributor command into universal README guidance. | A macOS/Linux user follows `.venv\Scripts\python.exe -m pytest tests\` literally. | Medium-High | High - the README's "works from zero" promise breaks for a common audience. | High | Reader reports that the documented test command does not exist on their platform. | Provide platform-specific commands or clearly label the Windows command as repo-local guidance and include a Unix equivalent. | Patch the README quickly and point users to the working command pair. | A3 | +| R3 | Organizational / Technical | **Known Known:** README may be updated while `.env.example` remains stale, creating two contradictory onboarding sources. | 08-03 rewrites README but leaves `.env.example` comments or examples lagging behind current behavior. | High - current evidence already shows drift pressure. | High - users commonly copy `.env.example` before reading deeper docs. | High | New users copy `.env.example` and miss or misread variables that README describes differently. | Expand the plan to update `.env.example` too, or explicitly declare one source authoritative and add tests enforcing alignment. | Ship a follow-up fix to `.env.example` and note the correction in README. | A4 | +| R4 | Technical | **Unknown Unknown surfaced by inversion:** string-based README tests can pass while the docs still fail to explain the copyable two-repo bootstrap and secret responsibilities in a way humans can execute correctly. | The README contains all required strings but leaves role boundaries or workflow-copy semantics ambiguous. | Medium | High | High | Green tests but repeated user confusion around "do I need this repo's source tree?" or "which token goes where?" | Add semantic assertions: README must explicitly say consumer repos copy the workflow, not the package source tree; README must separate workflow secrets from local env vars. | Rewrite the affected section without changing product behavior. | A5, A6 | +| R5 | Security | PAT-first quickstart may normalize a broader or more fragile auth setup unless minimum permissions and migration guidance are explicit. | Readers stop at the first working path and never revisit auth hardening. | Medium | High | High | PATs with overly broad permissions appear in setup guidance or support examples. | Document the minimum fine-grained PAT permission set and frame GitHub App as the preferred long-term service-account model. | Keep PAT support, but tighten docs and recommend migration. | A7 | +| R6 | Operational | Published-output guidance may still be too abstract, causing users to misunderstand that `kb\`, `kb\.manifest.json`, and `.github-pr-kb\cache\` have different lifecycle rules. | A maintainer reads the README quickly and infers that all tool-created files are either committed or ignored together. | Medium | Medium-High | High | Cache files appear in commits, or maintainers ask whether `kb\.manifest.json` belongs in git. | Add a short table showing "Committed" vs "Not committed" artifacts and keep the KB tree example concrete. | Clean up the commit, restore the manifest if omitted, and rerun generation. | A8 | +| R7 | Operational / Maintainability | The README may expose `KB_TOOL_REF` without giving maintainers an upgrade story, making the workflow either silently stale or manually unpinned later. | A copied workflow stays on an old immutable ref, or a maintainer changes it to a floating branch for convenience. | Medium | Medium-High | Medium-High | Consumer repos diverge widely in pinned refs or start using floating branches. | Document how to bump `KB_TOOL_REF` intentionally and why it should stay immutable between upgrades. | Publish an update note and restore an immutable ref. | A1, A6 | + +## Verdict & Recommendations + +**Overall Risk Level:** High + +**Verdict:** The plan is close, but it is **not ready as written**. The biggest remaining risks are no longer about workflow mechanics; they are about documentation becoming the new failure point. In particular, the current plan does not yet force clear separation of token roles, treats a Windows-specific test command as if it were README-grade guidance, and ignores the risk of leaving `.env.example` behind as a conflicting onboarding source. + +**Top 3 Risks** + +1. R1 - credential-role confusion between local runtime auth and repository-variable auth +2. R3 - README and `.env.example` drifting into contradictory onboarding sources +3. R2 - platform-specific local test guidance presented as universal + +**Recommended Actions** + +1. Expand `files_modified` to include `.env.example`, or explicitly declare it non-authoritative and add tests that prevent README and `.env.example` drift. +2. Add a README contract test that asserts the docs distinguish local `GITHUB_TOKEN` / `ANTHROPIC_API_KEY` from `KB_VARIABLES_TOKEN` or GitHub App credentials, and explain that the workflow maps repository-variable auth to `GH_TOKEN` internally. +3. Replace the single Windows-only pytest command requirement with either paired Windows + macOS/Linux commands or wording that clearly scopes the Windows command to this repo's contributor setup. +4. Add a compact consumer-repo bootstrap explainer: what gets copied (`.github\workflows\github-pr-kb.yml`), what stays external (this tool checkout), and how to update `KB_TOOL_REF`. +5. Add a small table in README for "Committed" (`kb\`, `kb\.manifest.json`) vs "Not committed" (`.github-pr-kb\cache\`) and pin that with tests. +6. Keep automation-first ordering, but add a fast path near the top for readers who only want local CLI usage so the README does not become workflow-only in practice. + +**Open Questions** + +- Should README be the authoritative onboarding source, or must `.env.example` remain equally authoritative? +- Do we want the README to recommend GitHub App as the preferred long-term setup even while PAT stays the quickstart? +- What is the preferred cross-platform local test guidance for contributors: paired `.venv` commands, or a simpler higher-level instruction? +- Does `KB_TOOL_REF` upgrade guidance belong in the README, in workflow comments, or in both? + +**What the Plan Does Well** + +- It correctly treats 08-03 as a documentation phase that must reflect the already-shipped workflow rather than re-litigate product shape. +- It anchors the rewrite to real code and config surfaces (`cli.py`, `config.py`, `.env.example`, and the workflow contract) instead of inventing new behavior. +- It preserves the important git-hygiene story around `kb\`, `kb\.manifest.json`, and `.github-pr-kb\cache\`. +- It explicitly removes stale architecture claims from the README instead of quietly layering new guidance on top of old text. From 96706685d7f90c3451f8631c7c907d200a08dd99 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 15:10:02 +0300 Subject: [PATCH 09/28] feat: ship phase 8 workflow and docs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .env.example | 29 +- .github/workflows/github-pr-kb.yml | 327 ++++++++++++++++++ .planning/ROADMAP.md | 8 +- .planning/STATE.md | 18 +- .../08-github-action-readme/08-01-SUMMARY.md | 31 ++ .../08-github-action-readme/08-02-SUMMARY.md | 32 ++ .../08-github-action-readme/08-03-SUMMARY.md | 33 ++ .../phases/08-github-action-readme/08-UAT.md | 48 +++ README.md | 171 +++++++-- src/github_pr_kb/action_state.py | 183 ++++++++++ tests/test_action_state.py | 150 ++++++++ tests/test_action_workflow.py | 116 +++++++ tests/test_readme.py | 91 +++++ 13 files changed, 1195 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/github-pr-kb.yml create mode 100644 .planning/phases/08-github-action-readme/08-01-SUMMARY.md create mode 100644 .planning/phases/08-github-action-readme/08-02-SUMMARY.md create mode 100644 .planning/phases/08-github-action-readme/08-03-SUMMARY.md create mode 100644 .planning/phases/08-github-action-readme/08-UAT.md create mode 100644 src/github_pr_kb/action_state.py create mode 100644 tests/test_action_state.py create mode 100644 tests/test_action_workflow.py create mode 100644 tests/test_readme.py diff --git a/.env.example b/.env.example index c565637..8c17479 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,22 @@ -# Required: GitHub Personal Access Token with repo read permissions -# Generate at: https://github.com/settings/tokens -# Scopes needed: repo (or public_repo for public repos only) +# Local CLI/runtime configuration for github-pr-kb. +# This file is for local development and manual runs only. +# Workflow repository secrets belong in GitHub repository secrets, not here. +# Do not add KB_VARIABLES_TOKEN, KB_VARIABLES_APP_ID, or KB_VARIABLES_APP_PRIVATE_KEY to .env. + +# Required for local extraction and for any command that reads GitHub PR data. GITHUB_TOKEN=your_github_token_here -# Future variables (added in their respective phases): -# ANTHROPIC_API_KEY=your_anthropic_api_key_here # Phase 4: Classification -# ANTHROPIC_MODEL=claude-haiku-4-5-20251001 # Optional: override classifier model -# ANTHROPIC_GENERATE_MODEL=claude-sonnet-4-5 # Optional: override article generator model -# KB_OUTPUT_DIR=kb # Optional: change the generated knowledge base output directory -# MIN_CONFIDENCE=0.5 # Optional: generator skips classifications below this confidence +# Required for local classify/generate runs. +ANTHROPIC_API_KEY=your_anthropic_api_key_here + +# Optional classifier model override. +ANTHROPIC_MODEL=claude-haiku-4-5-20251001 + +# Optional article-generation model override. +ANTHROPIC_GENERATE_MODEL=claude-sonnet-4-5 + +# Optional output directory for generated KB files. +KB_OUTPUT_DIR=kb + +# Optional minimum confidence threshold for KB generation. +MIN_CONFIDENCE=0.5 diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml new file mode 100644 index 0000000..67c8a4d --- /dev/null +++ b/.github/workflows/github-pr-kb.yml @@ -0,0 +1,327 @@ +name: Update PR knowledge base + +on: + pull_request: + types: [closed] + workflow_dispatch: + inputs: + since: + description: Optional ISO-8601 updated_at cursor override for recovery/backfill + required: false + type: string + force: + description: Run even when no newer merged PRs are detected + required: false + default: false + type: boolean + +permissions: + contents: write + pull-requests: write + actions: read + +concurrency: + group: github-pr-kb-${{ github.repository }} + cancel-in-progress: false + +env: + KB_TOOL_REPOSITORY: galzi/github-pr-kb + KB_TOOL_REF: 61276f03d9c952c8f9e5e16288178b92ea430b2d + KB_TOOL_PATH: .github-pr-kb-tool + KB_VARIABLE_NAME: KB_LAST_SUCCESSFUL_CURSOR + KB_BOT_BRANCH: automation/github-pr-kb + KB_PR_TITLE: "chore: update PR knowledge base" + +jobs: + update-kb: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }} + runs-on: ubuntu-latest + + steps: + - name: Checkout target repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + python-version: "3.11" + + - name: Checkout github-pr-kb tool + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ env.KB_TOOL_REPOSITORY }} + ref: ${{ env.KB_TOOL_REF }} + path: ${{ env.KB_TOOL_PATH }} + fetch-depth: 1 + + - name: Install tool dependencies + run: uv sync --project .github-pr-kb-tool --all-groups --frozen + + - name: Mint repository-variable GitHub App token + id: app-token + if: ${{ secrets.KB_VARIABLES_APP_ID != '' && secrets.KB_VARIABLES_APP_PRIVATE_KEY != '' }} + uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 + with: + app-id: ${{ secrets.KB_VARIABLES_APP_ID }} + private-key: ${{ secrets.KB_VARIABLES_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + + - name: Validate repository-variable auth configuration + if: ${{ steps.app-token.outputs.token == '' && secrets.KB_VARIABLES_TOKEN == '' }} + run: | + echo "Set KB_VARIABLES_TOKEN or KB_VARIABLES_APP_ID + KB_VARIABLES_APP_PRIVATE_KEY repository secrets." >&2 + exit 1 + + - name: Restore github-pr-kb cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: .github-pr-kb/cache/ + key: github-pr-kb-cache-${{ runner.os }}-${{ github.repository }} + restore-keys: | + github-pr-kb-cache-${{ runner.os }}- + + - name: Determine publication base branch + id: base-branch + shell: bash + run: | + set -euo pipefail + if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then + echo "name=${{ github.event.repository.default_branch }}" >> "$GITHUB_OUTPUT" + else + echo "name=${{ github.event.pull_request.base.ref }}" >> "$GITHUB_OUTPUT" + fi + + - name: Read stored KB cursor + id: stored-cursor + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + run: | + set -euo pipefail + endpoint="/repos/${GITHUB_REPOSITORY}/actions/variables/${KB_VARIABLE_NAME}" + stderr_file="$(mktemp)" + set +e + stored_cursor="$(gh api "${endpoint}" --jq '.value' 2>"${stderr_file}")" + status=$? + set -e + if [ "${status}" -eq 0 ]; then + : + elif grep -q "404" "${stderr_file}"; then + stored_cursor="" + else + cat "${stderr_file}" >&2 + exit "${status}" + fi + rm -f "${stderr_file}" + echo "value=${stored_cursor}" >> "$GITHUB_OUTPUT" + + - name: Find latest merged PR updated_at for manual runs + id: latest-merged + if: ${{ github.event_name == 'workflow_dispatch' }} + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + run: | + set -euo pipefail + latest_merged_at="$( + gh api "/repos/${GITHUB_REPOSITORY}/pulls?state=closed&sort=updated&direction=desc&per_page=100" \ + --jq 'map(select(.merged_at != null))[0].updated_at // ""' + )" + echo "value=${latest_merged_at}" >> "$GITHUB_OUTPUT" + + - name: Decide whether to run the pipeline + id: decision + shell: bash + run: | + set -euo pipefail + cmd=(uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state --event-name "${GITHUB_EVENT_NAME}") + if [ -n "${{ steps.stored-cursor.outputs.value }}" ]; then + cmd+=(--stored-cursor "${{ steps.stored-cursor.outputs.value }}") + fi + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + cmd+=(--merged --event-updated-at "${{ github.event.pull_request.updated_at }}") + else + if [ -n "${{ steps.latest-merged.outputs.value }}" ]; then + cmd+=(--latest-merged-at "${{ steps.latest-merged.outputs.value }}") + fi + if [ -n "${{ github.event.inputs.since }}" ]; then + cmd+=(--manual-since "${{ github.event.inputs.since }}") + fi + if [ "${{ github.event.inputs.force }}" = "true" ]; then + cmd+=(--force) + fi + fi + decision_json="$("${cmd[@]}")" + echo "${decision_json}" + DECISION_JSON="${decision_json}" python - <<'PY' >> "$GITHUB_OUTPUT" + import json + import os + + decision = json.loads(os.environ["DECISION_JSON"]) + for key, value in decision.items(): + print(f"{key}={'' if value is None else value}") + PY + + - name: Skip no-op run + if: ${{ steps.decision.outputs.should_run != 'true' }} + shell: bash + run: | + set -euo pipefail + if [ "${{ steps.decision.outputs.reason }}" = "already_processed_auto_event" ]; then + echo "Skipping repeated merged event: already_processed_auto_event" + else + echo "Skipping pipeline: ${{ steps.decision.outputs.reason }}" + fi + + - name: Extract PR comments + if: ${{ steps.decision.outputs.should_run == 'true' }} + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + cmd=(uv run --project .github-pr-kb-tool github-pr-kb extract --repo "${GITHUB_REPOSITORY}" --state closed) + if [ -n "${{ steps.decision.outputs.extract_since }}" ]; then + cmd+=(--since "${{ steps.decision.outputs.extract_since }}") + fi + "${cmd[@]}" + + - name: Classify extracted comments + if: ${{ steps.decision.outputs.should_run == 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: uv run --project .github-pr-kb-tool github-pr-kb classify + + - name: Generate knowledge base + if: ${{ steps.decision.outputs.should_run == 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: uv run --project .github-pr-kb-tool github-pr-kb generate + + - name: Configure git identity + if: ${{ steps.decision.outputs.should_run == 'true' }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Prepare publication branch + if: ${{ steps.decision.outputs.should_run == 'true' }} + shell: bash + run: | + set -euo pipefail + git fetch --no-tags origin "refs/heads/${{ steps.base-branch.outputs.name }}:refs/remotes/origin/${{ steps.base-branch.outputs.name }}" + git fetch --no-tags origin "refs/heads/${KB_BOT_BRANCH}:refs/remotes/origin/${KB_BOT_BRANCH}" || true + git checkout -B "${KB_BOT_BRANCH}" "refs/remotes/origin/${{ steps.base-branch.outputs.name }}" + + - name: Stage generated KB output + id: stage-kb + if: ${{ steps.decision.outputs.should_run == 'true' }} + shell: bash + run: | + set -euo pipefail + git add -- 'kb/INDEX.md' 'kb/.manifest.json' ':(glob)kb/**/*.md' + if git diff --cached --name-only | grep -q '^\.github-pr-kb/cache/'; then + echo ".github-pr-kb/cache must never be staged." >&2 + exit 1 + fi + if git diff --cached --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push KB update branch + if: ${{ steps.decision.outputs.should_run == 'true' && steps.stage-kb.outputs.changed == 'true' }} + shell: bash + run: | + set -euo pipefail + git commit -m "${KB_PR_TITLE}" + git push --force-with-lease origin "${KB_BOT_BRANCH}" + + - name: Create or reuse rolling KB pull request + if: ${{ steps.decision.outputs.should_run == 'true' && steps.stage-kb.outputs.changed == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + run: | + set -euo pipefail + existing_pr="$( + gh pr list \ + --state open \ + --head "${KB_BOT_BRANCH}" \ + --base "${{ steps.base-branch.outputs.name }}" \ + --json number \ + --jq '.[0].number // ""' + )" + if [ -n "${existing_pr}" ]; then + echo "Reusing rolling KB PR #${existing_pr}" + else + gh pr create \ + --base "${{ steps.base-branch.outputs.name }}" \ + --head "${KB_BOT_BRANCH}" \ + --title "${KB_PR_TITLE}" \ + --body "Automated update of the PR knowledge base." + fi + + - name: Persist KB_LAST_SUCCESSFUL_CURSOR + if: ${{ steps.decision.outputs.should_run == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + run: | + set -euo pipefail + candidate_cursor="${{ steps.decision.outputs.next_cursor }}" + if [ -z "${candidate_cursor}" ]; then + echo "No cursor candidate emitted; skipping persistence." + exit 0 + fi + endpoint="/repos/${GITHUB_REPOSITORY}/actions/variables/${KB_VARIABLE_NAME}" + stderr_file="$(mktemp)" + set +e + fresh_cursor="$(gh api "${endpoint}" --jq '.value' 2>"${stderr_file}")" + status=$? + set -e + if [ "${status}" -eq 0 ]; then + variable_exists="true" + elif grep -q "404" "${stderr_file}"; then + fresh_cursor="" + variable_exists="false" + else + cat "${stderr_file}" >&2 + exit "${status}" + fi + rm -f "${stderr_file}" + persisted_cursor="$( + python - "${fresh_cursor}" "${candidate_cursor}" <<'PY' + from datetime import datetime, timezone + import sys + + values = [] + for raw in sys.argv[1:]: + if not raw: + continue + dt = datetime.fromisoformat(raw.replace("Z", "+00:00")).astimezone(timezone.utc) + values.append(dt) + if values: + print(max(values).isoformat().replace("+00:00", "Z")) + PY + )" + if [ "${variable_exists}" = "true" ]; then + gh api --method PATCH "${endpoint}" -f name="${KB_VARIABLE_NAME}" -f value="${persisted_cursor}" >/dev/null + else + gh api --method POST "/repos/${GITHUB_REPOSITORY}/actions/variables" -f name="${KB_VARIABLE_NAME}" -f value="${persisted_cursor}" >/dev/null + fi + + - name: Upload debug artifacts on failure + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: github-pr-kb-debug + path: | + kb/ + .github-pr-kb/cache/ diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 7ccb781..d06587f 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -154,9 +154,9 @@ Plans: **Plans**: 3 plans Plans: -- [ ] 08-01-PLAN.md — testable action-state helper for no-new-PR guard and cursor decisions -- [ ] 08-02-PLAN.md — merged-PR workflow with dual-mode variable auth, cache reuse, and rolling KB PR publication -- [ ] 08-03-PLAN.md — automation-first README rewrite with PAT quickstart, GitHub App option, and local CLI docs +- [x] 08-01-PLAN.md — testable action-state helper for no-new-PR guard and cursor decisions +- [x] 08-02-PLAN.md — merged-PR workflow with dual-mode variable auth, cache reuse, and rolling KB PR publication +- [x] 08-03-PLAN.md — automation-first README rewrite with PAT quickstart, GitHub App option, and local CLI docs ## Progress @@ -172,4 +172,4 @@ Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 | 5. KB Generator | 2/2 | Complete | 2026-04-06 | | 6. CLI Integration | 1/1 | Complete | 2026-04-06 | | 7. Fix Article Generation Quality | 3/3 | Ready for verification | - | -| 8. GitHub Action + README | 0/3 | Not started | - | +| 8. GitHub Action + README | 3/3 | Ready for verification | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 4aae2c8..34d359c 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -3,14 +3,14 @@ gsd_state_version: 1.0 milestone: v1.0 milestone_name: milestone status: verifying -stopped_at: Phase 8 context gathered -last_updated: "2026-04-08T21:27:59.165Z" -last_activity: 2026-04-08 +stopped_at: Phase 8 plans complete +last_updated: "2026-04-10T09:49:16.639Z" +last_activity: 2026-04-10 progress: total_phases: 8 completed_phases: 7 - total_plans: 12 - completed_plans: 12 + total_plans: 15 + completed_plans: 15 percent: 100 --- @@ -21,16 +21,16 @@ progress: See: .planning/PROJECT.md (updated 2026-03-07) **Core value:** Preserve and make discoverable architectural decisions, code patterns, gotchas, and domain knowledge from PR discussions before they get lost in closed threads. -**Current focus:** Phase 07 — fix-article-generation-quality +**Current focus:** Phase 08 — github-action-readme ## Current Position -Phase: 7 +Phase: 8 Plan: 03 complete Status: Plans complete — ready for verification -Last activity: 2026-04-08 +Last activity: 2026-04-10 -Progress: [██░░░░░░░░] 29% +Progress: [██████████] 100% ## Performance Metrics diff --git a/.planning/phases/08-github-action-readme/08-01-SUMMARY.md b/.planning/phases/08-github-action-readme/08-01-SUMMARY.md new file mode 100644 index 0000000..bb2f8f7 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-01-SUMMARY.md @@ -0,0 +1,31 @@ +--- +phase: 08-github-action-readme +plan: 01 +subsystem: automation-helper +tags: [github-actions, cursor, automation, pytest] +requirements-completed: [ACTION-02, ACTION-03] +completed: 2026-04-10 +key-files: + created: + - src/github_pr_kb/action_state.py + - tests/test_action_state.py +key-decisions: + - "The workflow skip/cursor contract lives in a stdlib-only Python helper instead of opaque YAML." + - "Merged-event and manual-dispatch logic use separate formulas, both normalized to UTC ISO timestamps." + - "next_cursor is emitted as a candidate only; workflow persistence must re-read and write monotonically." +--- + +# Phase 8 Plan 01: Action State Helper Summary + +**Added a standalone `github_pr_kb.action_state` module that makes workflow run/skip decisions testable and deterministic.** + +## Accomplishments + +- Implemented `decide_action_run(...)` for merged-event and manual-dispatch paths +- Added a `python -m github_pr_kb.action_state` JSON CLI for workflow consumption +- Locked malformed or timezone-naive timestamp handling to deterministic non-zero exits +- Added focused pytest coverage for auto no-op, manual backfill, force runs, and isolated CLI execution + +## Outcome + +Phase 8 now has a reusable helper for cost-aware skipping and monotonic cursor candidates without importing runtime settings or other env-bound surfaces. diff --git a/.planning/phases/08-github-action-readme/08-02-SUMMARY.md b/.planning/phases/08-github-action-readme/08-02-SUMMARY.md new file mode 100644 index 0000000..955741f --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-02-SUMMARY.md @@ -0,0 +1,32 @@ +--- +phase: 08-github-action-readme +plan: 02 +subsystem: github-actions +tags: [github-actions, workflow, cache, gh-cli, pytest] +requirements-completed: [ACTION-01, ACTION-02, ACTION-03] +completed: 2026-04-10 +key-files: + created: + - .github/workflows/github-pr-kb.yml + - tests/test_action_workflow.py +key-decisions: + - "The shipped workflow is copyable into consumer repos because it bootstraps this tool from a second checkout." + - "Repository-variable auth prefers the GitHub App path and falls back to KB_VARIABLES_TOKEN." + - "Cursor persistence re-reads the latest repository variable and stores the monotonic max after publication succeeds." +--- + +# Phase 8 Plan 02: Workflow Automation Summary + +**Shipped the merged-PR plus `workflow_dispatch` automation workflow, including cache restore, rolling PR publication, and monotonic cursor persistence.** + +## Accomplishments + +- Added `.github/workflows/github-pr-kb.yml` with merged-only trigger handling and manual backfill inputs +- Bootstrapped the CLI from a pinned checkout of `galzi/github-pr-kb` via `KB_TOOL_REPOSITORY` and `KB_TOOL_REF` +- Wired explicit `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, and `GH_TOKEN` responsibilities +- Restored `.github-pr-kb/cache/` through Actions cache and staged only KB publication outputs +- Added contract tests covering triggers, SHA-pinned actions, helper invocation shape, rolling PR behavior, and cursor endpoints + +## Outcome + +The repository now ships the actual GitHub Actions automation required for incremental KB updates without committing transient cache data or burying the skip logic in YAML alone. diff --git a/.planning/phases/08-github-action-readme/08-03-SUMMARY.md b/.planning/phases/08-github-action-readme/08-03-SUMMARY.md new file mode 100644 index 0000000..aa77ada --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-03-SUMMARY.md @@ -0,0 +1,33 @@ +--- +phase: 08-github-action-readme +plan: 03 +subsystem: documentation +tags: [readme, docs, env, pytest] +requirements-completed: [INFRA-03] +completed: 2026-04-10 +key-files: + modified: + - README.md + - .env.example + created: + - tests/test_readme.py +key-decisions: + - "README is automation-first and documents the shipped workflow before local CLI usage." + - "PAT quickstart is documented before the GitHub App path, but credential roles stay explicitly separated." + - ".env.example documents only the local config surface and points workflow-only secrets to repository settings." +--- + +# Phase 8 Plan 03: README Rewrite Summary + +**Rewrote the README around the shipped workflow and aligned `.env.example` with the real local config surface.** + +## Accomplishments + +- Added README contract tests for automation-first ordering, credential-role clarity, uv install guidance, and git boundaries +- Replaced the placeholder README with setup instructions for the copyable workflow, PAT quickstart, GitHub App option, and local CLI usage +- Documented `kb/` plus `kb/.manifest.json` as committed output and `.github-pr-kb/cache/` as non-git working data +- Updated `.env.example` to cover `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL`, `ANTHROPIC_GENERATE_MODEL`, `KB_OUTPUT_DIR`, and `MIN_CONFIDENCE` + +## Outcome + +The docs now match the shipped product: maintainers can configure automation correctly, and local users can get from zero to a working install from the README alone. diff --git a/.planning/phases/08-github-action-readme/08-UAT.md b/.planning/phases/08-github-action-readme/08-UAT.md new file mode 100644 index 0000000..1dafb2b --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-UAT.md @@ -0,0 +1,48 @@ +--- +status: testing +phase: 08-github-action-readme +source: + - 08-01-SUMMARY.md + - 08-02-SUMMARY.md + - 08-03-SUMMARY.md +started: 2026-04-10T09:49:16Z +updated: 2026-04-10T09:49:16Z +--- + +## Current Test + +number: 1 +name: Copyable workflow bootstrap +expected: | + The shipped `.github/workflows/github-pr-kb.yml` can be copied into another repository, triggers on merged PRs plus `workflow_dispatch`, and bootstraps this tool from `.github-pr-kb-tool` using `KB_TOOL_REPOSITORY` and `KB_TOOL_REF` instead of assuming the consumer repo vendors this package. +awaiting: user response + +## Tests + +### 1. Copyable workflow bootstrap +expected: The shipped workflow is copy-ready for consumer repos and bootstraps the tool from `.github-pr-kb-tool` via `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`. +result: pending + +### 2. No-new-PR guard and cursor contract +expected: Repeated merged events and manual no-new-PR runs skip before extract/classify/generate, and successful runs use `KB_LAST_SUCCESSFUL_CURSOR` as a monotonic state boundary. +result: pending + +### 3. Workflow auth and publication contract +expected: The workflow clearly separates CLI/runtime auth from repository-variable auth, stages only `kb/` output plus `kb/.manifest.json`, and uses one rolling PR on `automation/github-pr-kb`. +result: pending + +### 4. README local onboarding +expected: README leads with automation, then gives local uv install, env vars, CLI commands, KB output example, and committed-vs-not-committed guidance that matches the shipped workflow. +result: pending + +## Summary + +total: 4 +passed: 0 +issues: 0 +pending: 4 +skipped: 0 + +## Gaps + +None yet. diff --git a/README.md b/README.md index c6e8ed7..cd4357b 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,167 @@ # github-pr-kb -A tool for extracting GitHub PR comments into a knowledgebase for AI agents (and humans) -## The vision: +Extract durable knowledge from GitHub pull request discussions, classify the useful comments, and publish the result as a markdown knowledge base. -You know this feeling. You are a software engineer working on a product with the rest of your team of talented software engineers. Each time any of you create a PR on GitHub for one of your repositories of your product, you get comments on it from your peers, fix issues, and merge. +## Automate with GitHub Actions -Once the PR get merged, all you see in the "regular" git worktree and history are the commits and commits messages. The PR comments went to waste, with all the organizational and product wisdom they contained. +The shipped workflow lives at `.github/workflows/github-pr-kb.yml`. It runs after merged PRs (`pull_request` with `types: [closed]` and a merged-only guard) and also supports `workflow_dispatch` for recovery and backfill runs. -This CLI tool allows you to analyze all the PRs (that got merged) in a GitHub repository that have 1 or more comments on them, take the interesting comments there, and pass them to LLM for analysis and building/updating the knowledgebase accordingly. +### Copy the workflow into a consumer repository -## Options: -- Selection between processing the X most recent PRs and processing all PRs starting from a specific date. -- Configure specific people that their opinions will matter more than others. -- Configure specific people/agents to ignore. +1. Copy `.github/workflows/github-pr-kb.yml` into the target repository. +2. Add the required repository secrets. +3. Merge the workflow and let future merged PRs update the KB automatically. -## Tech Stack: -- Python -- OpenRouter SDK -- Chroma vector DB -- GitHub API -- TOON (Token-Oriented Object Notation) for passing the data to the LLM +Consumer repositories **copy only the workflow file**. They **do not need this tool's source tree checked into your repository** because the workflow performs a second checkout of this repo into `.github-pr-kb-tool`, installs the tool there, and runs every `github-pr-kb` command from that checkout. -## Development +### Tool bootstrap and upgrades + +The workflow is intentionally copyable across repositories and uses two explicit bootstrap settings: + +| Setting | Default | Purpose | +| --- | --- | --- | +| `KB_TOOL_REPOSITORY` | `galzi/github-pr-kb` | Which repository to checkout for the CLI and helper code | +| `KB_TOOL_REF` | immutable full commit SHA | Which exact tool version to run | + +`KB_TOOL_REF` should stay pinned to an immutable release tag or full commit SHA. Treat it like any other supply-chain pin: update it intentionally, review the diff, and commit the workflow change when you want to upgrade. + +### Credential roles + +The docs separate **local/runtime credentials** from **workflow repository-variable credentials** because they do different jobs: + +| Credential | Used by | Where it belongs | +| --- | --- | --- | +| `GITHUB_TOKEN` | Local CLI extraction, or the workflow's extract/classify/generate steps via `${{ github.token }}` | Local `.env` for manual runs; GitHub-provided token inside Actions | +| `ANTHROPIC_API_KEY` | Local `classify` and `generate`, plus workflow classification/generation | Local `.env` and repository secret | +| `KB_VARIABLES_TOKEN` | Quickstart auth for workflow `gh api` and `gh pr` calls | Repository secret only | +| `KB_VARIABLES_APP_ID` + `KB_VARIABLES_APP_PRIVATE_KEY` | Advanced GitHub App auth for workflow `gh api` and `gh pr` calls | Repository secrets only | + +Inside the workflow, the resolved repository-variable credential is mapped to `GH_TOKEN` for `gh api` and `gh pr`, while the CLI steps receive `GITHUB_TOKEN` and `ANTHROPIC_API_KEY` separately. + +### PAT quickstart + +Set these repository secrets in the consumer repository: + +| Secret | Why | +| --- | --- | +| `ANTHROPIC_API_KEY` | Required for comment classification and article generation | +| `KB_VARIABLES_TOKEN` | Auth for repository variable reads/writes and rolling PR publication | + +For a fine-grained PAT quickstart, grant the token the minimum repository permissions needed for this workflow: + +- `Variables: Read and write` +- `Contents: Read and write` +- `Pull requests: Read and write` + +This is the fastest setup path and is documented first on purpose. + +### GitHub App + +For a longer-lived service-account setup, configure these repository secrets instead of `KB_VARIABLES_TOKEN`: + +| Secret | Why | +| --- | --- | +| `KB_VARIABLES_APP_ID` | GitHub App identifier | +| `KB_VARIABLES_APP_PRIVATE_KEY` | Private key used to mint an installation token | + +The workflow prefers the GitHub App path when both the app secrets and `KB_VARIABLES_TOKEN` are present, and falls back to `KB_VARIABLES_TOKEN` otherwise. + +### Manual recovery and backfill + +Use `workflow_dispatch` when you need to recover from a failed run or backfill older merged PRs. + +| Input | Meaning | +| --- | --- | +| `since` | Optional ISO-8601 `updated_at` cursor override for backfill | +| `force` | Run even when no newer merged PRs are detected | + +### Committed vs not committed + +The workflow keeps publication output and working cache separate: + +| Path | Git status | Why | +| --- | --- | --- | +| `kb/INDEX.md` | committed | Top-level KB index | +| `kb/**/*.md` | committed | Generated KB articles | +| `kb/.manifest.json` | committed | Generator dedup state required for incremental correctness | +| `.github-pr-kb/cache/` | not committed | Transient extract/classify cache for local runs and Actions cache reuse | + +## Run locally + +You can also run the tool directly without GitHub Actions. + +### Install uv + +Install uv first. Official docs: https://docs.astral.sh/uv/getting-started/installation/ + +```powershell +powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +``` + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +### Configure local environment + +Copy `.env.example` to `.env`. `.env.example` documents the local config surface only; it is not a workflow secret template. + +| Variable | Required | Purpose | +| --- | --- | --- | +| `GITHUB_TOKEN` | yes | GitHub API auth for extraction | +| `ANTHROPIC_API_KEY` | for `classify`/`generate` | Anthropic API auth | +| `ANTHROPIC_MODEL` | no | Override the classifier model | +| `ANTHROPIC_GENERATE_MODEL` | no | Override the article-generation model | +| `KB_OUTPUT_DIR` | no | Output directory for generated KB content | +| `MIN_CONFIDENCE` | no | Minimum confidence threshold for generated articles | + +Workflow-only secrets such as `KB_VARIABLES_TOKEN`, `KB_VARIABLES_APP_ID`, and `KB_VARIABLES_APP_PRIVATE_KEY` belong in repository secrets, not in `.env`. + +### Install dependencies ```bash -uv sync --all-groups # install dependencies including dev tools -uv run pre-commit install # activate the git pre-commit hook (one-time setup) +uv sync --all-groups --frozen ``` -The pre-commit hook runs `ruff` on staged files before each commit, auto-fixing what it can. If fixes are applied, the commit is blocked so you can review and re-stage them. +### Run tests + +Use the venv Python directly: + +```powershell +.venv/Scripts/python.exe -m pytest tests/ +``` -To run linting manually across the whole codebase: ```bash +.venv/bin/python -m pytest tests/ +``` + +### CLI commands + +```bash +github-pr-kb extract --repo owner/name +github-pr-kb extract --repo owner/name --state closed --since 2024-01-01 +github-pr-kb classify +github-pr-kb generate +github-pr-kb run --repo owner/name +``` + +### Example KB output + +```text +kb/ + INDEX.md + .manifest.json + architecture_decision/ + prefer-monotonic-cursor-updates.md + code_pattern/ + keep-cli-imports-lazy.md +``` + +## Development + +```bash +uv run pre-commit install uv run pre-commit run --all-files ``` + +The pre-commit hook is Ruff-only and may rewrite files before asking you to re-stage them. diff --git a/src/github_pr_kb/action_state.py b/src/github_pr_kb/action_state.py new file mode 100644 index 0000000..9d82916 --- /dev/null +++ b/src/github_pr_kb/action_state.py @@ -0,0 +1,183 @@ +"""Workflow decision helper for GitHub Action skip/cursor logic. + +This module is intentionally stdlib-only and import-safe: it does not import +repo settings or any other env-bound surfaces, so workflow consumers can run +`python -m github_pr_kb.action_state` before any repository secrets or local +CLI configuration are available. + +`next_cursor` is a candidate value only. The workflow must persist it +monotonically against the freshest repository variable value after successful +publication so overlapping runs cannot move state backward. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import asdict, dataclass +from datetime import datetime, timezone + +TimestampInput = datetime | str | None + + +@dataclass(frozen=True, slots=True) +class ActionRunDecision: + should_run: bool + extract_since: str | None + next_cursor: str | None + reason: str + + def to_json_dict(self) -> dict[str, bool | str | None]: + return asdict(self) + + +def _normalize_timestamp(value: TimestampInput, field_name: str) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + dt = value + else: + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError(f"{field_name} must be an ISO-8601 timestamp with timezone") from exc + + if dt.tzinfo is None or dt.utcoffset() is None: + raise ValueError(f"{field_name} must be an ISO-8601 timestamp with timezone") + return dt.astimezone(timezone.utc) + + +def _isoformat_utc(value: datetime | None) -> str | None: + if value is None: + return None + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def decide_action_run( + *, + event_name: str, + merged: bool = False, + event_updated_at: TimestampInput = None, + stored_cursor: TimestampInput = None, + latest_merged_at: TimestampInput = None, + manual_since: TimestampInput = None, + force: bool = False, +) -> ActionRunDecision: + """Decide whether the workflow should run and which cursor values to emit. + + Auto merged-event path: + - compare `event_updated_at` to `stored_cursor` + - use `stored_cursor` as `extract_since` + - emit `event_updated_at` as the next successful cursor candidate + + Manual workflow-dispatch path: + - use `manual_since` if provided, otherwise `stored_cursor`, as `extract_since` + - unless forced, skip when `latest_merged_at` is not newer than that effective cursor + - emit `max(stored_cursor, manual_since, latest_merged_at)` as the next cursor candidate + """ + + stored = _normalize_timestamp(stored_cursor, "--stored-cursor") + manual = _normalize_timestamp(manual_since, "--manual-since") + latest = _normalize_timestamp(latest_merged_at, "--latest-merged-at") + event_updated = _normalize_timestamp(event_updated_at, "--event-updated-at") + + if event_name == "pull_request": + if not merged: + return ActionRunDecision( + should_run=False, + extract_since=_isoformat_utc(stored), + next_cursor=_isoformat_utc(stored), + reason="pull_request_not_merged", + ) + if event_updated is None: + raise ValueError("--event-updated-at is required for merged pull_request events") + if stored is not None and event_updated <= stored: + return ActionRunDecision( + should_run=False, + extract_since=_isoformat_utc(stored), + next_cursor=_isoformat_utc(max(stored, event_updated)), + reason="already_processed_auto_event", + ) + return ActionRunDecision( + should_run=True, + extract_since=_isoformat_utc(stored), + next_cursor=_isoformat_utc(event_updated), + reason="new_merged_pr_event", + ) + + if event_name != "workflow_dispatch": + raise ValueError("--event-name must be pull_request or workflow_dispatch") + + effective_cursor = manual if manual is not None else stored + cursor_candidates = [dt for dt in (stored, manual, latest) if dt is not None] + next_cursor = max(cursor_candidates) if cursor_candidates else None + + if force: + return ActionRunDecision( + should_run=True, + extract_since=_isoformat_utc(effective_cursor), + next_cursor=_isoformat_utc(next_cursor), + reason="forced_manual_run", + ) + + if latest is None: + return ActionRunDecision( + should_run=False, + extract_since=_isoformat_utc(effective_cursor), + next_cursor=_isoformat_utc(next_cursor), + reason="no_new_closed_prs", + ) + + if effective_cursor is not None and latest <= effective_cursor: + return ActionRunDecision( + should_run=False, + extract_since=_isoformat_utc(effective_cursor), + next_cursor=_isoformat_utc(next_cursor), + reason="no_new_closed_prs", + ) + + return ActionRunDecision( + should_run=True, + extract_since=_isoformat_utc(effective_cursor), + next_cursor=_isoformat_utc(next_cursor), + reason="new_closed_prs_available", + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Emit workflow skip/cursor decisions as JSON.") + parser.add_argument("--event-name", required=True) + parser.add_argument("--merged", action="store_true") + parser.add_argument("--event-updated-at") + parser.add_argument("--stored-cursor") + parser.add_argument("--latest-merged-at") + parser.add_argument("--manual-since") + parser.add_argument("--force", action="store_true") + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + try: + decision = decide_action_run( + event_name=args.event_name, + merged=args.merged, + event_updated_at=args.event_updated_at, + stored_cursor=args.stored_cursor, + latest_merged_at=args.latest_merged_at, + manual_since=args.manual_since, + force=args.force, + ) + except ValueError as exc: + print(f"Invalid input: {exc}", file=sys.stderr) + return 2 + + print(json.dumps(decision.to_json_dict())) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_action_state.py b/tests/test_action_state.py new file mode 100644 index 0000000..ebd3561 --- /dev/null +++ b/tests/test_action_state.py @@ -0,0 +1,150 @@ +import json +import os +import subprocess +import sys +from datetime import datetime, timezone + +from github_pr_kb.action_state import decide_action_run + + +def _dt(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + + +def test_merged_pr_event_uses_event_updated_at() -> None: + decision = decide_action_run( + event_name="pull_request", + merged=True, + event_updated_at=_dt("2026-04-10T12:00:00Z"), + stored_cursor=_dt("2026-04-09T12:00:00Z"), + ) + + assert decision.should_run is True + assert decision.extract_since == "2026-04-09T12:00:00Z" + assert decision.next_cursor == "2026-04-10T12:00:00Z" + assert decision.reason == "new_merged_pr_event" + + +def test_repeated_auto_event_is_no_op_when_cursor_matches() -> None: + decision = decide_action_run( + event_name="pull_request", + merged=True, + event_updated_at=_dt("2026-04-10T12:00:00Z"), + stored_cursor=_dt("2026-04-10T12:00:00Z"), + ) + + assert decision.should_run is False + assert decision.extract_since == "2026-04-10T12:00:00Z" + assert decision.next_cursor == "2026-04-10T12:00:00Z" + assert decision.reason == "already_processed_auto_event" + + +def test_manual_dispatch_skips_when_cursor_is_current() -> None: + decision = decide_action_run( + event_name="workflow_dispatch", + stored_cursor=_dt("2026-04-10T12:00:00Z"), + latest_merged_at=_dt("2026-04-10T11:00:00Z"), + ) + + assert decision.should_run is False + assert decision.extract_since == "2026-04-10T12:00:00Z" + assert decision.next_cursor == "2026-04-10T12:00:00Z" + assert decision.reason == "no_new_closed_prs" + + +def test_manual_since_override_wins_over_stored_cursor() -> None: + decision = decide_action_run( + event_name="workflow_dispatch", + stored_cursor=_dt("2026-04-10T12:00:00Z"), + latest_merged_at=_dt("2026-04-12T09:00:00Z"), + manual_since=_dt("2026-04-01T00:00:00Z"), + ) + + assert decision.should_run is True + assert decision.extract_since == "2026-04-01T00:00:00Z" + assert decision.next_cursor == "2026-04-12T09:00:00Z" + assert decision.reason == "new_closed_prs_available" + + +def test_manual_next_cursor_never_regresses_past_stored_cursor() -> None: + decision = decide_action_run( + event_name="workflow_dispatch", + stored_cursor=_dt("2026-04-15T12:00:00Z"), + latest_merged_at=_dt("2026-04-12T09:00:00Z"), + manual_since=_dt("2026-04-01T00:00:00Z"), + ) + + assert decision.should_run is True + assert decision.extract_since == "2026-04-01T00:00:00Z" + assert decision.next_cursor == "2026-04-15T12:00:00Z" + assert decision.reason == "new_closed_prs_available" + + +def test_force_manual_run_bypasses_no_new_pr_guard() -> None: + decision = decide_action_run( + event_name="workflow_dispatch", + stored_cursor=_dt("2026-04-10T12:00:00Z"), + latest_merged_at=_dt("2026-04-10T11:00:00Z"), + manual_since=_dt("2026-04-05T00:00:00Z"), + force=True, + ) + + assert decision.should_run is True + assert decision.extract_since == "2026-04-05T00:00:00Z" + assert decision.next_cursor == "2026-04-10T12:00:00Z" + assert decision.reason == "forced_manual_run" + + +def test_action_state_cli_runs_without_repo_or_api_secrets() -> None: + env = os.environ.copy() + env.pop("GITHUB_TOKEN", None) + env.pop("ANTHROPIC_API_KEY", None) + + result = subprocess.run( + [ + sys.executable, + "-m", + "github_pr_kb.action_state", + "--event-name", + "workflow_dispatch", + "--latest-merged-at", + "2026-04-10T12:00:00Z", + "--manual-since", + "2026-04-01T00:00:00Z", + ], + capture_output=True, + check=False, + env=env, + text=True, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload == { + "should_run": True, + "extract_since": "2026-04-01T00:00:00Z", + "next_cursor": "2026-04-10T12:00:00Z", + "reason": "new_closed_prs_available", + } + + +def test_action_state_cli_rejects_malformed_timestamp_input() -> None: + result = subprocess.run( + [ + sys.executable, + "-m", + "github_pr_kb.action_state", + "--event-name", + "pull_request", + "--merged", + "--event-updated-at", + "not-a-timestamp", + ], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode != 0 + assert result.stdout == "" + assert "Invalid input: --event-updated-at must be an ISO-8601 timestamp with timezone" in result.stderr diff --git a/tests/test_action_workflow.py b/tests/test_action_workflow.py new file mode 100644 index 0000000..f747acc --- /dev/null +++ b/tests/test_action_workflow.py @@ -0,0 +1,116 @@ +import json +import re +import subprocess +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "github-pr-kb.yml" + + +def _workflow_text() -> str: + return WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _assert_pinned_action(text: str, action: str) -> None: + pattern = rf"uses:\s+{re.escape(action)}@[0-9a-f]{{40}}(?:\s+#.*)?" + assert re.search(pattern, text), f"{action} must be pinned by full commit SHA" + + +def test_workflow_has_merged_pr_and_dispatch_triggers() -> None: + text = _workflow_text() + + assert "pull_request:" in text + assert "types: [closed]" in text + assert "workflow_dispatch:" in text + assert "since:" in text + assert "force:" in text + assert "schedule:" not in text + assert "github.event.pull_request.merged == true" in text + assert "concurrency:" in text + assert "cancel-in-progress: false" in text + assert "KB_LAST_SUCCESSFUL_CURSOR" in text + + +def test_workflow_bootstraps_tool_checkout_for_copyable_repos() -> None: + text = _workflow_text() + + assert "KB_TOOL_REPOSITORY" in text + assert "KB_TOOL_REF" in text + assert "galzi/github-pr-kb" in text + assert "refs/heads/main" not in text + assert "KB_TOOL_REF: main" not in text + assert "KB_TOOL_REF: master" not in text + assert "uv sync --project .github-pr-kb-tool --all-groups --frozen" in text + assert "uv run --project .github-pr-kb-tool python -m github_pr_kb.action_state" in text + assert "uv run --project .github-pr-kb-tool github-pr-kb extract" in text + assert "uv run --project .github-pr-kb-tool github-pr-kb classify" in text + assert "uv run --project .github-pr-kb-tool github-pr-kb generate" in text + assert ".github-pr-kb/cache/" in text + assert "--event-updated-at" in text + assert "--stored-cursor" in text + assert "--latest-merged-at" in text + assert "--manual-since" in text + assert "--force" in text + assert "already_processed_auto_event" in text + assert "GITHUB_TOKEN:" in text + assert "ANTHROPIC_API_KEY:" in text + assert "GH_TOKEN:" in text + assert "KB_VARIABLES_TOKEN" in text + assert "KB_VARIABLES_APP_ID" in text + assert "KB_VARIABLES_APP_PRIVATE_KEY" in text + assert "actions/create-github-app-token" in text + + for action in ( + "actions/checkout", + "astral-sh/setup-uv", + "actions/cache", + "actions/upload-artifact", + "actions/create-github-app-token", + ): + _assert_pinned_action(text, action) + + +def test_workflow_publishes_rolling_pr_and_persists_cursor_monotonically() -> None: + text = _workflow_text() + + assert "automation/github-pr-kb" in text + assert "chore: update PR knowledge base" in text + assert "gh api" in text + assert "gh pr" in text + assert "kb/INDEX.md" in text + assert "kb/.manifest.json" in text + assert ":(glob)kb/**/*.md" in text + git_add_line = next(line for line in text.splitlines() if "git add --" in line) + assert ".github-pr-kb/cache" not in git_add_line + assert "actions/variables/${KB_VARIABLE_NAME}" in text + assert "KB_LAST_SUCCESSFUL_CURSOR" in text + assert "max(" in text or "python - <<'PY'" in text + + +def test_action_state_helper_command_runs_from_project_checkout() -> None: + result = subprocess.run( + [ + "uv", + "run", + "--project", + str(REPO_ROOT), + "python", + "-m", + "github_pr_kb.action_state", + "--event-name", + "pull_request", + "--merged", + "--event-updated-at", + "2026-04-10T12:00:00Z", + "--stored-cursor", + "2026-04-09T12:00:00Z", + ], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert set(payload) == {"should_run", "extract_since", "next_cursor", "reason"} + assert payload["should_run"] is True diff --git a/tests/test_readme.py b/tests/test_readme.py new file mode 100644 index 0000000..730b71f --- /dev/null +++ b/tests/test_readme.py @@ -0,0 +1,91 @@ +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +README_PATH = REPO_ROOT / "README.md" +ENV_EXAMPLE_PATH = REPO_ROOT / ".env.example" + + +def _readme() -> str: + return README_PATH.read_text(encoding="utf-8") + + +def _env_example() -> str: + return ENV_EXAMPLE_PATH.read_text(encoding="utf-8") + + +def test_readme_is_automation_first() -> None: + text = _readme() + + assert "## Automate with GitHub Actions" in text + assert "## Run locally" in text + assert text.index("## Automate with GitHub Actions") < text.index("## Run locally") + assert ".github/workflows/github-pr-kb.yml" in text + assert "workflow_dispatch" in text + assert "KB_TOOL_REPOSITORY" in text + assert "KB_TOOL_REF" in text + assert "copy only the workflow file" in text + assert "do not need this tool's source tree checked into your repository" in text + assert "### PAT quickstart" in text + assert "### GitHub App" in text + assert text.index("KB_VARIABLES_TOKEN") < text.index("KB_VARIABLES_APP_ID") + + +def test_readme_distinguishes_local_and_workflow_credentials() -> None: + text = _readme() + + assert "GITHUB_TOKEN" in text + assert "ANTHROPIC_API_KEY" in text + assert "KB_VARIABLES_TOKEN" in text + assert "KB_VARIABLES_APP_ID" in text + assert "KB_VARIABLES_APP_PRIVATE_KEY" in text + assert "GH_TOKEN" in text + assert "gh api" in text + assert "gh pr" in text + assert "local/runtime credentials" in text + assert "workflow repository-variable credentials" in text + + +def test_readme_documents_install_local_usage_and_git_boundaries() -> None: + text = _readme() + + assert "### Install uv" in text + assert text.index("### Install uv") < text.index("uv sync --all-groups --frozen") + assert "https://docs.astral.sh/uv/getting-started/installation/" in text + assert 'powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"' in text + assert "curl -LsSf https://astral.sh/uv/install.sh | sh" in text + assert ".venv/Scripts/python.exe -m pytest tests/" in text + assert ".venv/bin/python -m pytest tests/" in text + for token in ( + "GITHUB_TOKEN", + "ANTHROPIC_API_KEY", + "ANTHROPIC_MODEL", + "ANTHROPIC_GENERATE_MODEL", + "KB_OUTPUT_DIR", + "MIN_CONFIDENCE", + ): + assert token in text + assert "github-pr-kb extract --repo owner/name" in text + assert "github-pr-kb classify" in text + assert "github-pr-kb generate" in text + assert ".github-pr-kb/cache/" in text + assert "kb/.manifest.json" in text + assert "Committed vs not committed" in text + assert ".env.example" in text + + +def test_env_example_matches_local_config_surface_only() -> None: + text = _env_example() + + for token in ( + "GITHUB_TOKEN", + "ANTHROPIC_API_KEY", + "ANTHROPIC_MODEL", + "ANTHROPIC_GENERATE_MODEL", + "KB_OUTPUT_DIR", + "MIN_CONFIDENCE", + ): + assert token in text + assert "KB_VARIABLES_TOKEN=" not in text + assert "KB_VARIABLES_APP_ID=" not in text + assert "KB_VARIABLES_APP_PRIVATE_KEY=" not in text + assert "repository secrets" in text From f556828aef5bfd5c0981d3c22cca62d989852df4 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 15:10:53 +0300 Subject: [PATCH 10/28] chore: repin workflow bootstrap ref Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/github-pr-kb.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml index 67c8a4d..e4a315c 100644 --- a/.github/workflows/github-pr-kb.yml +++ b/.github/workflows/github-pr-kb.yml @@ -26,7 +26,7 @@ concurrency: env: KB_TOOL_REPOSITORY: galzi/github-pr-kb - KB_TOOL_REF: 61276f03d9c952c8f9e5e16288178b92ea430b2d + KB_TOOL_REF: 96706685d7f90c3451f8631c7c907d200a08dd99 KB_TOOL_PATH: .github-pr-kb-tool KB_VARIABLE_NAME: KB_LAST_SUCCESSFUL_CURSOR KB_BOT_BRANCH: automation/github-pr-kb From 87c68bdedfbfa76399acc3b96e4c32ac93b528d6 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 15:15:58 +0300 Subject: [PATCH 11/28] fix: allow workflow secret checks to parse Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/github-pr-kb.yml | 20 ++++++++++++-------- tests/test_action_workflow.py | 3 +++ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml index e4a315c..a75e551 100644 --- a/.github/workflows/github-pr-kb.yml +++ b/.github/workflows/github-pr-kb.yml @@ -36,6 +36,10 @@ jobs: update-kb: if: ${{ github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true }} runs-on: ubuntu-latest + env: + KB_VARIABLES_TOKEN: ${{ secrets.KB_VARIABLES_TOKEN }} + KB_VARIABLES_APP_ID: ${{ secrets.KB_VARIABLES_APP_ID }} + KB_VARIABLES_APP_PRIVATE_KEY: ${{ secrets.KB_VARIABLES_APP_PRIVATE_KEY }} steps: - name: Checkout target repository @@ -61,16 +65,16 @@ jobs: - name: Mint repository-variable GitHub App token id: app-token - if: ${{ secrets.KB_VARIABLES_APP_ID != '' && secrets.KB_VARIABLES_APP_PRIVATE_KEY != '' }} + if: ${{ env.KB_VARIABLES_APP_ID != '' && env.KB_VARIABLES_APP_PRIVATE_KEY != '' }} uses: actions/create-github-app-token@67018539274d69449ef7c02e8e71183d1719ab42 # v2.1.4 with: - app-id: ${{ secrets.KB_VARIABLES_APP_ID }} - private-key: ${{ secrets.KB_VARIABLES_APP_PRIVATE_KEY }} + app-id: ${{ env.KB_VARIABLES_APP_ID }} + private-key: ${{ env.KB_VARIABLES_APP_PRIVATE_KEY }} owner: ${{ github.repository_owner }} repositories: ${{ github.event.repository.name }} - name: Validate repository-variable auth configuration - if: ${{ steps.app-token.outputs.token == '' && secrets.KB_VARIABLES_TOKEN == '' }} + if: ${{ steps.app-token.outputs.token == '' && env.KB_VARIABLES_TOKEN == '' }} run: | echo "Set KB_VARIABLES_TOKEN or KB_VARIABLES_APP_ID + KB_VARIABLES_APP_PRIVATE_KEY repository secrets." >&2 exit 1 @@ -98,7 +102,7 @@ jobs: id: stored-cursor shell: bash env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token || env.KB_VARIABLES_TOKEN }} run: | set -euo pipefail endpoint="/repos/${GITHUB_REPOSITORY}/actions/variables/${KB_VARIABLE_NAME}" @@ -123,7 +127,7 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} shell: bash env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token || env.KB_VARIABLES_TOKEN }} run: | set -euo pipefail latest_merged_at="$( @@ -247,7 +251,7 @@ jobs: if: ${{ steps.decision.outputs.should_run == 'true' && steps.stage-kb.outputs.changed == 'true' }} shell: bash env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token || env.KB_VARIABLES_TOKEN }} run: | set -euo pipefail existing_pr="$( @@ -272,7 +276,7 @@ jobs: if: ${{ steps.decision.outputs.should_run == 'true' }} shell: bash env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.KB_VARIABLES_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token || env.KB_VARIABLES_TOKEN }} run: | set -euo pipefail candidate_cursor="${{ steps.decision.outputs.next_cursor }}" diff --git a/tests/test_action_workflow.py b/tests/test_action_workflow.py index f747acc..28b113b 100644 --- a/tests/test_action_workflow.py +++ b/tests/test_action_workflow.py @@ -58,6 +58,9 @@ def test_workflow_bootstraps_tool_checkout_for_copyable_repos() -> None: assert "KB_VARIABLES_TOKEN" in text assert "KB_VARIABLES_APP_ID" in text assert "KB_VARIABLES_APP_PRIVATE_KEY" in text + assert "if: ${{ secrets." not in text + assert "env.KB_VARIABLES_TOKEN" in text + assert "env.KB_VARIABLES_APP_ID" in text assert "actions/create-github-app-token" in text for action in ( From 4778121deb296e7e565af677575905c46cd7bbf7 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 15:18:46 +0300 Subject: [PATCH 12/28] fix: point workflow bootstrap to live repo Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/github-pr-kb.yml | 2 +- README.md | 2 +- tests/test_action_workflow.py | 2 +- tests/test_readme.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml index a75e551..123a696 100644 --- a/.github/workflows/github-pr-kb.yml +++ b/.github/workflows/github-pr-kb.yml @@ -25,7 +25,7 @@ concurrency: cancel-in-progress: false env: - KB_TOOL_REPOSITORY: galzi/github-pr-kb + KB_TOOL_REPOSITORY: Galzi1/github-pr-kb KB_TOOL_REF: 96706685d7f90c3451f8631c7c907d200a08dd99 KB_TOOL_PATH: .github-pr-kb-tool KB_VARIABLE_NAME: KB_LAST_SUCCESSFUL_CURSOR diff --git a/README.md b/README.md index cd4357b..345d363 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ The workflow is intentionally copyable across repositories and uses two explicit | Setting | Default | Purpose | | --- | --- | --- | -| `KB_TOOL_REPOSITORY` | `galzi/github-pr-kb` | Which repository to checkout for the CLI and helper code | +| `KB_TOOL_REPOSITORY` | `Galzi1/github-pr-kb` | Which repository to checkout for the CLI and helper code | | `KB_TOOL_REF` | immutable full commit SHA | Which exact tool version to run | `KB_TOOL_REF` should stay pinned to an immutable release tag or full commit SHA. Treat it like any other supply-chain pin: update it intentionally, review the diff, and commit the workflow change when you want to upgrade. diff --git a/tests/test_action_workflow.py b/tests/test_action_workflow.py index 28b113b..864bf39 100644 --- a/tests/test_action_workflow.py +++ b/tests/test_action_workflow.py @@ -36,7 +36,7 @@ def test_workflow_bootstraps_tool_checkout_for_copyable_repos() -> None: assert "KB_TOOL_REPOSITORY" in text assert "KB_TOOL_REF" in text - assert "galzi/github-pr-kb" in text + assert "Galzi1/github-pr-kb" in text assert "refs/heads/main" not in text assert "KB_TOOL_REF: main" not in text assert "KB_TOOL_REF: master" not in text diff --git a/tests/test_readme.py b/tests/test_readme.py index 730b71f..392f6fa 100644 --- a/tests/test_readme.py +++ b/tests/test_readme.py @@ -23,6 +23,7 @@ def test_readme_is_automation_first() -> None: assert "workflow_dispatch" in text assert "KB_TOOL_REPOSITORY" in text assert "KB_TOOL_REF" in text + assert "Galzi1/github-pr-kb" in text assert "copy only the workflow file" in text assert "do not need this tool's source tree checked into your repository" in text assert "### PAT quickstart" in text From b5de7541d453f57bcdbb1ea5e0fb69b8bc28cd70 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 16:33:09 +0300 Subject: [PATCH 13/28] fix: avoid heredoc break in cursor persistence Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/github-pr-kb.yml | 16 +--------------- tests/test_action_workflow.py | 4 +++- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml index 123a696..2c1424c 100644 --- a/.github/workflows/github-pr-kb.yml +++ b/.github/workflows/github-pr-kb.yml @@ -300,21 +300,7 @@ jobs: exit "${status}" fi rm -f "${stderr_file}" - persisted_cursor="$( - python - "${fresh_cursor}" "${candidate_cursor}" <<'PY' - from datetime import datetime, timezone - import sys - - values = [] - for raw in sys.argv[1:]: - if not raw: - continue - dt = datetime.fromisoformat(raw.replace("Z", "+00:00")).astimezone(timezone.utc) - values.append(dt) - if values: - print(max(values).isoformat().replace("+00:00", "Z")) - PY - )" + persisted_cursor="$(python -c "from datetime import datetime, timezone; import sys; values = [datetime.fromisoformat(raw.replace('Z', '+00:00')).astimezone(timezone.utc) for raw in sys.argv[1:] if raw]; print(max(values).isoformat().replace('+00:00', 'Z') if values else '')" "${fresh_cursor}" "${candidate_cursor}")" if [ "${variable_exists}" = "true" ]; then gh api --method PATCH "${endpoint}" -f name="${KB_VARIABLE_NAME}" -f value="${persisted_cursor}" >/dev/null else diff --git a/tests/test_action_workflow.py b/tests/test_action_workflow.py index 864bf39..c531bde 100644 --- a/tests/test_action_workflow.py +++ b/tests/test_action_workflow.py @@ -87,7 +87,9 @@ def test_workflow_publishes_rolling_pr_and_persists_cursor_monotonically() -> No assert ".github-pr-kb/cache" not in git_add_line assert "actions/variables/${KB_VARIABLE_NAME}" in text assert "KB_LAST_SUCCESSFUL_CURSOR" in text - assert "max(" in text or "python - <<'PY'" in text + assert "max(" in text + assert 'persisted_cursor="$(python -c "' in text + assert 'persisted_cursor="$(\n' not in text def test_action_state_helper_command_runs_from_project_checkout() -> None: From 9b23d40a29673f3e34d720e44bc76f5c7d054e82 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Fri, 10 Apr 2026 16:41:48 +0300 Subject: [PATCH 14/28] docs: record phase 8 validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../phases/08-github-action-readme/08-UAT.md | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/.planning/phases/08-github-action-readme/08-UAT.md b/.planning/phases/08-github-action-readme/08-UAT.md index 1dafb2b..d9899de 100644 --- a/.planning/phases/08-github-action-readme/08-UAT.md +++ b/.planning/phases/08-github-action-readme/08-UAT.md @@ -1,48 +1,45 @@ --- -status: testing +status: complete phase: 08-github-action-readme source: - 08-01-SUMMARY.md - 08-02-SUMMARY.md - 08-03-SUMMARY.md started: 2026-04-10T09:49:16Z -updated: 2026-04-10T09:49:16Z +updated: 2026-04-10T13:39:36Z --- ## Current Test -number: 1 -name: Copyable workflow bootstrap -expected: | - The shipped `.github/workflows/github-pr-kb.yml` can be copied into another repository, triggers on merged PRs plus `workflow_dispatch`, and bootstraps this tool from `.github-pr-kb-tool` using `KB_TOOL_REPOSITORY` and `KB_TOOL_REF` instead of assuming the consumer repo vendors this package. -awaiting: user response +[testing complete] ## Tests ### 1. Copyable workflow bootstrap expected: The shipped workflow is copy-ready for consumer repos and bootstraps the tool from `.github-pr-kb-tool` via `KB_TOOL_REPOSITORY` and `KB_TOOL_REF`. -result: pending +result: pass ### 2. No-new-PR guard and cursor contract expected: Repeated merged events and manual no-new-PR runs skip before extract/classify/generate, and successful runs use `KB_LAST_SUCCESSFUL_CURSOR` as a monotonic state boundary. -result: pending +result: pass ### 3. Workflow auth and publication contract expected: The workflow clearly separates CLI/runtime auth from repository-variable auth, stages only `kb/` output plus `kb/.manifest.json`, and uses one rolling PR on `automation/github-pr-kb`. -result: pending +result: pass ### 4. README local onboarding expected: README leads with automation, then gives local uv install, env vars, CLI commands, KB output example, and committed-vs-not-committed guidance that matches the shipped workflow. -result: pending +result: pass ## Summary total: 4 -passed: 0 +passed: 4 issues: 0 -pending: 4 +pending: 0 skipped: 0 +blocked: 0 ## Gaps -None yet. +[none yet] From b1a2ab6211a3ecb1ad68f3a8937e410224ad8f9c Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 09:16:51 +0300 Subject: [PATCH 15/28] fix: recover wrapped classifier JSON Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../08-DEMO-COMPLETE-RISK-REVIEW.md | 139 ++++++++ .../08-DEMO-COMPLETE.md | 303 ++++++++++++++++++ src/github_pr_kb/classifier.py | 43 ++- tests/test_classifier.py | 45 +++ 4 files changed, 527 insertions(+), 3 deletions(-) create mode 100644 .planning/phases/08-github-action-readme/08-DEMO-COMPLETE-RISK-REVIEW.md create mode 100644 .planning/phases/08-github-action-readme/08-DEMO-COMPLETE.md diff --git a/.planning/phases/08-github-action-readme/08-DEMO-COMPLETE-RISK-REVIEW.md b/.planning/phases/08-github-action-readme/08-DEMO-COMPLETE-RISK-REVIEW.md new file mode 100644 index 0000000..2d25c28 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-DEMO-COMPLETE-RISK-REVIEW.md @@ -0,0 +1,139 @@ +# Risk Review: `08-DEMO-COMPLETE.md` + +## 1. Plan Summary + +**Purpose.** Prove a complete, user-visible Phase 8 demo in `Galzi1/github-pr-kb-demo`, ending with a merged feature PR, a generated KB PR containing at least one real article, a stable post-merge workflow run, and a clean final repo state. + +**Key components touched.** + +| Component | Role in the plan | +| --- | --- | +| Demo repository `Galzi1/github-pr-kb-demo` | Consumer repo where the Phase 8 workflow is exercised end to end | +| GitHub Actions workflow | Triggers on merge events, runs extract/classify/generate, opens KB PRs, and persists cursor state | +| Extraction / classification / generation pipeline | Converts merged PR comments into KB artifacts | +| Anthropic-backed classifier | Converts raw comment text into one of the supported categories or review-needed output | +| KB output (`kb//`, `kb/INDEX.md`, `kb/.manifest.json`) | Observable proof that knowledge was generated and published | +| Human review / merge steps | Provide the source review comments, validate article fidelity, and close the loop by merging the KB PR | + +**Plan-stated assumptions.** + +1. Previous workflow hardening means the plumbing path is already sound enough that the remaining gap is content generation. +2. Existing logs and artifacts from PR `#1` are sufficient to diagnose why all 10 comments failed classification. +3. A new, deliberately small feature PR with high-signal review comments is likely to produce at least one publishable article. +4. One successful curated run is enough to justify calling the demo "100% confidence" complete. + +**Theory of success.** The plan succeeds if the current classification failure mode is first understood, then a fresh PR with clearly knowledge-bearing comments is merged, the workflow produces a real KB article, humans confirm the article is faithful, and the system reaches steady state without opening another automation PR. + +## 2. Assumptions & Evidence + +This plan is strongest where it is explicit about the current gap and weakest where it assumes a single successful rerun will convert uncertainty into confidence. + +| ID | Assumption | Explicit / Implicit | Class | Justification status | Blast radius if wrong | How to test before committing fully | +| --- | --- | --- | --- | --- | --- | --- | +| A1 | The PR `#1` failures are diagnosable from existing logs/artifacts without reproducing the run in a different way. | Explicit | Foundational | **Partial.** The plan names this as the first todo, but does not confirm artifact depth or whether failed model outputs were retained. | If false, the team may start a new demo without understanding whether the problem is content quality, classifier robustness, caching, or API behavior. | Pull the exact workflow logs/artifacts for the failed run and verify they reveal concrete failure reasons rather than only aggregate counts. | +| A2 | Workflow mechanics are already sufficiently proven, so the remaining uncertainty is mainly in the content/classifier path. | Implicit | Structural | **Mostly justified.** Multiple runs reportedly validated branch creation, persistence, and steady state. Still, content-path failures can expose workflow-path weaknesses hidden by zero-article runs. | If false, a second demo may fail for both workflow and content reasons, making diagnosis slower and conclusions weaker. | Run a preflight checklist on permissions, refs, secrets, and branch protections immediately before the next live demo. | +| A3 | Better-crafted review comments will classify cleanly. | Explicit | Foundational | **Weak.** The previous outcome was "10 failed" rather than merely "low confidence" or "other," which suggests a possibly systemic issue, not just low-signal input. | If false, the next demo repeats the same failure and consumes manual effort without increasing confidence. | Dry-run representative review comments through the classifier outside the live demo path or inspect the exact prior failure mode first. | +| A4 | Supported categories and prompt behavior are predictable enough that humans can intentionally author comments that land in `gotcha`, `code_pattern`, or `domain_knowledge`. | Explicit | Structural | **Partial.** This is plausible, but only if classifier output is stable and parsing is robust. The current evidence does not show that. | If false, the demo becomes dependent on prompt luck rather than reproducible behavior. | Create 2-3 example comments and validate local classification results before the merge-triggered workflow run. | +| A5 | A single curated successful run is enough to claim "100% confidence in the demo." | Explicit | Structural | **Unjustified.** One scripted success proves the happy path, not robustness across comment phrasing, timing, API variability, or cache state. | If false, the plan may overclaim readiness and hide brittleness that appears immediately after the demo. | Reframe the claim as "happy-path demo complete" unless at least one additional variant or replay also succeeds. | +| A6 | Manual reviewers will provide comments that are both product-realistic and classifier-friendly, then judge article fidelity consistently. | Implicit | Structural | **Weak.** The plan requires manual precision but does not define an authoring rubric or acceptance rubric. | If false, the demo outcome may hinge on ad hoc human choices rather than product behavior. | Provide explicit comment-writing examples and a short article-acceptance checklist before starting the live run. | +| A7 | External dependencies will behave consistently during the next run: GitHub permissions, workflow refs, model availability, API quotas, and repository state. | Implicit | Structural | **Partial.** Prior runs worked, but these are live dependencies outside the plan's control. | If false, a demo failure may be misread as a product failure when it is an environment failure. | Do a same-day preflight against secrets, permissions, tool ref reachability, and API readiness. | +| A8 | "No open PRs" is a valid final exit criterion for the demo repo. | Explicit | Peripheral | **Weak.** It is precise for automation churn, but too broad if unrelated human PRs can exist. | If false, the plan can fail its own exit criteria even when the automation loop worked correctly. | Narrow the criterion to "no open automation PRs created by github-pr-kb." | + +**Secrets vs. mysteries.** + +- **Secrets:** the exact cause of the 10 failed classifications, whether cache state affects replays, whether example comments classify locally, whether the workflow still has correct permissions and refs. +- **Mysteries:** whether a human reviewer will judge the generated article "useful" enough, whether future non-curated PR comments will produce similarly good output. + +The current plan is strongest when it treats unresolved classifier behavior as a **secret** to investigate. It becomes weaker when it treats long-term confidence as if it can be resolved by one curated run, which is closer to a **mystery** that must be managed with bounded claims. + +## 3. Ipcha Mistabra - Devil's Advocacy + +### 3a. The Inversion Test + +**Claim:** A fresh PR with better comments will likely complete the content demo. + +**Inversion:** Better comments may not matter at all. "10 failed" classifications points to a parser, prompt, API, schema, or cache defect more than a simple comment-quality issue. In that world, the next PR adds more manual work while reproducing the same opaque failure. + +**Why this inversion is compelling:** The prior evidence is not "0 useful comments" but "0 classified, 10 failed." That smells like system behavior, not merely weak content. + +--- + +**Claim:** Once one curated run succeeds, the demo can be called complete with 100% confidence. + +**Inversion:** A single curated success could actually reduce epistemic rigor by creating false confidence. It would prove that one carefully staged path worked once under current conditions, not that the system is stable, repeatable, or resilient to minor variation. + +**Why this inversion is compelling:** Demo success and product confidence are not the same thing. A curated demo is a marketing-quality proof, not an operational-quality proof. + +--- + +**Claim:** The workflow path is already validated, so the remaining work is mostly content-path verification. + +**Inversion:** Content-path verification may be exactly where workflow-path weaknesses reappear. A run that generates real articles can trigger branch, manifest, dedup, PR diff, and post-merge behavior that zero-article runs never exercised in the same way. + +**Why this inversion is compelling:** "Real content exists" is not just more of the same path; it can activate new edge cases. + +### 3b. The Little Boy from Copenhagen + +**A new engineer joining next month** would likely ask: "What exact failure signature distinguishes bad input from broken classification?" The current plan does not define that boundary. It assumes the first diagnosis task will discover it, but the execution plan after that still relies heavily on intuition. + +**An SRE or on-call maintainer at 3 AM** would likely ask: "If the next run produces no article, how do I tell in five minutes whether extraction, classification, generation, dedup, or permissions caused it?" The plan has outcome checks, but not a stage-specific observability checklist. + +**A repo maintainer** would likely ask: "Why is 'no open PRs' part of success?" If another unrelated PR exists, the plan fails its own exit criteria for a reason that has nothing to do with Phase 8. + +**A skeptical user of the product** would likely ask: "How do I know the generated article is actually faithful rather than superficially plausible?" The plan includes human review, which is good, but it does not define what fidelity means. + +### 3c. Failure of Imagination Check + +1. **Cache poisoning / stale-state scenario.** The next run may reuse cached classification or manifest state in a way that masks improvement or suppresses article generation, making a good comment set look like a product failure. +2. **Parseable-but-wrong scenario.** The classifier may stop "failing" but still return low-confidence or `other` results that technically pass through the pipeline while still not producing the proof the demo needs. +3. **Article-generated-but-filtered scenario.** Classification may succeed, but generation may deduplicate or skip the article because of manifest state, leading to another KB PR that looks empty in the place that matters. +4. **Environment-drift scenario.** The plan assumes the same repo settings, token scopes, action refs, and external model behavior remain stable between prior validation and the next demo. Live systems drift. +5. **Success-but-not-credible scenario.** The demo may succeed only because comments were written in a classifier-optimized way that real reviewers would never write, making the demo impressive but strategically misleading. + +## 4. Risk Register + +| Risk ID | Category | Description | Trigger | Probability | Severity | Priority | Detection | Mitigation | Contingency | Assumption link | Knowledge class | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| R1 | Technical | The real root cause of the PR `#1` "10 failed" outcome is not isolated before running the next demo. | A new feature PR is merged before the failure signature is understood. | High - the plan still lacks the actual cause. | High - the next demo can repeat the same failure and waste the main proving attempt. | High | Another run shows `0 classified` and failed counts without actionable diagnosis. | Make Todo #1 a hard gate: inspect logs/artifacts and, if needed, reproduce locally until the failure mode is concrete. | Pause the demo-completion claim and switch scope to classifier-path repair. | A1, A3 | Unknown known | +| R2 | Technical | The next review comments are human-meaningful but still do not map cleanly to the classifier's expected prompt/schema/category behavior. | Comments are merged without first validating example phrasing. | Medium | High - no article means the demo is still incomplete. | High | Low-confidence, `other`, empty, or failed classifications on otherwise good comments. | Create a short comment-authoring rubric and validate 2-3 samples before the live PR merge. | Open another small PR or adjust classifier robustness before retrying. | A3, A4, A6 | Known unknown | +| R3 | Operational | Observability is too weak to separate extraction, classification, generation, dedup, and permission failures quickly during the live demo. | The next run again produces no article or partial output. | Medium | High - diagnosis becomes slow and confidence claims remain fuzzy. | High | Logs show counts but not decisive per-stage reasons or retained failing payloads. | Add a pre-demo diagnostic checklist and preserve the exact artifacts/log lines needed for stage-level attribution. | Download artifacts immediately and perform a post-mortem before any retry. | A1, A2 | Known unknown | +| R4 | Organizational | A single curated successful run is over-interpreted as "100% confidence," masking fragility and shrinking future learning. | The first successful article-producing run is treated as full proof. | High | Medium - the product story becomes stronger than the evidence. | High | The final write-up contains categorical confidence claims unsupported by repeatability evidence. | Reframe the target as "complete happy-path demo" unless at least one additional variant also works. | Document confidence boundaries explicitly in the completion note. | A5 | Known known | +| R5 | Operational | External dependencies drift between the prior validated runs and the next live demo. | Token scopes, action permissions, refs, branch protections, or model/API behavior change. | Medium | High - a live demo can fail for reasons unrelated to the product logic. | High | Auth errors, checkout/ref errors, API failures, or PR creation failures appear in the workflow. | Perform a same-day environment preflight before merging the new feature PR. | Repair environment issues and rerun without counting the failed attempt as product evidence. | A2, A7 | Known unknown | +| R6 | Organizational | Manual steps are underspecified, so comment authoring and article-fidelity review vary too much between runs or reviewers. | Humans write comments or assess generated content without a shared rubric. | Medium | Medium | Medium | Comments are inconsistent in style/specificity, or reviewers disagree on whether the article is faithful. | Provide exact examples of acceptable review comments and a short fidelity checklist. | Re-review the KB PR or create a fresh demo PR with better source comments. | A6 | Mystery | +| R7 | Technical | Cache or manifest state suppresses article generation even when classification improves. | Prior cache entries or manifest mappings are unexpectedly reused during the next run. | Medium | High - the team may misdiagnose success as failure or failure as success. | High | Nonzero extraction/classification activity but `0 new` generation or unexplained skips. | Inspect cache and manifest behavior explicitly as part of pre-demo readiness. | Reset or isolate demo state, or rerun in a fresh environment with documented cache behavior. | A2, A3 | Unknown unknown | +| R8 | Operational | The exit criterion "no open PRs" blocks success for irrelevant reasons. | Any unrelated PR exists during or after the demo. | Medium | Low | Medium | Core demo behavior succeeds, but the repo still has an open non-automation PR. | Narrow the criterion to automation PRs created by the workflow. | Snapshot repo state and declare success relative to automation artifacts only. | A8 | Known known | + +## 5. Verdict & Recommendations + +**Overall risk level: High.** The plan is honest, structured, and close to a viable execution checklist, but it rests on one load-bearing uncertainty: whether the prior all-failed classification result was caused by input quality or by a systemic defect. Until that is resolved, the proposed next demo attempt is at meaningful risk of being an expensive repetition rather than a completion. + +**Top 3 risks.** + +1. **R1 - unresolved classifier failure mode** +2. **R3 - insufficient stage-level observability during the live run** +3. **R4 - overclaiming confidence from one curated success** + +**Recommended actions before the next live demo attempt.** + +1. **Promote diagnosis to a hard gate.** Do not merge a new feature PR for the "complete demo" path until the PR `#1` failure mode is written down concretely: malformed model output, parse mismatch, API failure, cache interaction, prompt mismatch, or something else. +2. **Add a comment-input contract.** Prepare 2-3 example review comments that are both realistic and intentionally aligned to the supported categories, then validate them before using them in the live demo. +3. **Add a five-minute triage checklist.** Define what evidence distinguishes extraction failure, classification failure, generation failure, dedup/manifest suppression, and permissions/environment failure. +4. **Reduce claim scope.** Replace "100% confidence" with a bounded statement unless you also demonstrate repeatability or at least one meaningful variant. +5. **Tighten exit criteria.** Change "no open PRs" to "no open automation PRs created by the workflow," and define a brief article-fidelity checklist so the manual review step is less subjective. + +**Open questions.** + +1. What exact error produced the "10 failed" classification result on the prior run? +2. Are the failing classifier outputs retained anywhere inspectable, or do logs only expose aggregate counts? +3. Could cache or manifest state suppress article creation even when classification succeeds? +4. What specific rubric will be used to decide that a generated article is faithful enough to merge? +5. Is the next demo intended to prove only the happy path, or also repeatability? + +**What the plan does well.** + +- It is unusually honest about the current gap: the missing proof is not workflow plumbing, but real article generation. +- It includes a non-negotiable gate that rejects false completion if the next run still produces no articles. +- It separates automatic and manual responsibilities clearly enough to execute. +- It defines a concrete desired end state in the repo rather than relying on vague notions of "done." + +**Bottom line.** This is a credible near-final demo checklist, but not yet a low-risk completion plan. Treat the prior classifier failure as the critical unknown, narrow the confidence language, and add sharper observability and manual rubrics before running the next "complete demo" attempt. diff --git a/.planning/phases/08-github-action-readme/08-DEMO-COMPLETE.md b/.planning/phases/08-github-action-readme/08-DEMO-COMPLETE.md new file mode 100644 index 0000000..77f12f2 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-DEMO-COMPLETE.md @@ -0,0 +1,303 @@ +# Phase 8 demo completion status for `github-pr-kb-demo` + +## Goal + +Reach a **complete end-to-end demo** of Phase 8 in `Galzi1/github-pr-kb-demo` and stop only when we have evidence for the full user-visible loop: + +1. a normal feature PR is opened in the demo repo, +2. review comments are added on that PR, +3. the PR is merged, +4. the Phase 8 workflow automatically extracts/classifies/generates KB output, +5. the workflow opens a KB PR that contains at least one real article under `kb//`, +6. that KB PR is reviewed and merged, +7. the follow-up workflow run succeeds and does **not** create another KB PR, +8. `main` ends with the generated article, updated `kb/INDEX.md`, updated `kb/.manifest.json`, no open automation PRs, and a successful latest workflow run. + +## What is already done + +| Status | Item | Owner | Evidence | +| --- | --- | --- | --- | +| Done | Public demo repo `Galzi1/github-pr-kb-demo` was created and scaffolded with the copied workflow. | Copilot | Repo exists and workflow is installed on `main`. | +| Done | Source workflow was hardened for real-world demo use. | Copilot | Fixed parse-safe secret checks, corrected `KB_TOOL_REPOSITORY` to `Galzi1/github-pr-kb`, and fixed cursor persistence. | +| Done | Demo PR `#1` was opened, reviewed, updated, and merged. | Copilot + user flow | PR `#1` merged successfully. | +| Done | Manual no-op path was validated. | Automatic GitHub Actions flow | Run `24242554122` succeeded. | +| Done | Real merged-PR publish path was validated. | Automatic GitHub Actions flow | Run `24245335864` reached KB publication and opened PR `#2`; later fixes removed its final cursor failure. | +| Done | Full publish-and-persist path was validated after fixes. | Automatic GitHub Actions flow | Run `24245538002` succeeded. | +| Done | Steady-state loop after merging the generated KB PR was validated. | Automatic GitHub Actions flow | Run `24245767479` succeeded and did not open another KB PR. | +| Done | New demo PR `#3` was opened, reviewed with explicit plain-text comments, and merged. | Copilot | PR `#3` merged at `2026-04-10T21:58:37Z`. | +| Blocked | Fresh content-demo attempt after PR `#3` did **not** produce a KB PR or article. | Automatic GitHub Actions flow + Copilot diagnosis | Run `24266078977` succeeded mechanically, but classification logged `0 new / 18 failed`, generation logged `0 new`, and no automation PR was opened. | + +## Important current limitation + +The demo is **not yet a complete content demo**. + +What we proved so far is: + +- the workflow boots correctly in a consumer repo, +- merged PR events trigger the pipeline, +- the automation branch / rolling KB PR flow works, +- cursor persistence works, +- the workflow settles cleanly after a generated KB PR is merged. + +What we **did not** prove yet is: + +- a merged demo PR with review comments results in **at least one generated KB article** under `kb//`. + +The latest attempt with PR `#3` did **not** close that gap. It reproduced the same high-level failure mode in a cleaner demo setup: + +- PR `#3` was merged successfully, +- the merged-PR workflow run `24266078977` started automatically and finished `success`, +- no KB PR was opened, +- `main` still contains only `kb/.manifest.json` and `kb/INDEX.md`, +- the run logs showed `Extracted 2 PRs, 8 comments cached.`, `Classified 0 new, 0 cached, 0 need review, 18 failed.`, and `Generated 0 new, 0 skipped, 0 filtered, 0 failed.`. + +So the demo is now blocked not on setup or review-comment quality alone, but on the classifier/content path itself. + +For the PR `#1` demo data in workflow run `24245335864`, the preserved logs and debug artifact showed: + +- `Extracted 1 PRs, 10 comments cached.` +- nine `Could not parse classification JSON for comment ...` warnings during classification, +- `Classified 1 new, 0 cached, 0 need review, 9 failed.` +- `Generated 1 new, 0 skipped, 0 filtered, 0 failed.` + +However, the rolling KB PR `#2` that was ultimately merged still only contained: + +- `kb/.manifest.json` +- `kb/INDEX.md` + +So the workflow mechanics were validated, but the repo still did **not** finish with a published article on `main`, which is why the content demo remains incomplete. + +## Remaining work required for a COMPLETE demo + +### Mandatory gates before the next live demo attempt + +The next feature-PR demo attempt must **not** start until all of the following are true: + +1. **Done:** we have a short written diagnosis of the PR `#1` classification failure that names the concrete failure mode from logs/artifacts. +2. **Done:** we know which prior evidence is still inspectable and what fallback to use if deeper payload-level evidence is needed. +3. **Done:** we have a **comment-input contract** with realistic example review comments intentionally aligned to `gotcha`, `code_pattern`, and `domain_knowledge`. +4. **Done:** we have a **five-minute triage checklist** that distinguishes: + - extraction failure, + - classification failure, + - generation failure, + - dedup / manifest suppression, + - permissions / environment failure. +5. **Done:** we have a brief **article-fidelity checklist** for the human review step so "looks good" is not purely ad hoc. + +### PR `#1` classification diagnosis + +The concrete failure mode for the PR `#1` demo data was **malformed model output / parse mismatch at the classifier boundary**, not extraction failure and not an Anthropic API outage. + +Evidence from run `24245335864` and its uploaded `github-pr-kb-debug` artifact: + +- extraction succeeded and cached all 10 comments for `pr-1.json`, +- the classifier emitted nine log lines of the form `Could not parse classification JSON for comment `, +- the current classifier only increments `failed` for two paths: `anthropic.APIError` or `json.loads(...)` failure, and the logs showed the JSON-parse path rather than the API-error path, +- one plain-text issue comment (`4223530384`) classified successfully as `domain_knowledge` and produced a generated article in the failure artifact, +- the nine failed comments therefore represent responses that came back from the model but were not valid bare JSON for the current parser contract. + +Short diagnosis: **the dominant PR `#1` classification failure was non-parseable classifier output, especially across the longer/more formatted comment set, so the next demo should not assume better review comments alone will fix the problem.** + +What is still unknown from existing evidence is the exact raw malformed payload for each failed comment, because the workflow logs did not preserve the model response text at info level. If we need payload-level examples, the fastest fallback is a focused reproduction against the saved `pr-1.json` comment bodies with response capture enabled. + +### Artifact and fallback status + +The prior failure evidence is still inspectable enough to support the next step. + +What is still available right now: + +- the workflow job logs for run `24245335864` are still downloadable and still show the stage-level evidence, including the nine `Could not parse classification JSON` warnings, +- the uploaded debug artifact `github-pr-kb-debug` (artifact `6371844189`, not expired as of this update) is still downloadable, +- that artifact still contains the key preserved files we need for replay and inspection: `.github-pr-kb/cache/pr-1.json`, `.github-pr-kb/cache/classified-pr-1.json`, `.github-pr-kb/cache/classification-index.json`, and the generated `kb/` output snapshot. + +What is **not** preserved in the current artifact set: + +- the raw malformed model responses for the nine failed comments, +- enough logging detail to tell whether the malformed outputs were wrapped in prose, fenced code blocks, partial JSON, or some other near-miss shape. + +Fastest reliable fallback if we need deeper evidence: + +- replay classification locally or in a one-off debug run against the saved `pr-1.json` comments, +- capture the raw Claude response text before `json.loads(...)`, +- compare the successful plain-text comment path with one or two failed heavily formatted comments to isolate the output-shape mismatch quickly. + +### Comment-input contract for the next demo PR + +Use **plain human review comments** for the live demo. Each comment should be: + +- plain text only, with no HTML, screenshots, copied bot templates, or fenced code blocks, +- 1-3 sentences focused on one idea, +- tied to a concrete code behavior or business rule, +- explicit about **why** the point matters, not just what changed, +- written before merge as normal review feedback, not as a post-fix status update like "fixed in commit ...". + +Recommended example comments for the next demo run: + +1. **Gotcha** + `We should reject negative discount_amount values explicitly. Otherwise this helper turns a "discount" path into a surcharge path, which is easy to miss in tests because the math still looks valid at a glance.` + +2. **Code pattern** + `I would keep a small pattern here: validate discount inputs first, normalize the discounted subtotal once, then delegate to calculate_total for tax and rounding. That keeps every pricing path using the same calculation flow instead of duplicating rounding rules in multiple helpers.` + +3. **Domain knowledge** + `Please document that this helper applies an absolute discount before tax, not after tax and not as a percentage. That ordering is a pricing rule the business will care about, so it should be explicit in the code or docstring rather than implied by the implementation.` + +These examples are intentionally shaped to be: + +- realistic code-review comments a human could naturally leave on a small pricing helper PR, +- short enough to stay readable in GitHub, +- knowledge-bearing enough to publish if the classifier and generator path works correctly. + +### Five-minute triage checklist for the next run + +When the next merged-PR workflow runs, inspect the latest `update-kb` job in this order and stop as soon as one row matches the evidence: + +| Area | Check in the run | Positive signal | Failure signal | Next action | +| --- | --- | --- | --- | --- | +| 1. Permissions / environment | Look first at `Validate repository-variable auth configuration`, `Checkout github-pr-kb tool`, `Install tool dependencies`, `Create or reuse rolling KB pull request`, and `Persist KB_LAST_SUCCESSFUL_CURSOR`. | These steps are `success` or intentionally `skipped`. | Missing secrets, checkout failure, dependency install failure, `gh` auth failure, push/PR creation failure, or cursor-persist failure. | Treat as workflow/environment breakage before reasoning about content quality. | +| 2. Extraction | Read the `Extract PR comments` step summary and, if needed, the uploaded debug artifact's `.github-pr-kb/cache/pr-.json`. | Expected merged PR is present and the log shows nonzero comments cached. | Step fails, target PR is absent from cache, or comment count is unexpectedly zero. | Diagnose extractor input scope, repo/token access, or comment filtering before looking at classify/generate. | +| 3. Classification | Read the `Classify extracted comments` step summary and warnings. | Nonzero `new` or `cached`, with failures at zero or low enough to still leave publishable output. | `failed > 0`, repeated `Could not parse classification JSON`, Anthropic/API errors, or `0 classified` when extracted comments were clearly substantive. | Treat as classifier/content-path failure; inspect `classified-pr-.json`, `classification-index.json`, and raw-response fallback if needed. | +| 4. Generation | Read the `Generate knowledge base` step summary and inspect `kb/` output or the KB PR diff. | At least one article file appears under `kb//`, plus matching `kb/INDEX.md` and `kb/.manifest.json` updates. | Generate step fails, or classify succeeded but no article file is produced. | Treat as generation failure unless the next row proves it was intentional dedup suppression. | +| 5. Dedup / manifest suppression | Compare generate output with `.manifest.json`, staged diff, and whether `Stage generated KB output` reports changes. | Existing manifest mapping explains why a repeated comment produced no new article. | `Generated 0 new` / no staged changes even though the PR introduced truly new review comments and no manifest entry already maps them. | If the manifest does not already explain the skip, escalate back to generation or classification diagnosis. | +| 6. Publication outcome | Check whether `Stage generated KB output`, `Commit and push KB update branch`, and `Create or reuse rolling KB pull request` actually produce a KB PR with article files. | Open or updated KB PR contains one or more `kb//*.md` files. | Only `kb/.manifest.json` and `kb/INDEX.md`, or no KB PR despite expected content. | Do **not** claim demo success; capture the artifact and continue diagnosis from the earliest failed stage above. | + +Fast rule of thumb for this demo: + +- **Extraction failure:** wrong or missing PR/comment data. +- **Classification failure:** comments were extracted, but classifier warnings/errors prevent usable classifications. +- **Generation failure:** classifications exist, but article files are not created from them. +- **Dedup / manifest suppression:** the system is behaving as designed because the comment was already published. +- **Permissions / environment failure:** the workflow cannot read, write, authenticate, or persist state even before content logic is trustworthy. + +### Article-fidelity checklist for human review of the KB PR + +When the generated KB PR is open, review each article against this checklist before merging: + +| Check | What to confirm | Merge only if... | +| --- | --- | --- | +| 1. Factual fidelity | The article stays grounded in the PR title and source review comment. | It does **not** invent causes, decisions, implications, or fixes that are missing from the source. | +| 2. Category fit | The article category matches the original comment's intent. | A warning/pitfall lands in `gotcha`, a reusable implementation approach lands in `code_pattern`, and a business/project rule lands in `domain_knowledge` (or another category only if clearly justified). | +| 3. Useful summary | The `#` heading and article framing capture the actual insight. | The title is specific and useful to a future reader, not vague, misleading, or broader than the source comment. | +| 4. Required structure | The article body uses the expected section headings for its category and fills unsupported sections honestly. | Missing evidence is rendered as `Not stated in the source comment.` rather than hallucinated prose. | +| 5. Source handling | The article paraphrases rather than echoing or bloating the original review text. | It preserves the meaning without copying large spans of the source comment verbatim. | +| 6. Metadata integrity | Frontmatter and cross-file references match the source. | `pr_url`, `comment_url`, `author`, `category`, `comment_id`, `needs_review`, `kb/INDEX.md`, and `kb/.manifest.json` all point to the same article and source comment. | +| 7. Review flag visibility | Low-confidence content stays visibly reviewable. | If `needs_review: true`, the article and `kb/INDEX.md` make that visible rather than hiding it. | +| 8. Diff hunk relevance | Review-comment articles include code context only when it helps. | Any included fenced diff block is the source diff hunk for that comment and is relevant to the article's point. | + +Practical merge rule for this demo: + +- merge if the article is faithful, correctly categorized, structurally complete, and useful to a teammate who did not read the original PR, +- do **not** merge if the article hallucinates, miscategorizes the insight, or turns a narrow review comment into a broader claim than the source supports. + +### Prepared next demo PR + +The next small feature PR is now ready for review in `github-pr-kb-demo`. + +- PR: `#3` — `feat: add percentage discount helper for demo review` +- URL: `https://github.com/Galzi1/github-pr-kb-demo/pull/3` +- Branch: `demo/percentage-discount-helper` +- Change scope: one small change to `demo_app.py` adding `calculate_total_with_percentage_discount(...)` +- Why this is a good demo candidate: it is small, easy to review on one screen, and naturally invites the exact kinds of comments we want for the next step — bounds/validation (`gotcha`), reuse of calculation flow (`code_pattern`), and business-rule clarity about tax/stacking semantics (`domain_knowledge`). + +### Review comments posted on the demo PR + +PR `#3` now contains the intended human-written review comments aligned to the comment-input contract: + +- `gotcha`: `https://github.com/Galzi1/github-pr-kb-demo/pull/3#discussion_r3066907088` +- `code_pattern`: `https://github.com/Galzi1/github-pr-kb-demo/pull/3#discussion_r3066907115` +- `domain_knowledge`: `https://github.com/Galzi1/github-pr-kb-demo/pull/3#discussion_r3066907152` + +These are plain-text review comments on the changed lines in `demo_app.py`, and they are ready for the merge step that will trigger the workflow. + +### Current live status after merging PR `#3` + +- PR `#3` merged successfully. +- Automatic workflow run `24266078977` started from that merge and completed `success`. +- `Commit and push KB update branch` and `Create or reuse rolling KB pull request` were both skipped because the workflow staged no KB changes. +- There is currently **no open automation PR** in `github-pr-kb-demo`. +- The latest run therefore failed the content-demo goal even though the workflow itself stayed green. + +### PR `#3` run diagnosis + +The new demo attempt confirms that the next task is **not** another review/merge exercise. It is a classifier/content-path fix. + +Evidence from run `24266078977`: + +- extraction worked (`Extracted 2 PRs, 8 comments cached.`), +- the classifier emitted repeated `Could not parse classification JSON for comment ...` warnings, +- classification ended at `Classified 0 new, 0 cached, 0 need review, 18 failed.`, +- generation then had nothing publishable to work with (`Generated 0 new, 0 skipped, 0 filtered, 0 failed.`), +- no KB files changed, so no KB PR was opened. + +Short conclusion: **the next demo should not continue until the classifier/content path is fixed or hardened enough to turn these comments into parseable classifications.** + +### Current remediation status in `github-pr-kb` + +The classifier/content-path fix is now in progress in the tool repository itself: + +- the local `github-pr-kb` classifier has been hardened to recover JSON objects from bare JSON, fenced ```json``` output, and prose-wrapped output instead of only accepting a raw top-level JSON string, +- regression tests were added for fenced JSON and prose-wrapped JSON classifier responses, +- local validation passed (`ruff check src tests` and the full pytest suite). + +Important scope note: run `24266078977` still reflects the **pre-fix** shipped tool behavior because the demo workflow is pinned to the older `KB_TOOL_REF`. The next live demo attempt should use the hardened classifier build, not the previous pinned commit. + +### Exit criteria + +We can claim the **happy-path Phase 8 demo is complete** only after all of the following are true at the same time: + +1. A new merged feature PR in `github-pr-kb-demo` has review comments that carry knowledge worth publishing. +2. The automatic merged-PR workflow succeeds. +3. The resulting KB PR contains: + - at least one article file under `kb//`, + - an updated `kb/INDEX.md` linking to that article, + - an updated `kb/.manifest.json` mapping the comment id to the article path. +4. The article content is reviewed and accepted as faithful to the source PR comment. +5. The KB PR is merged. +6. The automatic post-KB-merge workflow succeeds and does not open another KB PR. +7. The demo repo ends with no open **automation** PRs and the generated article committed on `main`. + +This is a **bounded claim**: it proves the intended end-to-end demo path on a real example. By itself it does **not** prove broad repeatability across varied comment styles, repo states, or external API conditions. + +### Todo checklist + +| # | Todo | Owner | Why this is required | Completion signal | +| --- | --- | --- | --- | --- | +| 1 | **Done:** inspect the current classification failures from the PR `#1` demo data and identify the exact failure mode (for example: malformed model output, empty output, API issue, parse mismatch, cache interaction, or prompt mismatch). Treat this as a **hard gate**, not a nice-to-have. | Copilot | We need to know whether the next demo should focus on better review-comment input, classifier robustness, cache handling, or both. | We have a short written diagnosis with the concrete failure reason from logs/artifacts, or a focused reproduction that isolates it. | +| 2 | **Done:** confirm whether the failing classifier outputs and related artifacts are still inspectable. If not, decide the fastest reliable fallback for reproducing the failure signal. | Copilot | Aggregate counts alone are not enough for a confidence-grade next step. | We know exactly what evidence we have and what fallback we will use if logs are insufficient. | +| 3 | **Done:** prepare a **comment-input contract**: 2-3 example review comments that are realistic, clearly knowledge-bearing, and intentionally aligned to supported categories such as `gotcha`, `code_pattern`, or `domain_knowledge`. | Copilot | The next live demo should not depend on vague human intuition about what "good comments" look like. | We have a short set of example comments ready to use as the review standard. | +| 4 | **Done:** define a **five-minute triage checklist** for the next run that distinguishes extraction failure, classification failure, generation failure, dedup / manifest suppression, and permissions / environment failure. | Copilot | If the next run still produces no article, we need rapid stage-level diagnosis instead of another ambiguous post-mortem. | We have a compact checklist or decision tree for reading the next run. | +| 5 | **Done:** define a brief **article-fidelity checklist** for human review of the generated KB PR. | Copilot + you | The final merge decision should use a consistent standard rather than a vague "looks good." | We have a small rubric covering factual fidelity, category fit, and usefulness. | +| 6 | **Done:** prepare a **new small feature PR** in `github-pr-kb-demo` that is easy to review and likely to produce one or more high-signal review comments. | Copilot | We need a fresh PR because PR `#1` has already been merged and its comments produced no article. | A new demo PR exists and is ready for review. | +| 7 | **Done:** add **clear, explicit review comments** on that new PR using the comment-input contract. | You (manual) | This is the actual user input the product is supposed to learn from. For a confidence-grade demo, the review comments should be intentionally high-signal and human-readable in GitHub itself. | The PR contains at least 2-3 substantive review comments that match the agreed examples/rubric. | +| 8 | **Done:** merge the new feature PR after the review comments are in place. | You (manual) | The workflow triggers from a merged PR, so this is the start of the real automatic demo path. | PR `#3` merged into `main`. | +| 9 | **Done:** run the Phase 8 workflow automatically from that merge. | Automatic GitHub Actions flow | This is the core Phase 8 behavior under demo. | Run `24266078977` started automatically from the PR `#3` merge and completed successfully. | +| 10 | **Failed / diagnosed:** verify that the automatic workflow produces a KB PR containing at least one real article file under `kb//`. If it does not, stop and diagnose instead of pretending the demo is complete. | Copilot | This is the missing proof from the current demo. Without an actual article, we only proved plumbing, not the content path. | Run `24266078977` produced no KB PR, no article files, and a classification-failure-only result (`18 failed`). | +| 11 | Review the generated article(s) against the article-fidelity checklist. | You (manual) | A complete demo needs human confirmation that the published knowledge is actually correct and useful, not just mechanically generated. | You are satisfied that the article reflects the source review comment accurately enough to merge. | +| 12 | Merge the generated KB PR. | You (manual) | This proves the publication loop closes successfully on real article content. | The KB PR is merged into `main`. | +| 13 | Let the post-KB-merge workflow run automatically and verify that it succeeds without opening another KB PR. | Automatic GitHub Actions flow | This proves the system reaches a stable steady state after publication. | Latest workflow run succeeds and no new automation PR is opened. | +| 14 | Perform the final repo-state verification: article exists on `main`, `kb/INDEX.md` links to it, `.manifest.json` maps it, latest workflow is green, and there are no open automation PRs created by this workflow. | Copilot | This is the final proof that the demo finished cleanly and left the repo in the expected end state. | All exit criteria above are satisfied simultaneously. | + +## Non-negotiable gate + +If the next merged demo PR again produces: + +- no article files, +- only `kb/.manifest.json` and `kb/INDEX.md`, or +- another classification-failure-only run, + +then the demo is **not complete** and we must not claim the happy-path demo is done. In that case, the next task is to diagnose and fix the classifier/content path before continuing. + +## Expected final repo state when the demo is truly complete + +`github-pr-kb-demo` should end with all of the following: + +- the feature PR merged, +- the generated KB PR merged, +- at least one article file under `kb//`, +- `kb/INDEX.md` linking to the article, +- `kb/.manifest.json` containing the article mapping, +- no open automation PRs created by this workflow, +- latest relevant workflow runs green, +- no follow-up automation churn after the KB PR merge. + +Only then should this demo be considered fully complete as a **happy-path end-to-end demonstration**. diff --git a/src/github_pr_kb/classifier.py b/src/github_pr_kb/classifier.py index 0d3ff94..009c647 100644 --- a/src/github_pr_kb/classifier.py +++ b/src/github_pr_kb/classifier.py @@ -4,6 +4,7 @@ import json import logging import os +import re import tempfile from datetime import datetime, timezone from pathlib import Path @@ -35,6 +36,8 @@ "other", } +_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(.*?)\s*```", re.IGNORECASE | re.DOTALL) + SYSTEM_PROMPT = """You are a technical knowledge classifier for GitHub PR comments. Classify the given comment into exactly one of these 5 categories: @@ -72,6 +75,41 @@ def _write_atomic(path: Path, data: str) -> None: raise +def _parse_classification_response(text: str) -> dict | None: + """Extract a JSON object from bare, fenced, or prose-wrapped model output.""" + stripped = text.strip() + if not stripped: + return None + + decoder = json.JSONDecoder() + candidates: list[str] = [stripped] + candidates.extend(match.group(1).strip() for match in _JSON_FENCE_RE.finditer(stripped)) + + seen: set[str] = set() + for candidate in candidates: + if not candidate or candidate in seen: + continue + seen.add(candidate) + try: + parsed = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + return parsed + + for idx, char in enumerate(stripped): + if char != "{": + continue + try: + parsed, _ = decoder.raw_decode(stripped[idx:]) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + return parsed + + return None + + class PRClassifier: """Reads cached PR comment files, classifies via Claude API, deduplicates by SHA-256 hash.""" @@ -210,9 +248,8 @@ def _classify_comment(self, comment: CommentRecord) -> ClassifiedComment | None: return None text = response.content[0].text - try: - result: dict = json.loads(text) - except json.JSONDecodeError: + result = _parse_classification_response(text) + if result is None: logger.warning( "Could not parse classification JSON for comment %d", comment.comment_id ) diff --git a/tests/test_classifier.py b/tests/test_classifier.py index 1afc5f5..8f9db21 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -226,6 +226,51 @@ def test_parse_failure_returns_none(cache_dir_with_pr): assert body_hash(comment.body) not in classifier._index +def test_markdown_fenced_json_is_parsed(cache_dir_with_pr): + from github_pr_kb.classifier import PRClassifier + + response_text = """```json + {"category": "gotcha", "confidence": 0.88, "summary": "Validate discount rate bounds."} + ```""" + mock_message = make_mock_message(response_text) + + with patch("github_pr_kb.classifier.Anthropic") as MockAnthropic: + mock_client = MagicMock() + mock_client.messages.create.return_value = mock_message + MockAnthropic.return_value = mock_client + + classifier = PRClassifier(cache_dir=cache_dir_with_pr, api_key="sk-ant-fake") + result = classifier.classify_pr(1) + + assert len(result.classifications) == 1 + assert result.classifications[0].category == "gotcha" + assert classifier._failed_count == 0 + + +def test_prose_wrapped_json_is_parsed(cache_dir_with_pr): + from github_pr_kb.classifier import PRClassifier + + response_text = ( + 'Here is the classification result:\n' + '{"category": "domain_knowledge", "confidence": 0.91, ' + '"summary": "Discount order is a business rule."}\n' + "Thanks." + ) + mock_message = make_mock_message(response_text) + + with patch("github_pr_kb.classifier.Anthropic") as MockAnthropic: + mock_client = MagicMock() + mock_client.messages.create.return_value = mock_message + MockAnthropic.return_value = mock_client + + classifier = PRClassifier(cache_dir=cache_dir_with_pr, api_key="sk-ant-fake") + result = classifier.classify_pr(1) + + assert len(result.classifications) == 1 + assert result.classifications[0].category == "domain_knowledge" + assert classifier._failed_count == 0 + + def test_load_index_filters_failed(tmp_path): from github_pr_kb.classifier import PRClassifier From 43d7a75a4ab978911b9dad07e0ec83583899e906 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 09:41:38 +0300 Subject: [PATCH 16/28] fix: validate parsed classifier payloads Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/github_pr_kb/classifier.py | 25 +++++++++++++++++-- tests/test_classifier.py | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/github_pr_kb/classifier.py b/src/github_pr_kb/classifier.py index 009c647..861ce2d 100644 --- a/src/github_pr_kb/classifier.py +++ b/src/github_pr_kb/classifier.py @@ -94,7 +94,7 @@ def _parse_classification_response(text: str) -> dict | None: parsed = json.loads(candidate) except json.JSONDecodeError: continue - if isinstance(parsed, dict): + if _is_valid_classification_payload(parsed): return parsed for idx, char in enumerate(stripped): @@ -104,12 +104,33 @@ def _parse_classification_response(text: str) -> dict | None: parsed, _ = decoder.raw_decode(stripped[idx:]) except json.JSONDecodeError: continue - if isinstance(parsed, dict): + if _is_valid_classification_payload(parsed): return parsed return None +def _is_valid_classification_payload(parsed: object) -> bool: + """Return True only for dicts that satisfy the classifier response contract.""" + if not isinstance(parsed, dict): + return False + + category = parsed.get("category") + summary = parsed.get("summary") + if not isinstance(category, str) or not isinstance(summary, str): + return False + + if "confidence" not in parsed: + return False + + try: + float(parsed["confidence"]) + except (TypeError, ValueError): + return False + + return True + + class PRClassifier: """Reads cached PR comment files, classifies via Claude API, deduplicates by SHA-256 hash.""" diff --git a/tests/test_classifier.py b/tests/test_classifier.py index 8f9db21..58b26d1 100644 --- a/tests/test_classifier.py +++ b/tests/test_classifier.py @@ -271,6 +271,51 @@ def test_prose_wrapped_json_is_parsed(cache_dir_with_pr): assert classifier._failed_count == 0 +def test_unrelated_json_dict_is_skipped_before_valid_classification(cache_dir_with_pr): + from github_pr_kb.classifier import PRClassifier + + response_text = ( + 'Noise: {"status": "ok"}\n' + '{"category": "code_pattern", "confidence": 0.83, ' + '"summary": "Reuse one validation flow for pricing helpers."}' + ) + mock_message = make_mock_message(response_text) + + with patch("github_pr_kb.classifier.Anthropic") as MockAnthropic: + mock_client = MagicMock() + mock_client.messages.create.return_value = mock_message + MockAnthropic.return_value = mock_client + + classifier = PRClassifier(cache_dir=cache_dir_with_pr, api_key="sk-ant-fake") + result = classifier.classify_pr(1) + + assert len(result.classifications) == 1 + assert result.classifications[0].category == "code_pattern" + assert classifier._failed_count == 0 + + +def test_schema_less_json_object_is_rejected(cache_dir_with_pr): + from github_pr_kb.classifier import PRClassifier, body_hash + + mock_message = make_mock_message("{}") + + with patch("github_pr_kb.classifier.Anthropic") as MockAnthropic: + mock_client = MagicMock() + mock_client.messages.create.return_value = mock_message + MockAnthropic.return_value = mock_client + + classifier = PRClassifier(cache_dir=cache_dir_with_pr, api_key="sk-ant-fake") + comment = PRFile.model_validate_json( + (cache_dir_with_pr / "pr-1.json").read_text(encoding="utf-8") + ).comments[0] + + result = classifier._classify_comment(comment) + + assert result is None + assert classifier._failed_count == 1 + assert body_hash(comment.body) not in classifier._index + + def test_load_index_filters_failed(tmp_path): from github_pr_kb.classifier import PRClassifier From 504227184649280461c37802ac86b1073d6fdb8a Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 09:50:30 +0300 Subject: [PATCH 17/28] fix: skip Qodo issue comment boilerplate Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/github_pr_kb/extractor.py | 13 ++++++++ tests/test_extractor.py | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/github_pr_kb/extractor.py b/src/github_pr_kb/extractor.py index c467ff3..d0fe962 100644 --- a/src/github_pr_kb/extractor.py +++ b/src/github_pr_kb/extractor.py @@ -38,6 +38,10 @@ DEFAULT_CACHE_DIR = Path(".github-pr-kb/cache") REACTION_KEYS = ["+1", "-1", "laugh", "hooray", "confused", "heart", "rocket", "eyes"] +QODO_BOILERPLATE_ISSUE_MARKERS = frozenset({ + "

Code Review by Qodo

", + "

Review Summary by Qodo

", +}) class RateLimitExhaustedError(Exception): @@ -58,6 +62,13 @@ def is_noise(login: str, body: str) -> bool: return False +def is_boilerplate_issue_comment(login: str, body: str, comment_type: str) -> bool: + """Return True for known non-knowledge-bearing issue comment templates.""" + if comment_type != "issue" or login != "qodo-code-review[bot]": + return False + return any(marker in body for marker in QODO_BOILERPLATE_ISSUE_MARKERS) + + def _extract_reactions(raw_reactions: Mapping[str, int]) -> dict[str, int]: """Extract non-zero reaction counts from a reactions dict. @@ -91,6 +102,8 @@ def _comment_to_record( ) -> Optional[CommentRecord]: """Build a CommentRecord from a PyGithub comment, or return None if noise.""" login = c.user.login if c.user is not None else "[deleted]" + if is_boilerplate_issue_comment(login, c.body, comment_type): + return None if is_noise(login, c.body): return None return CommentRecord( diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 589c44f..e0d3e6d 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -154,6 +154,69 @@ def test_review_bot_kept(tmp_path): assert pr_file.comments[0].author == "github-copilot[bot]" +def test_qodo_issue_boilerplate_is_filtered(tmp_path): + """Known Qodo issue-comment templates are ignored as automation noise.""" + cache_dir = tmp_path / "cache" + pr = make_mock_pr(number=13) + pr.get_issue_comments.return_value = [ + make_mock_issue_comment( + comment_id=6001, + login="qodo-code-review[bot]", + body=( + "

Code Review by Qodo

\n\n" + "Looking for bugs?\n" + "Check back in a few minutes. An AI review agent is analyzing this pull request." + ), + ), + make_mock_issue_comment( + comment_id=6002, + login="qodo-code-review[bot]", + body=( + "

Review Summary by Qodo

\n\n" + "Automated summary for file changes and walkthroughs." + ), + ), + ] + + with patch("github_pr_kb.extractor.Github") as MockGithub: + mock_repo = MagicMock() + mock_repo.get_pulls.return_value = [pr] + MockGithub.return_value.get_repo.return_value = mock_repo + + extractor = GitHubExtractor("owner/repo", cache_dir=cache_dir) + extractor.extract() + + data = json.loads((cache_dir / "pr-13.json").read_text()) + pr_file = PRFile.model_validate(data) + assert len(pr_file.comments) == 0 + + +def test_qodo_review_comment_is_kept(tmp_path): + """Substantive Qodo inline review comments remain eligible input.""" + cache_dir = tmp_path / "cache" + pr = make_mock_pr(number=14) + pr.get_review_comments.return_value = [ + make_mock_review_comment( + comment_id=7001, + login="qodo-code-review[bot]", + body="Consider validating discount_rate so values outside 0..1 do not create bad totals.", + ), + ] + + with patch("github_pr_kb.extractor.Github") as MockGithub: + mock_repo = MagicMock() + mock_repo.get_pulls.return_value = [pr] + MockGithub.return_value.get_repo.return_value = mock_repo + + extractor = GitHubExtractor("owner/repo", cache_dir=cache_dir) + extractor.extract() + + data = json.loads((cache_dir / "pr-14.json").read_text()) + pr_file = PRFile.model_validate(data) + assert len(pr_file.comments) == 1 + assert pr_file.comments[0].author == "qodo-code-review[bot]" + + def test_state_filter(tmp_path): """get_pulls called with state='closed' when state filter is 'closed'.""" cache_dir = tmp_path / "cache" From a1c9a68ccdde00ac28273a0b815955c7d5b85dae Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 09:53:58 +0300 Subject: [PATCH 18/28] fix: skip automation kb pull requests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/github_pr_kb/config.py | 1 + src/github_pr_kb/extractor.py | 11 ++++++++ tests/test_extractor.py | 51 ++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/github_pr_kb/config.py b/src/github_pr_kb/config.py index 24dcad5..c02a372 100644 --- a/src/github_pr_kb/config.py +++ b/src/github_pr_kb/config.py @@ -14,6 +14,7 @@ class Settings(BaseSettings): anthropic_api_key: str | None = None # Phase 4: required for classify command anthropic_model: str | None = None # Override default classifier model via ANTHROPIC_MODEL kb_output_dir: str = "kb" # Phase 5: KB output directory (D-17) + kb_bot_branch: str = "automation/github-pr-kb" anthropic_generate_model: str | None = None min_confidence: float = 0.5 diff --git a/src/github_pr_kb/extractor.py b/src/github_pr_kb/extractor.py index d0fe962..aa4bdaf 100644 --- a/src/github_pr_kb/extractor.py +++ b/src/github_pr_kb/extractor.py @@ -119,6 +119,13 @@ def _comment_to_record( ) +def _is_automation_kb_pr(pr: PullRequest) -> bool: + """Return True when the PR is the tool's rolling KB publication PR.""" + head = getattr(pr, "head", None) + head_ref = getattr(head, "ref", None) + return head_ref == settings.kb_bot_branch + + class GitHubExtractor: """Authenticates with GitHub via PAT and extracts PR comments into per-PR JSON cache files.""" @@ -257,6 +264,10 @@ def extract( pulls = self.repo.get_pulls(state=state, sort="updated", direction="desc") for pr in pulls: + if _is_automation_kb_pr(pr): + logger.info("PR #%d: skipped automation KB PR", pr.number) + continue + # Early-stop: PRs are sorted desc by updated_at. # Once we see a PR older than since, all remaining are older too. if since is not None and pr.updated_at < since: diff --git a/tests/test_extractor.py b/tests/test_extractor.py index e0d3e6d..84746a5 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -13,7 +13,14 @@ # Mock helpers # --------------------------------------------------------------------------- -def make_mock_pr(number=42, state="open", updated_at=None, title="Test PR", body="PR description"): +def make_mock_pr( + number=42, + state="open", + updated_at=None, + title="Test PR", + body="PR description", + head_ref="feature/test-pr", +): pr = MagicMock() pr.number = number pr.title = title @@ -21,6 +28,8 @@ def make_mock_pr(number=42, state="open", updated_at=None, title="Test PR", body pr.state = state pr.html_url = f"https://github.com/owner/repo/pull/{number}" pr.updated_at = updated_at or datetime(2024, 1, 15, tzinfo=timezone.utc) + pr.head = MagicMock() + pr.head.ref = head_ref pr.get_review_comments.return_value = [] pr.get_issue_comments.return_value = [] return pr @@ -217,6 +226,46 @@ def test_qodo_review_comment_is_kept(tmp_path): assert pr_file.comments[0].author == "qodo-code-review[bot]" +def test_automation_kb_pr_is_skipped(tmp_path): + """The rolling KB publication PR is excluded from extraction entirely.""" + cache_dir = tmp_path / "cache" + automation_pr = make_mock_pr( + number=15, + title="chore: update PR knowledge base", + head_ref="automation/github-pr-kb", + ) + automation_pr.get_issue_comments.return_value = [ + make_mock_issue_comment( + comment_id=8001, + login="alice", + body="This explanatory close comment should not be republished.", + ), + ] + normal_pr = make_mock_pr(number=16, title="feat: keep me") + normal_pr.get_issue_comments.return_value = [ + make_mock_issue_comment( + comment_id=8002, + login="bob", + body="This normal PR comment should still be extracted.", + ), + ] + + with patch("github_pr_kb.extractor.Github") as MockGithub: + mock_repo = MagicMock() + mock_repo.get_pulls.return_value = [automation_pr, normal_pr] + MockGithub.return_value.get_repo.return_value = mock_repo + + extractor = GitHubExtractor("owner/repo", cache_dir=cache_dir) + paths = extractor.extract() + + assert paths == [cache_dir / "pr-16.json"] + assert not (cache_dir / "pr-15.json").exists() + data = json.loads((cache_dir / "pr-16.json").read_text()) + pr_file = PRFile.model_validate(data) + assert len(pr_file.comments) == 1 + assert pr_file.comments[0].comment_id == 8002 + + def test_state_filter(tmp_path): """get_pulls called with state='closed' when state filter is 'closed'.""" cache_dir = tmp_path / "cache" From 8ea86dd741d1ea6e190264c2ca891c0c5d514891 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 10:12:04 +0300 Subject: [PATCH 19/28] fix: ignore Qodo issue comments Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/github_pr_kb/extractor.py | 14 ++++---------- tests/test_extractor.py | 15 ++++----------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/github_pr_kb/extractor.py b/src/github_pr_kb/extractor.py index aa4bdaf..b6bd0ed 100644 --- a/src/github_pr_kb/extractor.py +++ b/src/github_pr_kb/extractor.py @@ -38,10 +38,6 @@ DEFAULT_CACHE_DIR = Path(".github-pr-kb/cache") REACTION_KEYS = ["+1", "-1", "laugh", "hooray", "confused", "heart", "rocket", "eyes"] -QODO_BOILERPLATE_ISSUE_MARKERS = frozenset({ - "

Code Review by Qodo

", - "

Review Summary by Qodo

", -}) class RateLimitExhaustedError(Exception): @@ -62,11 +58,9 @@ def is_noise(login: str, body: str) -> bool: return False -def is_boilerplate_issue_comment(login: str, body: str, comment_type: str) -> bool: - """Return True for known non-knowledge-bearing issue comment templates.""" - if comment_type != "issue" or login != "qodo-code-review[bot]": - return False - return any(marker in body for marker in QODO_BOILERPLATE_ISSUE_MARKERS) +def is_ignored_issue_comment(login: str, comment_type: str) -> bool: + """Return True for automation-authored issue comments that should never be published.""" + return comment_type == "issue" and login == "qodo-code-review[bot]" def _extract_reactions(raw_reactions: Mapping[str, int]) -> dict[str, int]: @@ -102,7 +96,7 @@ def _comment_to_record( ) -> Optional[CommentRecord]: """Build a CommentRecord from a PyGithub comment, or return None if noise.""" login = c.user.login if c.user is not None else "[deleted]" - if is_boilerplate_issue_comment(login, c.body, comment_type): + if is_ignored_issue_comment(login, comment_type): return None if is_noise(login, c.body): return None diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 84746a5..f7f9f31 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -163,27 +163,20 @@ def test_review_bot_kept(tmp_path): assert pr_file.comments[0].author == "github-copilot[bot]" -def test_qodo_issue_boilerplate_is_filtered(tmp_path): - """Known Qodo issue-comment templates are ignored as automation noise.""" +def test_qodo_issue_comments_are_filtered(tmp_path): + """Qodo issue comments are ignored, even when they contain substantive text.""" cache_dir = tmp_path / "cache" pr = make_mock_pr(number=13) pr.get_issue_comments.return_value = [ make_mock_issue_comment( comment_id=6001, login="qodo-code-review[bot]", - body=( - "

Code Review by Qodo

\n\n" - "Looking for bugs?\n" - "Check back in a few minutes. An AI review agent is analyzing this pull request." - ), + body="Automated review summary placeholder from Qodo.", ), make_mock_issue_comment( comment_id=6002, login="qodo-code-review[bot]", - body=( - "

Review Summary by Qodo

\n\n" - "Automated summary for file changes and walkthroughs." - ), + body="calculate_total_with_capped_percentage_discount() can produce negative subtotals when the cap exceeds subtotal.", ), ] From 89cf0ec324db8b1a90b307f38e80f3fd31ba6354 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 10:34:00 +0300 Subject: [PATCH 20/28] docs(phase-8): add security threat verification Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../08-github-action-readme/08-SECURITY.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .planning/phases/08-github-action-readme/08-SECURITY.md diff --git a/.planning/phases/08-github-action-readme/08-SECURITY.md b/.planning/phases/08-github-action-readme/08-SECURITY.md new file mode 100644 index 0000000..f407f27 --- /dev/null +++ b/.planning/phases/08-github-action-readme/08-SECURITY.md @@ -0,0 +1,77 @@ +--- +phase: 8 +slug: github-action-readme +status: verified +threats_open: 0 +asvs_level: 1 +created: 2026-04-11 +updated: 2026-04-11 +--- + +# Phase 8 — Security + +> Per-phase security contract: threat register, accepted risks, and audit trail. + +--- + +## Trust Boundaries + +| Boundary | Description | Data Crossing | +|----------|-------------|---------------| +| workflow inputs -> action_state helper | GitHub event payload fields and manual dispatch inputs drive skip/cursor decisions | `pull_request.updated_at`, `latest_merged_at`, `manual_since`, `force`, stored cursor | +| action_state helper -> workflow outputs | Helper output decides whether the pipeline runs and what cursor candidate may be persisted | `should_run`, `extract_since`, `next_cursor`, `reason` | +| workflow job -> repository variable API | The workflow writes durable incremental state outside git | `KB_LAST_SUCCESSFUL_CURSOR` | +| workflow job -> rolling PR branch | The workflow can publish tracked KB output to a dedicated branch and PR | `kb/INDEX.md`, `kb/**/*.md`, `kb/.manifest.json` | +| README guidance -> maintainer setup | Documentation influences token setup, workflow bootstrap, and repo hygiene in consumer repos | workflow file path, secret names, `KB_TOOL_REPOSITORY`, `KB_TOOL_REF`, token roles | +| README env docs -> local CLI usage | Users populate local config and run commands from the documented env surface | `.env.example`, local tokens, CLI commands, platform-specific test commands | + +--- + +## Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation | Status | +|-----------|----------|-----------|-------------|------------|--------| +| T-08-01 | T | `src/github_pr_kb/action_state.py` timestamp parsing and input contract | mitigate | `action_state.py` normalizes timestamps to UTC, rejects malformed or timezone-naive input, and `tests/test_action_state.py` locks deterministic CLI failure behavior. | closed | +| T-08-02 | D | manual dispatch skip/cursor logic | mitigate | Manual no-new-PR short-circuit, `force` bypass, and non-regressing `next_cursor` behavior are implemented in `action_state.py` and covered by `tests/test_action_state.py`. | closed | +| T-08-03 | R | helper output -> workflow persistence handoff | mitigate | `action_state.py` documents `next_cursor` as candidate-only, and `.github/workflows/github-pr-kb.yml` re-reads the stored cursor and persists the monotonic max under workflow concurrency. | closed | +| T-08-04 | S | variable-auth token selection | mitigate | `.github/workflows/github-pr-kb.yml` prefers the GitHub App path when app secrets are present, falls back to `KB_VARIABLES_TOKEN`, and fails closed when neither auth path is configured; `tests/test_action_workflow.py` covers the contract. | closed | +| T-08-05 | T | cursor persistence step | mitigate | The workflow writes `KB_LAST_SUCCESSFUL_CURSOR` only after extract/classify/generate and publication steps succeed, preventing premature state advancement. | closed | +| T-08-06 | I | git publication scope | mitigate | The workflow stages only `kb/INDEX.md`, `kb/.manifest.json`, and `kb/**/*.md`; it does not stage `.github-pr-kb/cache/`, and workflow tests verify the staging contract. | closed | +| T-08-07 | D | rerun/no-new-PR behavior | mitigate | The workflow invokes `python -m github_pr_kb.action_state` before the CLI pipeline and exits early on no-op runs; helper and workflow tests both cover this guard. | closed | +| T-08-08 | T | out-of-order workflow completion | mitigate | Workflow-level concurrency plus post-publication fresh-read/max-write cursor persistence prevent overlapping runs from regressing durable state. | closed | +| T-08-09 | S | external workflow/tool refs | mitigate | `.github/workflows/github-pr-kb.yml` pins third-party actions by full commit SHA and treats `KB_TOOL_REF` as an immutable pinned tool reference; tests assert the pinned contract. | closed | +| T-08-10 | D | rolling PR branch drift/conflict | mitigate | The workflow uses dedicated branch `automation/github-pr-kb`, updates it with a bounded safe strategy, and performs cursor persistence only after publication succeeds. | closed | +| T-08-08-docs | S | auth setup documentation | mitigate | `README.md` documents PAT quickstart before the GitHub App path, names the exact workflow-only secrets, separates local/runtime auth from repository-variable auth, and explains the internal `GH_TOKEN` mapping; `tests/test_readme.py` verifies those promises. | closed | +| T-08-09-docs | T | repo hygiene documentation | mitigate | `README.md` includes explicit committed-vs-not-committed guidance covering `kb/` output, `kb/.manifest.json`, and `.github-pr-kb/cache/`; `tests/test_readme.py` enforces it. | closed | +| T-08-10-docs | I | README and `.env.example` examples | mitigate | `README.md` and `.env.example` align on the local config surface and explicitly keep workflow-only secrets out of `.env`; README review shows no stale architecture/setup claims remain. | closed | +| T-08-11 | D | platform-specific local guidance | mitigate | `README.md` provides distinct Windows and macOS/Linux setup and test commands so the onboarding path does not assume one platform as universal. | closed | + +*Status: open · closed* +*Disposition: mitigate (implementation required) · accept (documented risk) · transfer (third-party)* + +--- + +## Accepted Risks Log + +No accepted risks. + +--- + +## Security Audit Trail + +| Audit Date | Threats Total | Closed | Open | Run By | +|------------|---------------|--------|------|--------| +| 2026-04-11 | 14 | 14 | 0 | Copilot | + +No `## Threat Flags` sections were present in the Phase 8 summary artifacts, so the audit was based on the plan threat models plus the implemented workflow, helper, docs, and tests. + +--- + +## Sign-Off + +- [x] All threats have a disposition (mitigate / accept / transfer) +- [x] Accepted risks documented in Accepted Risks Log +- [x] `threats_open: 0` confirmed +- [x] `status: verified` set in frontmatter + +**Approval:** verified 2026-04-11 From 8cfbb36ef252018370d2251cbd6b857de1af4ecd Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 10:36:41 +0300 Subject: [PATCH 21/28] docs(phase-8): update nyquist validation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../08-github-action-readme/08-VALIDATION.md | 101 +++++++++++------- 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/.planning/phases/08-github-action-readme/08-VALIDATION.md b/.planning/phases/08-github-action-readme/08-VALIDATION.md index 6508083..e7fe6e9 100644 --- a/.planning/phases/08-github-action-readme/08-VALIDATION.md +++ b/.planning/phases/08-github-action-readme/08-VALIDATION.md @@ -1,10 +1,11 @@ --- phase: 8 slug: github-action-readme -status: draft -nyquist_compliant: false -wave_0_complete: false +status: complete +nyquist_compliant: true +wave_0_complete: true created: 2026-04-09 +updated: 2026-04-11 --- # Phase 8 — Validation Strategy @@ -17,63 +18,91 @@ created: 2026-04-09 | Property | Value | |----------|-------| -| **Framework** | pytest (existing) | -| **Config file** | `pyproject.toml` | -| **Quick run command** | `.venv/Scripts/python.exe -m pytest tests/ -k "workflow or action or readme or cursor" -q` | -| **Full suite command** | `.venv/Scripts/python.exe -m pytest tests/ -q` | -| **Estimated runtime** | ~10 seconds | +| **Framework** | pytest | +| **Config file** | `pyproject.toml` (`[tool.pytest.ini_options]`) | +| **Quick run command** | `.\.venv\Scripts\python.exe -m pytest tests\test_action_state.py tests\test_action_workflow.py tests\test_readme.py -q` | +| **Full suite command** | `.\.venv\Scripts\python.exe -m pytest tests -q` | +| **Estimated runtime** | ~3.59s targeted / ~6.67s full (observed on 2026-04-11) | --- ## Sampling Rate -- **After every task commit:** Run `.venv/Scripts/python.exe -m pytest tests/ -k "workflow or action or readme or cursor" -q` -- **After every plan wave:** Run `.venv/Scripts/python.exe -m pytest tests/ -q` -- **Before `/gsd-verify-work`:** Full suite must be green -- **Max feedback latency:** 10 seconds +- **After every task commit:** Run the targeted pytest file for the touched Phase 8 surface (`test_action_state.py`, `test_action_workflow.py`, or `test_readme.py`). +- **After every plan wave:** Run `.\.venv\Scripts\python.exe -m pytest tests -q`. +- **Before `/gsd-verify-work`:** Full suite must be green. +- **Max feedback latency:** ~7 seconds for the full pytest audit in this repository. --- ## Per-Task Verification Map -| Task ID | Plan | Wave | Requirement | Test Type | Automated Command | File Exists | Status | -|---------|------|------|-------------|-----------|-------------------|-------------|--------| -| 08-01-01 | 01 | 0 | ACTION-01 | unit / file assertion | `.venv/Scripts/python.exe -m pytest tests/ -k "workflow and trigger" -q` | Likely W0 | pending | -| 08-01-02 | 01 | 0 | ACTION-02 | unit | `.venv/Scripts/python.exe -m pytest tests/ -k "no_new_pr or cost_guard" -q` | Likely W0 | pending | -| 08-01-03 | 01 | 0 | ACTION-03 | unit | `.venv/Scripts/python.exe -m pytest tests/ -k "cursor or state" -q` | Likely W0 | pending | -| 08-01-04 | 01 | 0 | ACTION-01/ACTION-03 | integration-lite | `.venv/Scripts/python.exe -m pytest tests/ -k "cache or manifest" -q` | Likely W0 | pending | -| 08-02-01 | 02 | 1 | INFRA-03 | doc contract | `.venv/Scripts/python.exe -m pytest tests/ -k "readme" -q` | Likely W0 | pending | +| Task ID | Plan | Wave | Requirement | Threat Ref | Secure Behavior | Test Type | Automated Command | File Exists | Status | +|---------|------|------|-------------|------------|-----------------|-----------|-------------------|-------------|--------| +| 08-01-01 | 01 | 1 | ACTION-02, ACTION-03 | T-08-01, T-08-02 | Auto and manual cursor decisions are deterministic, reject malformed timestamps, and no-op repeated auto events instead of rerunning expensive work. | unit | `.\.venv\Scripts\python.exe -m pytest tests\test_action_state.py -x -q` | ✅ | ✅ green | +| 08-01-02 | 01 | 1 | ACTION-02, ACTION-03 | T-08-01, T-08-02, T-08-03 | `python -m github_pr_kb.action_state` stays import-safe, emits candidate-only cursor JSON, and preserves non-regressing manual state. | unit | `.\.venv\Scripts\python.exe -m pytest tests\test_action_state.py -x -q` | ✅ | ✅ green | +| 08-02-01 | 02 | 2 | ACTION-01, ACTION-02, ACTION-03 | T-08-04, T-08-07, T-08-08, T-08-09 | The shipped workflow is copyable, pinned, concurrency-safe, and correctly wired to helper execution plus workflow auth paths. | contract | `.\.venv\Scripts\python.exe -m pytest tests\test_action_workflow.py -x -q` | ✅ | ✅ green | +| 08-02-02 | 02 | 2 | ACTION-01, ACTION-02, ACTION-03 | T-08-05, T-08-06, T-08-08, T-08-10 | Publication stages only KB artifacts, reuses the rolling automation PR, and persists `KB_LAST_SUCCESSFUL_CURSOR` only after successful publication with monotonic max semantics. | contract | `.\.venv\Scripts\python.exe -m pytest tests\test_action_workflow.py -x -q` | ✅ | ✅ green | +| 08-03-01 | 03 | 3 | INFRA-03 | T-08-08-docs, T-08-09-docs, T-08-10-docs | README and `.env.example` stay aligned with the shipped workflow, local config surface, and committed-vs-not-committed guidance. | docs contract | `.\.venv\Scripts\python.exe -m pytest tests\test_readme.py -x -q` | ✅ | ✅ green | +| 08-03-02 | 03 | 3 | INFRA-03 | T-08-11 | Local onboarding remains platform-aware and documents install, env vars, CLI commands, and KB output examples. | docs contract | `.\.venv\Scripts\python.exe -m pytest tests\test_readme.py -x -q` | ✅ | ✅ green | -*Status: pending / green / red / flaky* +*Status: ⬜ pending · ✅ green · ❌ red · ⚠️ flaky* --- ## Wave 0 Requirements -- [ ] Workflow/state decision tests for merged-only trigger, cursor handling, and cost guard -- [ ] README contract coverage if docs expectations are not already asserted elsewhere - -*Existing infrastructure covers framework and fixtures. Phase 8 should add tests, not new tooling.* +Existing infrastructure covers all phase requirements. --- ## Manual-Only Verifications -| Behavior | Requirement | Why Manual | Test Instructions | -|----------|-------------|------------|-------------------| -| Rolling bot PR is created once and updated on subsequent successful runs | ACTION-01 | Requires live GitHub PR behavior | Run `workflow_dispatch` in a test repo twice and confirm one open KB PR is updated rather than duplicated | -| No-new-PR run skips extract/classify/generate work in a real repository | ACTION-02 | Requires live workflow/event context | Trigger the workflow with the saved cursor newer than all merged PRs and confirm the pipeline exits without KB changes | -| Dual auth setup docs are understandable for maintainers | INFRA-03 | Human readability / setup burden | Follow the README once with PAT path and once with GitHub App path in a test repo | +All phase behaviors have automated verification. + +--- + +## Field Validation Evidence + +Phase 8 also has live consumer-repo validation in `Galzi1/github-pr-kb-demo`, which supplements the automated Nyquist map above: + +| Behavior | Evidence | Result | +|----------|----------|--------| +| Fresh feature PR with human review comments is ingested into the pipeline | Demo PR `#10` | Pass | +| Final workflow replay on corrected tool SHA | Run `24277455924` | Pass | +| Generated KB PR opened with real article files | Demo PR `#11` | Pass | +| Generated KB PR merged cleanly | Demo PR `#11` merged | Pass | +| Post-merge steady-state run completes without automation churn | Run `24277477172` | Pass | + +This supports the strongest claim that Phase 8 is validated as a **happy-path end-to-end consumer workflow**, while the Nyquist audit confirms every planned requirement already has automated verification in-repo. + +--- + +## Validation Audit 2026-04-11 + +| Metric | Count | +|--------|-------| +| Gaps found | 0 | +| Resolved | 0 | +| Escalated | 0 | + +Audit notes: + +- `workflow.nyquist_validation` is enabled. +- Existing `08-VALIDATION.md` evidence was audited and converted into the structured Nyquist format. +- Automated coverage remains green via: + - `.\.venv\Scripts\python.exe -m pytest tests\test_action_state.py tests\test_action_workflow.py tests\test_readme.py -q` + - `.\.venv\Scripts\python.exe -m pytest tests -q` --- ## Validation Sign-Off -- [ ] All tasks have `` verify or Wave 0 dependencies -- [ ] Sampling continuity: no 3 consecutive tasks without automated verify -- [ ] Wave 0 covers all MISSING references -- [ ] No watch-mode flags -- [ ] Feedback latency < 10s -- [ ] `nyquist_compliant: true` set in frontmatter +- [x] All tasks have `` verify or Wave 0 dependencies +- [x] Sampling continuity: no 3 consecutive tasks without automated verify +- [x] Wave 0 covers all MISSING references +- [x] No watch-mode flags +- [x] Feedback latency < 10s +- [x] `nyquist_compliant: true` set in frontmatter -**Approval:** pending +**Approval:** approved 2026-04-11 From ddf93a5ff4a222ef1ff9d61ee5b9df66d0c950f8 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 10:40:17 +0300 Subject: [PATCH 22/28] docs(08): ship phase 8 - PR #16 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .planning/STATE.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.planning/STATE.md b/.planning/STATE.md index 34d359c..3f32a7a 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -2,13 +2,13 @@ gsd_state_version: 1.0 milestone: v1.0 milestone_name: milestone -status: verifying -stopped_at: Phase 8 plans complete -last_updated: "2026-04-10T09:49:16.639Z" -last_activity: 2026-04-10 +status: shipped +stopped_at: Phase 8 shipped — PR #16 +last_updated: "2026-04-11T07:39:54.989Z" +last_activity: 2026-04-11 progress: total_phases: 8 - completed_phases: 7 + completed_phases: 8 total_plans: 15 completed_plans: 15 percent: 100 @@ -27,8 +27,8 @@ See: .planning/PROJECT.md (updated 2026-03-07) Phase: 8 Plan: 03 complete -Status: Plans complete — ready for verification -Last activity: 2026-04-10 +Status: Phase 8 shipped — PR #16 +Last activity: 2026-04-11 Progress: [██████████] 100% From ec4ce2509de71acd39db401c608f3bef3ea18577 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 11:07:12 +0300 Subject: [PATCH 23/28] fix: address PR #16 review comments - Use Search API for reliable merged-PR lookup in workflow_dispatch (fixes potential skip when first 100 closed PRs have no merges) - Add comment explaining keyword-only (*,) syntax in action_state.py - Remove test_readme.py (README structural tests deemed unnecessary) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/github-pr-kb.yml | 4 +- src/github_pr_kb/action_state.py | 2 +- tests/test_readme.py | 92 ------------------------------ 3 files changed, 3 insertions(+), 95 deletions(-) delete mode 100644 tests/test_readme.py diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml index 2c1424c..5462d5a 100644 --- a/.github/workflows/github-pr-kb.yml +++ b/.github/workflows/github-pr-kb.yml @@ -131,8 +131,8 @@ jobs: run: | set -euo pipefail latest_merged_at="$( - gh api "/repos/${GITHUB_REPOSITORY}/pulls?state=closed&sort=updated&direction=desc&per_page=100" \ - --jq 'map(select(.merged_at != null))[0].updated_at // ""' + gh api "/search/issues?q=repo:${GITHUB_REPOSITORY}+is:pr+is:merged&sort=updated&order=desc&per_page=1" \ + --jq '.items[0].updated_at // ""' )" echo "value=${latest_merged_at}" >> "$GITHUB_OUTPUT" diff --git a/src/github_pr_kb/action_state.py b/src/github_pr_kb/action_state.py index 9d82916..db92848 100644 --- a/src/github_pr_kb/action_state.py +++ b/src/github_pr_kb/action_state.py @@ -55,7 +55,7 @@ def _isoformat_utc(value: datetime | None) -> str | None: def decide_action_run( - *, + *, # keyword-only: all arguments below must be passed by name event_name: str, merged: bool = False, event_updated_at: TimestampInput = None, diff --git a/tests/test_readme.py b/tests/test_readme.py deleted file mode 100644 index 392f6fa..0000000 --- a/tests/test_readme.py +++ /dev/null @@ -1,92 +0,0 @@ -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parent.parent -README_PATH = REPO_ROOT / "README.md" -ENV_EXAMPLE_PATH = REPO_ROOT / ".env.example" - - -def _readme() -> str: - return README_PATH.read_text(encoding="utf-8") - - -def _env_example() -> str: - return ENV_EXAMPLE_PATH.read_text(encoding="utf-8") - - -def test_readme_is_automation_first() -> None: - text = _readme() - - assert "## Automate with GitHub Actions" in text - assert "## Run locally" in text - assert text.index("## Automate with GitHub Actions") < text.index("## Run locally") - assert ".github/workflows/github-pr-kb.yml" in text - assert "workflow_dispatch" in text - assert "KB_TOOL_REPOSITORY" in text - assert "KB_TOOL_REF" in text - assert "Galzi1/github-pr-kb" in text - assert "copy only the workflow file" in text - assert "do not need this tool's source tree checked into your repository" in text - assert "### PAT quickstart" in text - assert "### GitHub App" in text - assert text.index("KB_VARIABLES_TOKEN") < text.index("KB_VARIABLES_APP_ID") - - -def test_readme_distinguishes_local_and_workflow_credentials() -> None: - text = _readme() - - assert "GITHUB_TOKEN" in text - assert "ANTHROPIC_API_KEY" in text - assert "KB_VARIABLES_TOKEN" in text - assert "KB_VARIABLES_APP_ID" in text - assert "KB_VARIABLES_APP_PRIVATE_KEY" in text - assert "GH_TOKEN" in text - assert "gh api" in text - assert "gh pr" in text - assert "local/runtime credentials" in text - assert "workflow repository-variable credentials" in text - - -def test_readme_documents_install_local_usage_and_git_boundaries() -> None: - text = _readme() - - assert "### Install uv" in text - assert text.index("### Install uv") < text.index("uv sync --all-groups --frozen") - assert "https://docs.astral.sh/uv/getting-started/installation/" in text - assert 'powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"' in text - assert "curl -LsSf https://astral.sh/uv/install.sh | sh" in text - assert ".venv/Scripts/python.exe -m pytest tests/" in text - assert ".venv/bin/python -m pytest tests/" in text - for token in ( - "GITHUB_TOKEN", - "ANTHROPIC_API_KEY", - "ANTHROPIC_MODEL", - "ANTHROPIC_GENERATE_MODEL", - "KB_OUTPUT_DIR", - "MIN_CONFIDENCE", - ): - assert token in text - assert "github-pr-kb extract --repo owner/name" in text - assert "github-pr-kb classify" in text - assert "github-pr-kb generate" in text - assert ".github-pr-kb/cache/" in text - assert "kb/.manifest.json" in text - assert "Committed vs not committed" in text - assert ".env.example" in text - - -def test_env_example_matches_local_config_surface_only() -> None: - text = _env_example() - - for token in ( - "GITHUB_TOKEN", - "ANTHROPIC_API_KEY", - "ANTHROPIC_MODEL", - "ANTHROPIC_GENERATE_MODEL", - "KB_OUTPUT_DIR", - "MIN_CONFIDENCE", - ): - assert token in text - assert "KB_VARIABLES_TOKEN=" not in text - assert "KB_VARIABLES_APP_ID=" not in text - assert "KB_VARIABLES_APP_PRIVATE_KEY=" not in text - assert "repository secrets" in text From a1004dec669889eb7b39c4edbc109d1d0bd57581 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 11:10:22 +0300 Subject: [PATCH 24/28] fix: defensive cursor parsing in persistence step Replace fragile one-liner with inline Python that catches ValueError/TypeError on malformed KB_LAST_SUCCESSFUL_CURSOR values, emits a GitHub Actions warning, and continues with any valid cursor. Prevents late-pipeline crash when cursor is manually repaired with a formatting mistake. Addresses Qodo review: 'Bad cursor breaks persistence' Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/github-pr-kb.yml | 20 +++++++++++++++++++- tests/test_action_workflow.py | 5 +++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-pr-kb.yml b/.github/workflows/github-pr-kb.yml index 5462d5a..4e70ed8 100644 --- a/.github/workflows/github-pr-kb.yml +++ b/.github/workflows/github-pr-kb.yml @@ -300,7 +300,25 @@ jobs: exit "${status}" fi rm -f "${stderr_file}" - persisted_cursor="$(python -c "from datetime import datetime, timezone; import sys; values = [datetime.fromisoformat(raw.replace('Z', '+00:00')).astimezone(timezone.utc) for raw in sys.argv[1:] if raw]; print(max(values).isoformat().replace('+00:00', 'Z') if values else '')" "${fresh_cursor}" "${candidate_cursor}")" + persisted_cursor="$(python - "${fresh_cursor}" "${candidate_cursor}" <<'PY' + import sys + from datetime import datetime, timezone + + valid = [] + for raw in sys.argv[1:]: + if not raw: + continue + try: + dt = datetime.fromisoformat(raw.replace("Z", "+00:00")) + valid.append(dt.astimezone(timezone.utc)) + except (ValueError, TypeError): + print(f"::warning::Ignoring unparsable cursor value: {raw}", file=sys.stderr) + if valid: + print(max(valid).isoformat().replace("+00:00", "Z")) + else: + print("") + PY + )" if [ "${variable_exists}" = "true" ]; then gh api --method PATCH "${endpoint}" -f name="${KB_VARIABLE_NAME}" -f value="${persisted_cursor}" >/dev/null else diff --git a/tests/test_action_workflow.py b/tests/test_action_workflow.py index c531bde..5d40f37 100644 --- a/tests/test_action_workflow.py +++ b/tests/test_action_workflow.py @@ -88,8 +88,9 @@ def test_workflow_publishes_rolling_pr_and_persists_cursor_monotonically() -> No assert "actions/variables/${KB_VARIABLE_NAME}" in text assert "KB_LAST_SUCCESSFUL_CURSOR" in text assert "max(" in text - assert 'persisted_cursor="$(python -c "' in text - assert 'persisted_cursor="$(\n' not in text + assert 'persisted_cursor="$(python -' in text + assert "fromisoformat" in text + assert "::warning::Ignoring unparsable cursor value" in text def test_action_state_helper_command_runs_from_project_checkout() -> None: From 54f8da34bd4b6380b032d7fa70a4d7c3b82f96d3 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 11:39:38 +0300 Subject: [PATCH 25/28] feat: add E2E pipeline integration test against demo repo - New test_e2e_pipeline.py runs extract -> classify -> generate against Galzi1/github-pr-kb-demo with real API keys - 9 assertions covering cache files, classification, KB articles, frontmatter, manifest, INDEX.md, and category structure - Gated: requires RUN_INTEGRATION_TESTS=1 + real tokens; skipped in unit test runs (marked @pytest.mark.e2e) - New e2e-pipeline CI job runs only on pushes to main (cost guard) - Uses same pinned actions (checkout, setup-uv) as existing jobs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 23 +++++ tests/test_e2e_pipeline.py | 206 +++++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 tests/test_e2e_pipeline.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51309fd..748a142 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,3 +59,26 @@ jobs: - name: Run phase 7 e2e harness run: .venv/bin/python -m pytest tests/test_phase7_uat_envs.py -v -m e2e + + e2e-pipeline: + name: e2e-pipeline + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: uv sync --all-groups --frozen + + - name: Run E2E pipeline against demo repo + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + RUN_INTEGRATION_TESTS: "1" + run: .venv/bin/python -m pytest tests/test_e2e_pipeline.py -v -m e2e diff --git a/tests/test_e2e_pipeline.py b/tests/test_e2e_pipeline.py new file mode 100644 index 0000000..b4c8f82 --- /dev/null +++ b/tests/test_e2e_pipeline.py @@ -0,0 +1,206 @@ +"""End-to-end pipeline test: extract → classify → generate against github-pr-kb-demo. + +Run with: + RUN_INTEGRATION_TESTS=1 .venv/Scripts/python.exe -m pytest tests/test_e2e_pipeline.py -v -m e2e + +Skipped automatically unless all conditions hold: + - RUN_INTEGRATION_TESTS=1 env var is set + - GITHUB_TOKEN is a real token (not the unit-test dummy value) + - ANTHROPIC_API_KEY is a real key (not the unit-test dummy value) +""" + +import json +import os +import re + +import pytest + +_DUMMY_GH_TOKEN = "ghp_test000000000000000000000000000fake" +_DUMMY_ANTHROPIC_KEY = "sk-ant-test000000000000000000000000000fake" +_SKIP_REASON = ( + "E2E pipeline tests require RUN_INTEGRATION_TESTS=1, " + "a real GITHUB_TOKEN, and a real ANTHROPIC_API_KEY" +) + +DEMO_REPO = "Galzi1/github-pr-kb-demo" +VALID_CATEGORIES = { + "architecture_decision", + "code_pattern", + "gotcha", + "domain_knowledge", + "other", +} + + +def _e2e_enabled() -> bool: + if os.getenv("RUN_INTEGRATION_TESTS") != "1": + return False + from github_pr_kb.config import settings + + if settings.github_token == _DUMMY_GH_TOKEN: + return False + api_key = settings.anthropic_api_key + return api_key is not None and api_key != _DUMMY_ANTHROPIC_KEY + + +pytestmark = [ + pytest.mark.e2e, + pytest.mark.skipif(not _e2e_enabled(), reason=_SKIP_REASON), +] + + +@pytest.fixture(scope="module") +def pipeline_dirs(tmp_path_factory: pytest.TempPathFactory) -> dict: + """Run the full extract → classify → generate pipeline once, return paths.""" + cache_dir = tmp_path_factory.mktemp("e2e_cache") + kb_dir = tmp_path_factory.mktemp("e2e_kb") + + # --- Extract --- + from github_pr_kb.extractor import GitHubExtractor + + extractor = GitHubExtractor(DEMO_REPO, cache_dir=cache_dir) + paths = extractor.extract(state="closed") + assert len(paths) > 0, "Extraction produced no PR cache files" + + # --- Classify --- + from github_pr_kb.classifier import PRClassifier + + classifier = PRClassifier(cache_dir=cache_dir) + classifier.classify_all() + + # --- Generate --- + from github_pr_kb.generator import KBGenerator + + generator = KBGenerator(cache_dir=cache_dir, kb_dir=kb_dir) + result = generator.generate_all() + assert result.written > 0 or result.skipped > 0, "Generation produced no articles" + + return {"cache_dir": cache_dir, "kb_dir": kb_dir, "gen_result": result} + + +# --------------------------------------------------------------------------- +# Extraction assertions +# --------------------------------------------------------------------------- + + +def test_extraction_produces_pr_cache_files(pipeline_dirs: dict) -> None: + """At least one PR cache file exists and contains valid JSON.""" + cache_dir = pipeline_dirs["cache_dir"] + pr_files = sorted(cache_dir.glob("pr-*.json")) + assert len(pr_files) >= 1 + + from github_pr_kb.models import PRFile + + for pf in pr_files: + data = json.loads(pf.read_text(encoding="utf-8")) + pr = PRFile.model_validate(data) + assert pr.pr.number > 0 + assert len(pr.comments) > 0 + + +def test_extraction_skips_automation_prs(pipeline_dirs: dict) -> None: + """No cache file should correspond to an automation KB bot PR.""" + cache_dir = pipeline_dirs["cache_dir"] + for pf in cache_dir.glob("pr-*.json"): + data = json.loads(pf.read_text(encoding="utf-8")) + title = data.get("pr", {}).get("title", "") + assert "update PR knowledge base" not in title.lower(), ( + f"{pf.name} contains a bot PR: {title}" + ) + + +# --------------------------------------------------------------------------- +# Classification assertions +# --------------------------------------------------------------------------- + + +def test_classification_produces_classified_files(pipeline_dirs: dict) -> None: + """At least one classified-pr-*.json exists with valid structure.""" + cache_dir = pipeline_dirs["cache_dir"] + classified_files = sorted(cache_dir.glob("classified-pr-*.json")) + assert len(classified_files) >= 1 + + from github_pr_kb.models import ClassifiedFile + + for cf in classified_files: + data = json.loads(cf.read_text(encoding="utf-8")) + classified = ClassifiedFile.model_validate(data) + assert classified.pr_number > 0 + for comment in classified.comments: + assert comment.category in VALID_CATEGORIES + assert 0.0 <= comment.confidence <= 1.0 + assert isinstance(comment.summary, str) + assert len(comment.summary) > 0 + + +def test_classification_index_exists(pipeline_dirs: dict) -> None: + """classification-index.json exists and maps body hashes to classifications.""" + cache_dir = pipeline_dirs["cache_dir"] + index_file = cache_dir / "classification-index.json" + assert index_file.exists() + index = json.loads(index_file.read_text(encoding="utf-8")) + assert len(index) > 0 + for key, entry in index.items(): + assert len(key) == 64, f"Expected SHA-256 hash, got: {key}" + assert entry.get("category") in VALID_CATEGORIES + + +# --------------------------------------------------------------------------- +# Generation assertions +# --------------------------------------------------------------------------- + + +def test_generation_creates_kb_articles(pipeline_dirs: dict) -> None: + """At least one markdown article exists under kb//.""" + kb_dir = pipeline_dirs["kb_dir"] + articles = list(kb_dir.glob("*/*.md")) + content_articles = [a for a in articles if a.name != "INDEX.md"] + assert len(content_articles) >= 1 + + +def test_generation_articles_have_valid_frontmatter(pipeline_dirs: dict) -> None: + """Every generated article has YAML frontmatter with required fields.""" + kb_dir = pipeline_dirs["kb_dir"] + frontmatter_re = re.compile(r"^---\n(.*?)\n---", re.DOTALL) + + for article_path in kb_dir.glob("*/*.md"): + if article_path.name == "INDEX.md": + continue + content = article_path.read_text(encoding="utf-8") + match = frontmatter_re.match(content) + assert match, f"{article_path} missing YAML frontmatter" + fm = match.group(1) + assert "category:" in fm, f"{article_path} frontmatter missing category" + assert "source_pr:" in fm, f"{article_path} frontmatter missing source_pr" + + +def test_generation_creates_index(pipeline_dirs: dict) -> None: + """INDEX.md exists at kb root.""" + kb_dir = pipeline_dirs["kb_dir"] + index = kb_dir / "INDEX.md" + assert index.exists() + text = index.read_text(encoding="utf-8") + assert "# Knowledge Base Index" in text or "# PR Knowledge Base" in text + + +def test_generation_creates_manifest(pipeline_dirs: dict) -> None: + """kb/.manifest.json exists and maps comment IDs to article paths.""" + kb_dir = pipeline_dirs["kb_dir"] + manifest_file = kb_dir / ".manifest.json" + assert manifest_file.exists() + manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + assert len(manifest) > 0 + for comment_id, rel_path in manifest.items(): + assert rel_path.endswith(".md") + assert (kb_dir / rel_path).exists(), f"Manifest entry {rel_path} not found on disk" + + +def test_generation_articles_under_valid_categories(pipeline_dirs: dict) -> None: + """Every article directory is a valid classification category.""" + kb_dir = pipeline_dirs["kb_dir"] + category_dirs = { + d.name for d in kb_dir.iterdir() if d.is_dir() and not d.name.startswith(".") + } + assert category_dirs, "No category directories found" + unexpected = category_dirs - VALID_CATEGORIES + assert not unexpected, f"Unexpected category directories: {unexpected}" From 5486981487fbaad58bcbd0c3935343a1c4c9f3cc Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 11:46:23 +0300 Subject: [PATCH 26/28] ci: allow manual workflow_dispatch trigger for e2e-pipeline job Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 748a142..d1e5ec6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ on: merge_group: branches: - main + workflow_dispatch: permissions: contents: read @@ -62,7 +63,7 @@ jobs: e2e-pipeline: name: e2e-pipeline - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: From dd7b5799c91b033c4024837061c1ab7a4033077a Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 11:53:52 +0300 Subject: [PATCH 27/28] fix: correct E2E test assertions for ClassifiedFile.pr and pr_url frontmatter Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/test_e2e_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_e2e_pipeline.py b/tests/test_e2e_pipeline.py index b4c8f82..f9da679 100644 --- a/tests/test_e2e_pipeline.py +++ b/tests/test_e2e_pipeline.py @@ -125,7 +125,7 @@ def test_classification_produces_classified_files(pipeline_dirs: dict) -> None: for cf in classified_files: data = json.loads(cf.read_text(encoding="utf-8")) classified = ClassifiedFile.model_validate(data) - assert classified.pr_number > 0 + assert classified.pr.number > 0 for comment in classified.comments: assert comment.category in VALID_CATEGORIES assert 0.0 <= comment.confidence <= 1.0 @@ -171,7 +171,7 @@ def test_generation_articles_have_valid_frontmatter(pipeline_dirs: dict) -> None assert match, f"{article_path} missing YAML frontmatter" fm = match.group(1) assert "category:" in fm, f"{article_path} frontmatter missing category" - assert "source_pr:" in fm, f"{article_path} frontmatter missing source_pr" + assert "pr_url:" in fm, f"{article_path} frontmatter missing pr_url" def test_generation_creates_index(pipeline_dirs: dict) -> None: From f341ba149f3a0621ce7e07cff3b21ccf87650091 Mon Sep 17 00:00:00 2001 From: Gal Ziv Date: Sat, 11 Apr 2026 11:56:08 +0300 Subject: [PATCH 28/28] fix: use ClassifiedFile.classifications not .comments in E2E test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/test_e2e_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_e2e_pipeline.py b/tests/test_e2e_pipeline.py index f9da679..1b49555 100644 --- a/tests/test_e2e_pipeline.py +++ b/tests/test_e2e_pipeline.py @@ -126,7 +126,7 @@ def test_classification_produces_classified_files(pipeline_dirs: dict) -> None: data = json.loads(cf.read_text(encoding="utf-8")) classified = ClassifiedFile.model_validate(data) assert classified.pr.number > 0 - for comment in classified.comments: + for comment in classified.classifications: assert comment.category in VALID_CATEGORIES assert 0.0 <= comment.confidence <= 1.0 assert isinstance(comment.summary, str)