From cba87348196f60cd7ff6284f9d960d0d2bffdc3e Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Wed, 24 Jun 2026 15:30:46 -0400 Subject: [PATCH 1/3] Govern the solution server hint prompt as a versioned Jinja2 template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the live generate_hint_v3 prompt out of server.py into a versioned Jinja2 asset under src/kai_mcp_solution_server/prompts/, rendered via render_prompt(). The incident loop now lives in the template; the AST-diff computation stays in Python and is passed in as a variable. This brings the solution server's hint prompt under the same governance the editor-extensions repo adopted for its prompts (ISO 42001 A.5.2) — the surface that work explicitly left to be governed here. - prompts/: Jinja2 template + manifest (semver + checksum), render registry; ships as package data (verified present in the built wheel) - byte-exact parity tests prove the template reproduces the previous prompt string exactly across the incident / AST-diff matrix, plus a deterministic mock-model semantic-regression check - scripts/prompts_version.py + prompt-validation.yml validate Jinja syntax, declared variables, and manifest checksums on any prompts/** change - PROMPT_GOVERNANCE.md, a new CODEOWNERS, and an audit-evidence doc - remove the never-called generate_hint_v1/v2 (dead code) and the now-unused get_diff import Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Fabian von Feilitzsch --- .github/CODEOWNERS | 5 + .github/workflows/prompt-validation.yml | 48 +++++ kai_mcp_solution_server/PROMPT_GOVERNANCE.md | 78 +++++++ .../docs/ISO-42001-A5.2-evidence.md | 45 ++++ kai_mcp_solution_server/pyproject.toml | 1 + kai_mcp_solution_server/requirements.txt | 1 + .../scripts/prompts_version.py | 140 +++++++++++++ .../prompts/__init__.py | 42 ++++ .../prompts/manifest.yaml | 12 ++ .../templates/generate_hint_v3.md.jinja | 32 +++ .../src/kai_mcp_solution_server/server.py | 194 +----------------- .../tests/prompts/__init__.py | 0 .../tests/prompts/oracle.py | 49 +++++ .../tests/prompts/test_parity.py | 75 +++++++ .../tests/prompts/test_semantic_regression.py | 73 +++++++ kai_mcp_solution_server/uv.lock | 19 +- 16 files changed, 623 insertions(+), 191 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/prompt-validation.yml create mode 100644 kai_mcp_solution_server/PROMPT_GOVERNANCE.md create mode 100644 kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md create mode 100644 kai_mcp_solution_server/scripts/prompts_version.py create mode 100644 kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py create mode 100644 kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml create mode 100644 kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja create mode 100644 kai_mcp_solution_server/tests/prompts/__init__.py create mode 100644 kai_mcp_solution_server/tests/prompts/oracle.py create mode 100644 kai_mcp_solution_server/tests/prompts/test_parity.py create mode 100644 kai_mcp_solution_server/tests/prompts/test_semantic_regression.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..607e8e11c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Governed prompt templates (ISO 42001 A.5.2). Any change to a prompt asset +# requires sign-off from a designated Prompt Engineer / Product Architect. +# See kai_mcp_solution_server/PROMPT_GOVERNANCE.md. +/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/ @konveyor/kai-prompt-reviewers +/kai_mcp_solution_server/scripts/prompts_version.py @konveyor/kai-prompt-reviewers diff --git a/.github/workflows/prompt-validation.yml b/.github/workflows/prompt-validation.yml new file mode 100644 index 000000000..1d790ae51 --- /dev/null +++ b/.github/workflows/prompt-validation.yml @@ -0,0 +1,48 @@ +name: Prompt Template Validation (ISO 42001 A.5.2) + +# Governance gate for the versioned prompt template set in the solution server. +# Runs whenever a governed prompt asset, the manifest, the validation script, the +# render registry, or this workflow changes. Required-status + CODEOWNERS review +# for the prompts directory are documented in PROMPT_GOVERNANCE.md. +on: + pull_request: + paths: + - "kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/**" + - "kai_mcp_solution_server/scripts/prompts_version.py" + - "kai_mcp_solution_server/tests/prompts/**" + - ".github/workflows/prompt-validation.yml" + workflow_dispatch: + +jobs: + validate-prompts: + name: Validate prompt templates + runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: ./kai_mcp_solution_server + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + with: + version: "latest" + + - name: Install dependencies + run: uv sync + + # Syntactic verification + version governance: every template declared in + # manifest.yaml (and vice-versa), checksums match, Jinja parses, declared + # variables match the template's referenced variables, no leftover f-strings. + - name: Syntactic & manifest validation + run: uv run python scripts/prompts_version.py check + + # Byte-exact parity + deterministic (mock-model) semantic regression. + - name: Parity & semantic-regression tests + run: uv run python -m pytest tests/prompts -v diff --git a/kai_mcp_solution_server/PROMPT_GOVERNANCE.md b/kai_mcp_solution_server/PROMPT_GOVERNANCE.md new file mode 100644 index 000000000..dfc4a42b9 --- /dev/null +++ b/kai_mcp_solution_server/PROMPT_GOVERNANCE.md @@ -0,0 +1,78 @@ +# Prompt Template Governance + +This document defines the change-management lifecycle for the LLM **prompt templates** the Kai +solution server uses to generate migration hints. It satisfies ISO/IEC 42001 Control **A.5.2 +(Governance and lifecycle of AI systems)**: the hint prompt is the instruction set defining the +server's hint-generation behavior, so it is treated as governed source code — version-controlled, +peer-reviewed, and validated by CI before distribution. + +## Scope — what is governed + +All model-bound prompt templates live as individual Jinja2 assets under +[`src/kai_mcp_solution_server/prompts/templates/`](src/kai_mcp_solution_server/prompts/templates/) and +are enumerated in +[`src/kai_mcp_solution_server/prompts/manifest.yaml`](src/kai_mcp_solution_server/prompts/manifest.yaml). +Today this is the live hint prompt, `generate_hint_v3`. Prompts are rendered by +[Jinja2](https://jinja.palletsprojects.com/) via `render_prompt(prompt_id, **context)` from +`kai_mcp_solution_server.prompts`; loops and structure live in the template, not in `server.py`. The +prompt-set is semantically versioned by `version:` in the manifest, kept in lockstep with the package +version in `pyproject.toml`. + +The `ast_diff_str` value (computed from `associate_files` + `extract_ast_info`) is data marshalling +and stays in Python; the template receives it as a variable. + +## Roles + +- **Prompt Engineer / Product Architect** (`@konveyor/kai-prompt-reviewers`) — required approver for any + change under `prompts/`. Owns prompt wording, the threat-model review, and the regression baseline. +- **Solution-server maintainers** — co-reviewers for the code that calls `render_prompt`. + +The prompts directory is assigned to the reviewers in [`.github/CODEOWNERS`](../.github/CODEOWNERS). + +## Lifecycle of a prompt change + +1. **Branch & edit** the template asset(s). Do not reintroduce prompt strings inline in `server.py`. +2. **Refresh checksums.** Run `python scripts/prompts_version.py update`. Bump `version:` in the + manifest (and `pyproject.toml`) if the change ships in a new release. +3. **Update parity expectations.** The byte-exact oracle (`tests/prompts/oracle.py`) is the historical + baseline. For an _intentional_ wording change, update the oracle and call it out in the PR. +4. **Complete the prompt-injection threat-model checklist** (below) in the PR. +5. **Open a PR** with DCO sign-off (`git commit -s`). CODEOWNERS routes it to a Prompt Engineer; the + [`prompt-validation`](../.github/workflows/prompt-validation.yml) workflow runs automatically. +6. **Merge** only after required review + green CI. + +## CI gates + +The `prompt-validation` workflow (triggered on `prompts/**`) runs `scripts/prompts_version.py check` +and `pytest tests/prompts`: + +- **Syntactic verification** — every template parses as Jinja2; declared variables exactly match the + template's referenced variables; no leftover `{var}` f-string interpolation. +- **Manifest/version governance** — every asset is declared (and vice-versa); content checksums match. +- **Byte-exact parity** — the template reproduces the pre-extraction prompt string exactly across the + incident / AST-diff matrix. +- **Semantic regression (deterministic, mock model)** — renders the prompt against a baseline of + migration scenarios and asserts the required scaffolding (output-format contract, each incident's + fields, the `AST Diff:` anchor) survives. + +## Prompt-injection threat-model checklist + +Complete for every template change that adds or alters an interpolation point: + +- [ ] **Untrusted interpolation** — `incident.message`, `incident.code_snip`, and `ast_diff_str` are + derived from analyzed user code. Confirm they remain data-only and cannot be read by the model + as instructions overriding the output-format contract. +- [ ] **Output-contract integrity** — do the `SUMMARY:` / `HINT:` anchors remain unambiguous so a + malicious code snippet can't spoof the expected response shape? +- [ ] **Rendering** — output is non-escaped by design (these are plain-text prompts); confirm no new + context (JSON, code fence) where unescaped interpolation enables injection. +- [ ] **Regression** — the semantic-regression baseline still passes. + +## Branch protection (repo admin action — not in code) + +A repository admin must, for `main` and `release-*`: + +1. **Require the status check** `Validate prompt templates`. +2. **Require review from Code Owners** so `prompts/**` edits need `@konveyor/kai-prompt-reviewers`. +3. **Create the `@konveyor/kai-prompt-reviewers` team** with the designated Prompt Engineers / + Product Architects. diff --git a/kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md b/kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md new file mode 100644 index 000000000..1ee04b987 --- /dev/null +++ b/kai_mcp_solution_server/docs/ISO-42001-A5.2-evidence.md @@ -0,0 +1,45 @@ +# ISO/IEC 42001 — Control A.5.2 Evidence Ledger (Kai Solution Server) + +**Control:** A.5.2 — Governance and lifecycle of AI systems +**Subject:** Prompt template governance, versioning, and automated validation for the Kai solution +server's hint-generation prompt. +**Status:** Implemented — the prompt is decoupled, version-controlled, peer-review mandated, and +CI-validated. + +This is the counterpart to the editor-extensions evidence ledger, which named `konveyor/kai` as the +owner of the solution-server prompt surface. This ledger lists the governed paths. + +## 1. Structured versioning & decoupling + +| Evidence | Path | +| ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------- | +| Dedicated prompt template directory (individual file assets) | [`src/kai_mcp_solution_server/prompts/templates/`](../src/kai_mcp_solution_server/prompts/templates/) | +| Semantic version + per-template manifest with checksums | [`src/kai_mcp_solution_server/prompts/manifest.yaml`](../src/kai_mcp_solution_server/prompts/manifest.yaml) | +| Render registry (template decoupled from app logic) | [`src/kai_mcp_solution_server/prompts/__init__.py`](../src/kai_mcp_solution_server/prompts/__init__.py) | + +The live hint prompt (`generate_hint_v3`) is rendered via `render_prompt(...)`; `server.py` no longer +embeds the prompt string. The dead `generate_hint_v1`/`v2` were removed in the same change. + +## 2. Automated CI validation pipeline + +| Evidence | Path | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| Pipeline definition (triggered on `prompts/**`) | [`.github/workflows/prompt-validation.yml`](../../.github/workflows/prompt-validation.yml) | +| Syntactic verification + checksum/version governance | [`scripts/prompts_version.py`](../scripts/prompts_version.py) | +| Byte-exact parity + oracle baseline | [`tests/prompts/test_parity.py`](../tests/prompts/test_parity.py), [`tests/prompts/oracle.py`](../tests/prompts/oracle.py) | +| Semantic regression (deterministic, mock model, baseline dataset) | [`tests/prompts/test_semantic_regression.py`](../tests/prompts/test_semantic_regression.py) | + +## 3. Peer review & security policy + +| Evidence | Path | +| -------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| Governance process, roles, lifecycle | [`PROMPT_GOVERNANCE.md`](../PROMPT_GOVERNANCE.md) | +| Mandatory code-owner review for prompt assets | [`.github/CODEOWNERS`](../../.github/CODEOWNERS) | +| Prompt-injection threat-model checklist | [`PROMPT_GOVERNANCE.md`](../PROMPT_GOVERNANCE.md#prompt-injection-threat-model-checklist) | +| Branch-protection requirements (repo-admin action) | [`PROMPT_GOVERNANCE.md`](../PROMPT_GOVERNANCE.md#branch-protection-repo-admin-action--not-in-code) | + +## Outstanding (repo-admin / org action) + +- Create the `@konveyor/kai-prompt-reviewers` team (Prompt Engineers / Product Architects). +- Enable branch protection on `main` / `release-*` requiring the `Validate prompt templates` check and + Code-Owner review for the prompts directory. diff --git a/kai_mcp_solution_server/pyproject.toml b/kai_mcp_solution_server/pyproject.toml index e318c4a93..3e283a49e 100644 --- a/kai_mcp_solution_server/pyproject.toml +++ b/kai_mcp_solution_server/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "tree-sitter>=0.24.0", "tree-sitter-java>=0.23.5", "types-pyyaml>=6.0.12.20250516", + "jinja2>=3.1.6", ] # https://docs.astral.sh/uv/configuration/build-backend/#modules diff --git a/kai_mcp_solution_server/requirements.txt b/kai_mcp_solution_server/requirements.txt index 4335e65f3..3c892a065 100644 --- a/kai_mcp_solution_server/requirements.txt +++ b/kai_mcp_solution_server/requirements.txt @@ -72,6 +72,7 @@ jaraco-classes==3.4.0 jaraco-context==6.1.0 jaraco-functools==4.4.0 jeepney==0.9.0 +jinja2==3.1.6 jiter==0.12.0 jmespath==1.0.1 jsonpatch==1.33 diff --git a/kai_mcp_solution_server/scripts/prompts_version.py b/kai_mcp_solution_server/scripts/prompts_version.py new file mode 100644 index 000000000..c7000a134 --- /dev/null +++ b/kai_mcp_solution_server/scripts/prompts_version.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Prompt template governance checks (ISO 42001 A.5.2). + +Validates the versioned prompt template set under +``src/kai_mcp_solution_server/prompts/``: + + - every ``.jinja`` template is declared in ``manifest.yaml`` (and vice-versa) + - content checksums in the manifest match the files on disk (drift detection) + - every template parses as Jinja2 (syntactic verification) + - declared variables exactly match the template's referenced variables, and no + leftover ``{var}`` f-string interpolation escaped into a template asset + +Usage: + python scripts/prompts_version.py check # CI gate (non-zero exit on problems) + python scripts/prompts_version.py update # recompute + write checksums +""" + +from __future__ import annotations + +import hashlib +import re +import sys +from pathlib import Path + +import yaml +from jinja2 import Environment, meta + +ROOT = Path(__file__).resolve().parent.parent +PROMPTS_DIR = ROOT / "src" / "kai_mcp_solution_server" / "prompts" +TEMPLATES_DIR = PROMPTS_DIR / "templates" +MANIFEST_PATH = PROMPTS_DIR / "manifest.yaml" + +# Matches a leftover Python f-string placeholder like ``{incident.uri}`` while +# ignoring Jinja's own ``{{ ... }}`` / ``{% ... %}``. +_FSTRING_RE = re.compile(r"(? str: + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def load_manifest() -> dict: + return yaml.safe_load(MANIFEST_PATH.read_text()) + + +def check() -> int: + manifest = load_manifest() + templates = manifest.get("templates", []) + problems: list[str] = [] + env = Environment() # noqa: S701 # nosec B701 - parses templates, never renders + + on_disk = {p.relative_to(ROOT).as_posix() for p in TEMPLATES_DIR.rglob("*.jinja")} + declared = {t["path"] for t in templates} + for path in on_disk - declared: + problems.append(f"Template on disk is not declared in manifest.yaml: {path}") + for path in declared - on_disk: + problems.append( + f"Manifest declares a template that does not exist on disk: {path}" + ) + + for entry in templates: + abs_path = ROOT / entry["path"] + if not abs_path.exists(): + continue # missing-file already reported + src = abs_path.read_text() + + actual = sha256(src) + if entry.get("checksum") != actual: + problems.append( + f"Checksum drift for {entry['id']} ({entry['path']}). " + f"Manifest: {entry.get('checksum') or ''}, actual: {actual}. " + 'Run "python scripts/prompts_version.py update" and review the change.' + ) + + try: + ast = env.parse(src) + except Exception as err: # noqa: BLE001 - report any parse failure + problems.append( + f"Jinja2 parse error in {entry['id']} ({entry['path']}): {err}" + ) + continue + + if _FSTRING_RE.search(src): + problems.append( + f"Leftover {{...}} f-string interpolation found in {entry['id']} ({entry['path']})." + ) + + referenced = meta.find_undeclared_variables(ast) + declared_vars = set(entry.get("variables", [])) + for missing in declared_vars - referenced: + problems.append( + f'Declared variable "{missing}" is not referenced in {entry["id"]} ({entry["path"]}).' + ) + for undeclared in referenced - declared_vars: + problems.append( + f'Variable "{undeclared}" is used in {entry["id"]} ({entry["path"]}) ' + "but not declared in manifest.yaml." + ) + + if problems: + print("Prompt template validation FAILED:\n", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + print(f"\n{len(problems)} problem(s) found.", file=sys.stderr) + return 1 + + print( + f"Prompt template validation passed. version={manifest.get('version')}, " + f"templates={len(templates)}." + ) + return 0 + + +def update() -> int: + manifest = load_manifest() + for entry in manifest.get("templates", []): + entry["checksum"] = sha256((ROOT / entry["path"]).read_text()) + header = ( + "# Prompt template manifest — governed under ISO 42001 A.5.2.\n" + "# Checksums are maintained by scripts/prompts_version.py; do not edit by hand.\n" + ) + MANIFEST_PATH.write_text( + header + yaml.safe_dump(manifest, sort_keys=False, width=1000) + ) + print(f"Updated checksums for {len(manifest.get('templates', []))} template(s).") + return 0 + + +def main() -> int: + cmd = sys.argv[1] if len(sys.argv) > 1 else "" + if cmd == "check": + return check() + if cmd == "update": + return update() + print("Usage: python scripts/prompts_version.py ", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py new file mode 100644 index 000000000..e728cd835 --- /dev/null +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/__init__.py @@ -0,0 +1,42 @@ +"""Governed prompt templates for the solution server (ISO 42001 A.5.2). + +Prompts live as individual Jinja2 assets under ``templates/`` and are rendered +through :func:`render_prompt`. They are versioned and validated independently of +the application code; see ``manifest.yaml`` and ``PROMPT_GOVERNANCE.md``. +""" + +from __future__ import annotations + +from functools import lru_cache + +from jinja2 import Environment, PackageLoader, StrictUndefined, Template + +# One id per governed prompt -> its template filename. +_TEMPLATES: dict[str, str] = { + "generate_hint_v3": "generate_hint_v3.md.jinja", +} + +# autoescape is intentionally off: these are plain-text LLM prompts, not HTML, so +# HTML-escaping would corrupt code snippets in the rendered prompt (B701 N/A). +# keep_trailing_newline preserves the template's final newline verbatim; +# StrictUndefined makes a missing context variable fail loudly. +_env = Environment( # nosec B701 + loader=PackageLoader("kai_mcp_solution_server", "prompts/templates"), + autoescape=False, + keep_trailing_newline=True, + undefined=StrictUndefined, +) + + +@lru_cache(maxsize=None) +def _template(filename: str) -> Template: + return _env.get_template(filename) + + +def render_prompt(prompt_id: str, /, **context: object) -> str: + """Render a governed prompt template to its final string.""" + try: + filename = _TEMPLATES[prompt_id] + except KeyError: + raise KeyError(f"Unknown prompt id: {prompt_id!r}") from None + return _template(filename).render(**context) diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml new file mode 100644 index 000000000..975a6ecb8 --- /dev/null +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/manifest.yaml @@ -0,0 +1,12 @@ +# Prompt template manifest — governed under ISO 42001 A.5.2. +# Checksums are maintained by scripts/prompts_version.py; do not edit by hand. +version: 0.1.0 +engine: jinja2 +templates: + - id: generate_hint_v3 + path: src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja + governed: true + variables: + - incidents + - ast_diff_str + checksum: sha256:bd4dcf69a3525a0c914626551c521edaf326c4cb63e0fa7066d5c908a00a00aa diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja new file mode 100644 index 000000000..fbc0fa945 --- /dev/null +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/prompts/templates/generate_hint_v3.md.jinja @@ -0,0 +1,32 @@ +The following incidents had this accepted solution. Use the AST diffs below as a guiding pattern for migration. + +Generate a hint for the user so that they can migrate the code. + +IMPORTANT: Follow this EXACT output format: +--- +SUMMARY: +[concise summary of necessary changes] + +HINT: +[numbered steps with generic, reusable code examples] +--- + +Guidelines for high-quality response: +1. Keep SUMMARY concise and focused +2. Use numbered steps (1, 2, 3) in HINT section +3. Provide generic before/after code examples that can be reused and mark them as examples (e.g. 'Example 1: Before: ... After: ...') +4. Write in direct, actionable tone +Incidents: +{% for incident in incidents -%} +Incident {{ loop.index }}: + URI: {{ incident.uri }} + Message: {{ incident.message }} + Code Snippet: {{ incident.code_snip }} + Line Number: {{ incident.line_number }} + Variables: {{ incident.variables }} + Violation: {{ incident.violation.ruleset_name }} - {{ incident.violation.violation_name }} + +{% endfor -%} +AST Diff: +{{ ast_diff_str }} + diff --git a/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py b/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py index 1f45d4dea..9d48987fd 100644 --- a/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py +++ b/kai_mcp_solution_server/src/kai_mcp_solution_server/server.py @@ -36,15 +36,15 @@ SolutionStatus, ViolationID, associate_files, - get_diff, ) +from kai_mcp_solution_server.prompts import render_prompt P = ParamSpec("P") T = TypeVar("T") def with_db_recovery( - func: Callable[..., Coroutine[Any, Any, T]] + func: Callable[..., Coroutine[Any, Any, T]], ) -> Callable[..., Coroutine[Any, Any, T]]: """Decorator to execute database operations with automatic recovery on connection errors. @@ -559,158 +559,6 @@ async def tool_create_solution( ) -@with_db_recovery -async def generate_hint_v1( - kai_ctx: KaiSolutionServerContext, - client_id: str, -) -> None: - if kai_ctx.session_maker is None: - raise RuntimeError("Session maker not initialized") - async with kai_ctx.session_maker.begin() as session: - solutions_stmt = select(DBSolution).where( - DBSolution.client_id == client_id, - or_( - DBSolution.solution_status == SolutionStatus.ACCEPTED, - DBSolution.solution_status == SolutionStatus.MODIFIED, - ), - ) - solutions = (await session.execute(solutions_stmt)).scalars().all() - if len(solutions) == 0: - log( - f"No accepted or modified solutions found for client {client_id}. No hint generated." - ) - return - - for solution in solutions: - prompt = ( - "The following incidents had this accepted solution. " - "Generate a hint for the user so that they can perform the same solution:\n" - ) - - for i, incident in enumerate(solution.incidents): - prompt += ( - f"Incident {i + 1}:\n" - f" URI: {incident.uri}\n" - f" Message: {incident.message}\n" - f" Code Snippet: {incident.code_snip}\n" - f" Line Number: {incident.line_number}\n" - f" Variables: {incident.variables}\n" - f" Violation: {incident.violation.ruleset_name} - " - f" {incident.violation.violation_name}\n\n" - ) - - diff = get_diff( - [SolutionFile(uri=f.uri, content=f.content) for f in solution.before], - [SolutionFile(uri=f.uri, content=f.content) for f in solution.after], - ) - - prompt += "Solution:\n" f"{diff}\n\n" - - log(f"Generating hint for client {client_id} with prompt:\n{prompt}") - - if kai_ctx.model is None: - raise RuntimeError("Model not initialized") - response = await kai_ctx.model.ainvoke(prompt) - - log(f"Generated hint: {response.content}") - - hint = DBHint( - text=str(response.content), - violations=set( - incident.violation - for incident in solution.incidents - if incident.violation is not None - ), - solutions=set([solution]), - ) - session.add(hint) - - await session.flush() - - -@with_db_recovery -async def generate_hint_v2( - kai_ctx: KaiSolutionServerContext, - client_id: str, -) -> None: - # print(f"Generating hint for client {client_id}", file=sys.stderr) - if kai_ctx.session_maker is None: - raise RuntimeError("Session maker not initialized") - async with kai_ctx.session_maker.begin() as session: - solutions_stmt = select(DBSolution).where( - DBSolution.client_id == client_id, - or_( - DBSolution.solution_status == SolutionStatus.ACCEPTED, - DBSolution.solution_status == SolutionStatus.MODIFIED, - ), - ) - solutions = (await session.execute(solutions_stmt)).scalars().all() - if len(solutions) == 0: - print( - f"No accepted solutions found for client {client_id}. No hint generated.", - file=sys.stderr, - ) - return - - for solution in solutions: - prompt = ( - "The following incidents had this accepted solution. " - "Generate a hint for the user so that they can create the same solution:\n" - ) - - for i, incident in enumerate(solution.incidents): - prompt += ( - f"Incident {i + 1}:\n" - f" URI: {incident.uri}\n" - f" Message: {incident.message}\n" - f" Code Snippet: {incident.code_snip}\n" - f" Line Number: {incident.line_number}\n" - f" Variables: {incident.variables}\n" - f" Violation: {incident.violation.ruleset_name} - " - f" {incident.violation.violation_name}\n\n" - ) - - diff = associate_files( - [SolutionFile(uri=f.uri, content=f.content) for f in solution.before], - [SolutionFile(uri=f.uri, content=f.content) for f in solution.after], - ) - - ast_diffs: list[dict[str, Any]] = [] - for (_before_uri, _after_uri), (before_file, after_file) in diff.items(): - if before_file.content == after_file.content: - continue - - ast_diffs.append( - extract_ast_info(before_file.content, language=Language.JAVA).diff( - extract_ast_info(after_file.content, language=Language.JAVA) - ) - ) - - ast_diff_str = "\n\n".join(str(a) for a in ast_diffs if a is not None) - prompt += f"AST Diff:\n{ast_diff_str}\n\n" - - # print(f"Generating hint for client {client_id} with prompt:\n{prompt}", file=sys.stderr) - - if kai_ctx.model is None: - raise RuntimeError("Model not initialized") - response = await kai_ctx.model.ainvoke(prompt) - - # print(f"Generated hint: {response.content}", file=sys.stderr) - - hint = DBHint( - text=str(response.content), - violations=set( - incident.violation - for incident in solution.incidents - if incident.violation is not None - ), - solutions=set([solution]), - ) - session.add(hint) - - await session.flush() - - @with_db_recovery async def generate_hint_v3( kai_ctx: KaiSolutionServerContext, @@ -738,37 +586,6 @@ async def generate_hint_v3( return for solution in solutions: - prompt = ( - "The following incidents had this accepted solution. " - "Use the AST diffs below as a guiding pattern for migration.\n\n" - "Generate a hint for the user so that they can migrate the code.\n\n" - "IMPORTANT: Follow this EXACT output format:\n" - "---\n" - "SUMMARY:\n" - "[concise summary of necessary changes]\n\n" - "HINT:\n" - "[numbered steps with generic, reusable code examples]\n" - "---\n\n" - "Guidelines for high-quality response:\n" - "1. Keep SUMMARY concise and focused\n" - "2. Use numbered steps (1, 2, 3) in HINT section\n" - "3. Provide generic before/after code examples that can be reused and mark them as examples (e.g. 'Example 1: Before: ... After: ...')\n" - "4. Write in direct, actionable tone\n" - "Incidents:\n" - ) - - for i, incident in enumerate(solution.incidents): - prompt += ( - f"Incident {i + 1}:\n" - f" URI: {incident.uri}\n" - f" Message: {incident.message}\n" - f" Code Snippet: {incident.code_snip}\n" - f" Line Number: {incident.line_number}\n" - f" Variables: {incident.variables}\n" - f" Violation: {incident.violation.ruleset_name} - " - f" {incident.violation.violation_name}\n\n" - ) - diff = associate_files( [SolutionFile(uri=f.uri, content=f.content) for f in solution.before], [SolutionFile(uri=f.uri, content=f.content) for f in solution.after], @@ -786,7 +603,12 @@ async def generate_hint_v3( ) ast_diff_str = "\n\n".join(str(a) for a in ast_diffs if a is not None) - prompt += f"AST Diff:\n{ast_diff_str}\n\n" + + prompt = render_prompt( + "generate_hint_v3", + incidents=solution.incidents, + ast_diff_str=ast_diff_str, + ) if kai_ctx.model is None: raise RuntimeError("Model not initialized") diff --git a/kai_mcp_solution_server/tests/prompts/__init__.py b/kai_mcp_solution_server/tests/prompts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kai_mcp_solution_server/tests/prompts/oracle.py b/kai_mcp_solution_server/tests/prompts/oracle.py new file mode 100644 index 000000000..869623e15 --- /dev/null +++ b/kai_mcp_solution_server/tests/prompts/oracle.py @@ -0,0 +1,49 @@ +"""Parity oracle — the byte-exact baseline for the prompt-template migration. + +This is a VERBATIM copy of the ``generate_hint_v3`` prompt-building logic as it +existed inline in ``server.py`` immediately before extraction. The parity suite +asserts that ``render_prompt("generate_hint_v3", ...)`` reproduces this string +byte-for-byte. Do not "clean up" anything here — quirks (the ``- `` triple +space before the violation name, the trailing blank line) are intentional and +are exactly what the template must reproduce. +""" + +from __future__ import annotations + +from typing import Any + + +def oracle_generate_hint_v3(incidents: list[Any], ast_diff_str: str) -> str: + prompt = ( + "The following incidents had this accepted solution. " + "Use the AST diffs below as a guiding pattern for migration.\n\n" + "Generate a hint for the user so that they can migrate the code.\n\n" + "IMPORTANT: Follow this EXACT output format:\n" + "---\n" + "SUMMARY:\n" + "[concise summary of necessary changes]\n\n" + "HINT:\n" + "[numbered steps with generic, reusable code examples]\n" + "---\n\n" + "Guidelines for high-quality response:\n" + "1. Keep SUMMARY concise and focused\n" + "2. Use numbered steps (1, 2, 3) in HINT section\n" + "3. Provide generic before/after code examples that can be reused and mark them as examples (e.g. 'Example 1: Before: ... After: ...')\n" + "4. Write in direct, actionable tone\n" + "Incidents:\n" + ) + + for i, incident in enumerate(incidents): + prompt += ( + f"Incident {i + 1}:\n" + f" URI: {incident.uri}\n" + f" Message: {incident.message}\n" + f" Code Snippet: {incident.code_snip}\n" + f" Line Number: {incident.line_number}\n" + f" Variables: {incident.variables}\n" + f" Violation: {incident.violation.ruleset_name} - " + f" {incident.violation.violation_name}\n\n" + ) + + prompt += f"AST Diff:\n{ast_diff_str}\n\n" + return prompt diff --git a/kai_mcp_solution_server/tests/prompts/test_parity.py b/kai_mcp_solution_server/tests/prompts/test_parity.py new file mode 100644 index 000000000..02c40d851 --- /dev/null +++ b/kai_mcp_solution_server/tests/prompts/test_parity.py @@ -0,0 +1,75 @@ +"""Byte-exact parity: the Jinja template must reproduce the original prompt +string exactly, across the full incident/ast-diff matrix.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from kai_mcp_solution_server.prompts import render_prompt + +from .oracle import oracle_generate_hint_v3 + + +def _incident( + uri: str = "file:///src/Foo.java", + message: str = "javax.* must become jakarta.*", + code_snip: str = "import javax.persistence.Entity;", + line_number: int = 12, + variables: object = None, + ruleset_name: str = "javax-to-jakarta", + violation_name: str = "javax-import", +) -> SimpleNamespace: + if variables is None: + variables = {"kind": "import"} + return SimpleNamespace( + uri=uri, + message=message, + code_snip=code_snip, + line_number=line_number, + variables=variables, + violation=SimpleNamespace( + ruleset_name=ruleset_name, violation_name=violation_name + ), + ) + + +_AST_DIFF = ( + "class Foo {\n- @javax.persistence.Entity\n+ @jakarta.persistence.Entity\n}" +) + +_CASES = { + "0 incidents, empty ast": ([], ""), + "1 incident, empty ast": ([_incident()], ""), + "1 incident, populated ast": ([_incident()], _AST_DIFF), + "N incidents, populated ast": ( + [ + _incident(), + _incident( + uri="file:///src/Bar.java", + message="multi\nline\nmessage", + code_snip="line1\nline2", + line_number=99, + variables={"a": 1, "b": ["x", "y"]}, + violation_name="ejb-usage", + ), + _incident(message="special <>&\"' chars", variables=None), + ], + _AST_DIFF, + ), +} + + +@pytest.mark.parametrize("name", list(_CASES)) +def test_v3_renders_byte_identical_to_oracle(name: str) -> None: + incidents, ast_diff_str = _CASES[name] + rendered = render_prompt( + "generate_hint_v3", incidents=incidents, ast_diff_str=ast_diff_str + ) + assert rendered == oracle_generate_hint_v3(incidents, ast_diff_str) + + +def test_unknown_prompt_id_raises() -> None: + with pytest.raises(KeyError): + render_prompt("does-not-exist") diff --git a/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py b/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py new file mode 100644 index 000000000..bf0c9c199 --- /dev/null +++ b/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py @@ -0,0 +1,73 @@ +"""Deterministic (mock-model) semantic regression. + +Renders the hint prompt against a baseline of migration scenarios and asserts the +scaffolding a model needs to produce a usable hint survives any wording change: +the output-format contract, every incident's identifying fields, and the AST diff. +No live model is involved. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from kai_mcp_solution_server.prompts import render_prompt + +_OUTPUT_CONTRACT = [ + "IMPORTANT: Follow this EXACT output format:", + "SUMMARY:", + "HINT:", + "Incidents:", + "AST Diff:", +] + + +def _incident(uri: str, message: str, violation_name: str) -> SimpleNamespace: + return SimpleNamespace( + uri=uri, + message=message, + code_snip="import javax.persistence.Entity;", + line_number=1, + variables={"kind": "import"}, + violation=SimpleNamespace( + ruleset_name="javax-to-jakarta", violation_name=violation_name + ), + ) + + +_BASELINE = { + "javaee->quarkus": ( + [ + _incident( + "file:///Foo.java", + "Replace javax.persistence with jakarta.persistence", + "javax-import", + ) + ], + "class Foo {\n- @javax.persistence.Entity\n+ @jakarta.persistence.Entity\n}", + ), + "multi-incident": ( + [ + _incident("file:///A.java", "Remove EJB usage", "ejb-usage"), + _incident("file:///B.java", "Migrate persistence.xml", "persistence-xml"), + ], + "class A {}\n\nclass B {}", + ), +} + + +@pytest.mark.parametrize("name", list(_BASELINE)) +def test_v3_preserves_migration_scaffolding(name: str) -> None: + incidents, ast_diff_str = _BASELINE[name] + rendered = render_prompt( + "generate_hint_v3", incidents=incidents, ast_diff_str=ast_diff_str + ) + + for anchor in _OUTPUT_CONTRACT: + assert anchor in rendered, f"missing output-contract anchor: {anchor!r}" + for incident in incidents: + assert incident.uri in rendered + assert incident.message in rendered + assert incident.violation.violation_name in rendered + assert ast_diff_str in rendered diff --git a/kai_mcp_solution_server/uv.lock b/kai_mcp_solution_server/uv.lock index f11114557..ff8e0388e 100644 --- a/kai_mcp_solution_server/uv.lock +++ b/kai_mcp_solution_server/uv.lock @@ -1356,7 +1356,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/cb/48e964c452ca2b92175a9b2dca037a553036cb053ba69e284650ce755f13/greenlet-3.3.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e29f3018580e8412d6aaf5641bb7745d38c85228dacf51a73bd4e26ddf2a6a8e", size = 274908, upload-time = "2025-12-04T14:23:26.435Z" }, { url = "https://files.pythonhosted.org/packages/28/da/38d7bff4d0277b594ec557f479d65272a893f1f2a716cad91efeb8680953/greenlet-3.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a687205fb22794e838f947e2194c0566d3812966b41c78709554aa883183fb62", size = 577113, upload-time = "2025-12-04T14:50:05.493Z" }, { url = "https://files.pythonhosted.org/packages/3c/f2/89c5eb0faddc3ff014f1c04467d67dee0d1d334ab81fadbf3744847f8a8a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4243050a88ba61842186cb9e63c7dfa677ec146160b0efd73b855a3d9c7fcf32", size = 590338, upload-time = "2025-12-04T14:57:41.136Z" }, - { url = "https://files.pythonhosted.org/packages/80/d7/db0a5085035d05134f8c089643da2b44cc9b80647c39e93129c5ef170d8f/greenlet-3.3.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:670d0f94cd302d81796e37299bcd04b95d62403883b24225c6b5271466612f45", size = 601098, upload-time = "2025-12-04T15:07:11.898Z" }, { url = "https://files.pythonhosted.org/packages/dc/a6/e959a127b630a58e23529972dbc868c107f9d583b5a9f878fb858c46bc1a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb3a8ec3db4a3b0eb8a3c25436c2d49e3505821802074969db017b87bc6a948", size = 590206, upload-time = "2025-12-04T14:26:01.254Z" }, { url = "https://files.pythonhosted.org/packages/48/60/29035719feb91798693023608447283b266b12efc576ed013dd9442364bb/greenlet-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2de5a0b09eab81fc6a382791b995b1ccf2b172a9fec934747a7a23d2ff291794", size = 1550668, upload-time = "2025-12-04T15:04:22.439Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/783a23754b691bfa86bd72c3033aa107490deac9b2ef190837b860996c9f/greenlet-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4449a736606bd30f27f8e1ff4678ee193bc47f6ca810d705981cfffd6ce0d8c5", size = 1615483, upload-time = "2025-12-04T14:27:28.083Z" }, @@ -1364,7 +1363,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/0a/a3871375c7b9727edaeeea994bfff7c63ff7804c9829c19309ba2e058807/greenlet-3.3.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:b01548f6e0b9e9784a2c99c5651e5dc89ffcbe870bc5fb2e5ef864e9cc6b5dcb", size = 276379, upload-time = "2025-12-04T14:23:30.498Z" }, { url = "https://files.pythonhosted.org/packages/43/ab/7ebfe34dce8b87be0d11dae91acbf76f7b8246bf9d6b319c741f99fa59c6/greenlet-3.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349345b770dc88f81506c6861d22a6ccd422207829d2c854ae2af8025af303e3", size = 597294, upload-time = "2025-12-04T14:50:06.847Z" }, { url = "https://files.pythonhosted.org/packages/a4/39/f1c8da50024feecd0793dbd5e08f526809b8ab5609224a2da40aad3a7641/greenlet-3.3.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e8e18ed6995e9e2c0b4ed264d2cf89260ab3ac7e13555b8032b25a74c6d18655", size = 607742, upload-time = "2025-12-04T14:57:42.349Z" }, - { url = "https://files.pythonhosted.org/packages/77/cb/43692bcd5f7a0da6ec0ec6d58ee7cddb606d055ce94a62ac9b1aa481e969/greenlet-3.3.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c024b1e5696626890038e34f76140ed1daf858e37496d33f2af57f06189e70d7", size = 622297, upload-time = "2025-12-04T15:07:13.552Z" }, { url = "https://files.pythonhosted.org/packages/75/b0/6bde0b1011a60782108c01de5913c588cf51a839174538d266de15e4bf4d/greenlet-3.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:047ab3df20ede6a57c35c14bf5200fcf04039d50f908270d3f9a7a82064f543b", size = 609885, upload-time = "2025-12-04T14:26:02.368Z" }, { url = "https://files.pythonhosted.org/packages/49/0e/49b46ac39f931f59f987b7cd9f34bfec8ef81d2a1e6e00682f55be5de9f4/greenlet-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d9ad37fc657b1102ec880e637cccf20191581f75c64087a549e66c57e1ceb53", size = 1567424, upload-time = "2025-12-04T15:04:23.757Z" }, { url = "https://files.pythonhosted.org/packages/05/f5/49a9ac2dff7f10091935def9165c90236d8f175afb27cbed38fb1d61ab6b/greenlet-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83cd0e36932e0e7f36a64b732a6f60c2fc2df28c351bae79fbaf4f8092fe7614", size = 1636017, upload-time = "2025-12-04T14:27:29.688Z" }, @@ -1372,7 +1370,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/2f/28592176381b9ab2cafa12829ba7b472d177f3acc35d8fbcf3673d966fff/greenlet-3.3.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a1e41a81c7e2825822f4e068c48cb2196002362619e2d70b148f20a831c00739", size = 275140, upload-time = "2025-12-04T14:23:01.282Z" }, { url = "https://files.pythonhosted.org/packages/2c/80/fbe937bf81e9fca98c981fe499e59a3f45df2a04da0baa5c2be0dca0d329/greenlet-3.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f515a47d02da4d30caaa85b69474cec77b7929b2e936ff7fb853d42f4bf8808", size = 599219, upload-time = "2025-12-04T14:50:08.309Z" }, { url = "https://files.pythonhosted.org/packages/c2/ff/7c985128f0514271b8268476af89aee6866df5eec04ac17dcfbc676213df/greenlet-3.3.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d2d9fd66bfadf230b385fdc90426fcd6eb64db54b40c495b72ac0feb5766c54", size = 610211, upload-time = "2025-12-04T14:57:43.968Z" }, - { url = "https://files.pythonhosted.org/packages/79/07/c47a82d881319ec18a4510bb30463ed6891f2ad2c1901ed5ec23d3de351f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30a6e28487a790417d036088b3bcb3f3ac7d8babaa7d0139edbaddebf3af9492", size = 624311, upload-time = "2025-12-04T15:07:14.697Z" }, { url = "https://files.pythonhosted.org/packages/fd/8e/424b8c6e78bd9837d14ff7df01a9829fc883ba2ab4ea787d4f848435f23f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:087ea5e004437321508a8d6f20efc4cfec5e3c30118e1417ea96ed1d93950527", size = 612833, upload-time = "2025-12-04T14:26:03.669Z" }, { url = "https://files.pythonhosted.org/packages/b5/ba/56699ff9b7c76ca12f1cdc27a886d0f81f2189c3455ff9f65246780f713d/greenlet-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab97cf74045343f6c60a39913fa59710e4bd26a536ce7ab2397adf8b27e67c39", size = 1567256, upload-time = "2025-12-04T15:04:25.276Z" }, { url = "https://files.pythonhosted.org/packages/1e/37/f31136132967982d698c71a281a8901daf1a8fbab935dce7c0cf15f942cc/greenlet-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5375d2e23184629112ca1ea89a53389dddbffcf417dad40125713d88eb5f96e8", size = 1636483, upload-time = "2025-12-04T14:27:30.804Z" }, @@ -1380,7 +1377,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/7c/f0a6d0ede2c7bf092d00bc83ad5bafb7e6ec9b4aab2fbdfa6f134dc73327/greenlet-3.3.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:60c2ef0f578afb3c8d92ea07ad327f9a062547137afe91f38408f08aacab667f", size = 275671, upload-time = "2025-12-04T14:23:05.267Z" }, { url = "https://files.pythonhosted.org/packages/44/06/dac639ae1a50f5969d82d2e3dd9767d30d6dbdbab0e1a54010c8fe90263c/greenlet-3.3.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a5d554d0712ba1de0a6c94c640f7aeba3f85b3a6e1f2899c11c2c0428da9365", size = 646360, upload-time = "2025-12-04T14:50:10.026Z" }, { url = "https://files.pythonhosted.org/packages/e0/94/0fb76fe6c5369fba9bf98529ada6f4c3a1adf19e406a47332245ef0eb357/greenlet-3.3.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3a898b1e9c5f7307ebbde4102908e6cbfcb9ea16284a3abe15cab996bee8b9b3", size = 658160, upload-time = "2025-12-04T14:57:45.41Z" }, - { url = "https://files.pythonhosted.org/packages/93/79/d2c70cae6e823fac36c3bbc9077962105052b7ef81db2f01ec3b9bf17e2b/greenlet-3.3.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dcd2bdbd444ff340e8d6bdf54d2f206ccddbb3ccfdcd3c25bf4afaa7b8f0cf45", size = 671388, upload-time = "2025-12-04T15:07:15.789Z" }, { url = "https://files.pythonhosted.org/packages/b8/14/bab308fc2c1b5228c3224ec2bf928ce2e4d21d8046c161e44a2012b5203e/greenlet-3.3.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5773edda4dc00e173820722711d043799d3adb4f01731f40619e07ea2750b955", size = 660166, upload-time = "2025-12-04T14:26:05.099Z" }, { url = "https://files.pythonhosted.org/packages/4b/d2/91465d39164eaa0085177f61983d80ffe746c5a1860f009811d498e7259c/greenlet-3.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac0549373982b36d5fd5d30beb8a7a33ee541ff98d2b502714a09f1169f31b55", size = 1615193, upload-time = "2025-12-04T15:04:27.041Z" }, { url = "https://files.pythonhosted.org/packages/42/1b/83d110a37044b92423084d52d5d5a3b3a73cafb51b547e6d7366ff62eff1/greenlet-3.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d198d2d977460358c3b3a4dc844f875d1adb33817f0613f663a656f463764ccc", size = 1683653, upload-time = "2025-12-04T14:27:32.366Z" }, @@ -1388,7 +1384,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/66/bd6317bc5932accf351fc19f177ffba53712a202f9df10587da8df257c7e/greenlet-3.3.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d6ed6f85fae6cdfdb9ce04c9bf7a08d666cfcfb914e7d006f44f840b46741931", size = 282638, upload-time = "2025-12-04T14:25:20.941Z" }, { url = "https://files.pythonhosted.org/packages/30/cf/cc81cb030b40e738d6e69502ccbd0dd1bced0588e958f9e757945de24404/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9125050fcf24554e69c4cacb086b87b3b55dc395a8b3ebe6487b045b2614388", size = 651145, upload-time = "2025-12-04T14:50:11.039Z" }, { url = "https://files.pythonhosted.org/packages/9c/ea/1020037b5ecfe95ca7df8d8549959baceb8186031da83d5ecceff8b08cd2/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87e63ccfa13c0a0f6234ed0add552af24cc67dd886731f2261e46e241608bee3", size = 654236, upload-time = "2025-12-04T14:57:47.007Z" }, - { url = "https://files.pythonhosted.org/packages/69/cc/1e4bae2e45ca2fa55299f4e85854606a78ecc37fead20d69322f96000504/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2662433acbca297c9153a4023fe2161c8dcfdcc91f10433171cf7e7d94ba2221", size = 662506, upload-time = "2025-12-04T15:07:16.906Z" }, { url = "https://files.pythonhosted.org/packages/57/b9/f8025d71a6085c441a7eaff0fd928bbb275a6633773667023d19179fe815/greenlet-3.3.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c6e9b9c1527a78520357de498b0e709fb9e2f49c3a513afd5a249007261911b", size = 653783, upload-time = "2025-12-04T14:26:06.225Z" }, { url = "https://files.pythonhosted.org/packages/f6/c7/876a8c7a7485d5d6b5c6821201d542ef28be645aa024cfe1145b35c120c1/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:286d093f95ec98fdd92fcb955003b8a3d054b4e2cab3e2707a5039e7b50520fd", size = 1614857, upload-time = "2025-12-04T15:04:28.484Z" }, { url = "https://files.pythonhosted.org/packages/4f/dc/041be1dff9f23dac5f48a43323cd0789cb798342011c19a248d9c9335536/greenlet-3.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c10513330af5b8ae16f023e8ddbfb486ab355d04467c4679c5cfe4659975dd9", size = 1676034, upload-time = "2025-12-04T14:27:33.531Z" }, @@ -1615,6 +1610,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jiter" version = "0.12.0" @@ -1781,6 +1788,7 @@ dependencies = [ { name = "alembic" }, { name = "asyncpg" }, { name = "fastmcp" }, + { name = "jinja2" }, { name = "langchain" }, { name = "langchain-aws" }, { name = "langchain-azure-ai" }, @@ -1823,6 +1831,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.15.2" }, { name = "asyncpg", specifier = ">=0.30.0" }, { name = "fastmcp", specifier = ">=2.8.0" }, + { name = "jinja2", specifier = ">=3.1.6" }, { name = "langchain", specifier = ">=1.2.4" }, { name = "langchain-aws", specifier = ">=1.2.1" }, { name = "langchain-azure-ai", specifier = ">=1.0.0" }, From e007b11fcd2e2683db4b8a0edf9436a2c2a87290 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Wed, 24 Jun 2026 16:48:49 -0400 Subject: [PATCH 2/3] Refresh Trunk tooling so Markdown/TOML linting works in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trunk Check fails on any PR touching Markdown or TOML because two tools can't run in CI: - markdownlint@0.48.0 crashes at startup under the bundled node 18.12.1 ("SyntaxError: Invalid regular expression flags" — string-width uses the unicode-sets `v` regex flag, which needs node 20+). - taplo@0.10.0's Linux binary fails to install ("Binary not found") with the old plugins definitions. Bump the node runtime to 22.16.0, the trunk-io/plugins ref to v1.10.2 (refreshes tool download definitions; pinned tool versions are unchanged), and markdownlint to 0.49.0. `trunk check` reports no new issues across the changed files. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Fabian von Feilitzsch --- .trunk/trunk.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 26b9ed115..6f9f00abe 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -7,13 +7,13 @@ cli: plugins: sources: - id: trunk - ref: v1.6.3 + ref: v1.10.2 uri: https://github.com/trunk-io/plugins # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: enabled: - go@1.25.0 - - node@18.12.1 + - node@22.16.0 - python@3.10.8 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) lint: @@ -36,7 +36,7 @@ lint: - checkov@3.2.521 - git-diff-check - isort@8.0.1 - - markdownlint@0.48.0 + - markdownlint@0.49.0 - osv-scanner@2.3.5 - oxipng@10.1.0 - prettier@3.8.3 From 009c511efd326d87f0935298d75e9b8c1d639705 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Fri, 26 Jun 2026 12:44:39 -0400 Subject: [PATCH 3/3] Address review: enforce version lockstep and broaden regression asserts From CodeRabbit's review: - prompts_version.py: the check now fails if the prompt-set version in manifest.yaml drifts from the package version in pyproject.toml, enforcing the lockstep the governance doc documents. - test_semantic_regression.py: also assert code_snip, line_number, and variables render, so the prompt contract covers every incident field the template depends on, not just uri/message/violation. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Fabian von Feilitzsch --- kai_mcp_solution_server/scripts/prompts_version.py | 13 +++++++++++++ .../tests/prompts/test_semantic_regression.py | 3 +++ 2 files changed, 16 insertions(+) diff --git a/kai_mcp_solution_server/scripts/prompts_version.py b/kai_mcp_solution_server/scripts/prompts_version.py index c7000a134..75a9fd6b8 100644 --- a/kai_mcp_solution_server/scripts/prompts_version.py +++ b/kai_mcp_solution_server/scripts/prompts_version.py @@ -20,6 +20,7 @@ import hashlib import re import sys +import tomllib from pathlib import Path import yaml @@ -29,6 +30,7 @@ PROMPTS_DIR = ROOT / "src" / "kai_mcp_solution_server" / "prompts" TEMPLATES_DIR = PROMPTS_DIR / "templates" MANIFEST_PATH = PROMPTS_DIR / "manifest.yaml" +PYPROJECT_PATH = ROOT / "pyproject.toml" # Matches a leftover Python f-string placeholder like ``{incident.uri}`` while # ignoring Jinja's own ``{{ ... }}`` / ``{% ... %}``. @@ -49,6 +51,17 @@ def check() -> int: problems: list[str] = [] env = Environment() # noqa: S701 # nosec B701 - parses templates, never renders + # The prompt-set version must stay in lockstep with the package version so + # release metadata can't drift (see PROMPT_GOVERNANCE.md). + pkg_version = ( + tomllib.loads(PYPROJECT_PATH.read_text()).get("project", {}).get("version") + ) + if manifest.get("version") != pkg_version: + problems.append( + f"Prompt-set version ({manifest.get('version')}) does not match the package " + f"version in pyproject.toml ({pkg_version}). Keep them in lockstep." + ) + on_disk = {p.relative_to(ROOT).as_posix() for p in TEMPLATES_DIR.rglob("*.jinja")} declared = {t["path"] for t in templates} for path in on_disk - declared: diff --git a/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py b/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py index bf0c9c199..dabc13105 100644 --- a/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py +++ b/kai_mcp_solution_server/tests/prompts/test_semantic_regression.py @@ -69,5 +69,8 @@ def test_v3_preserves_migration_scaffolding(name: str) -> None: for incident in incidents: assert incident.uri in rendered assert incident.message in rendered + assert incident.code_snip in rendered + assert str(incident.line_number) in rendered + assert str(incident.variables) in rendered assert incident.violation.violation_name in rendered assert ast_diff_str in rendered