diff --git a/.agents/roadmap_v1.md b/.agents/roadmap_v1.md index d938f39cc..ac0422822 100644 --- a/.agents/roadmap_v1.md +++ b/.agents/roadmap_v1.md @@ -56,8 +56,12 @@ issue is not yet placed. Keyed record: update in place, never append. | [#374](https://github.com/mudler/vllm.cpp/issues/374) | `ENG-NOW-DERIVED` | NOW.md is still a surface every PR must write: the doc-checkpoint gate marches them into it | bug | | [#364](https://github.com/mudler/vllm.cpp/issues/364) | `ENG-RECORD-CONFLICT-SURFACES` | Shared record surfaces are a lock: 16/29 open PRs conflict, 13 of them in bookkeeping only | bug | | [#460](https://github.com/mudler/vllm.cpp/issues/460) | `ENG-RECORD-CONFLICT-SURFACES` | `docs/BENCHMARKS.md` cannot be compacted: `check_links` validates links inside fenced blocks, so no row with a `docs/`-relative link can be archived byte-for-byte | bug | +| [#495](https://github.com/mudler/vllm.cpp/issues/495) | `ENG-RECORD-CONFLICT-SURFACES` | The gate and the rollup disagree on what a section is: `_h2_headers` ignores code fences, so the failure it reports has a remedy that does nothing | bug | +| [#498](https://github.com/mudler/vllm.cpp/issues/498) | `ENG-RECORD-CONFLICT-SURFACES` | `README.md` has 35 chars of headroom: `MAX_README_CHARS` is the same whole-file lock retired three times already, and one commit already landed 52 over | bug | +| [#507](https://github.com/mudler/vllm.cpp/issues/507) | `ENG-RECORD-CONFLICT-SURFACES` | `_prose_paragraphs` excludes any line starting with `*`, so an EMPHASIS-lead prose wall is counted by nothing; closing it turns four already-shipped `docs/BENCHMARKS.md` paragraphs red | bug | | [#117](https://github.com/mudler/vllm.cpp/issues/117) | `ENG-RELEASE-WINDOWS` | Binary release, including requested native Windows CPU/AVX2 and Vulkan bundles | feature | | [#447](https://github.com/mudler/vllm.cpp/issues/447) | `ENG-RELEASE-WINDOWS` | Native Windows release PR exposed stale version and archive-format contracts in hosted CI | bug | +| [#503](https://github.com/mudler/vllm.cpp/issues/503) | `ENG-RELEASE-WINDOWS` | `main` does not compile under MSVC (`test_cpu_isa_x86.cpp` lacks ``), and `main-baseline.py` reports GREEN because `windows-msvc-*` are PR-only jobs its lane never runs | bug | | [#448](https://github.com/mudler/vllm.cpp/issues/448) | `ENG-RELEASE-WINDOWS` | PR #446 device-leakage repair violates the per-commit documentation checkpoint | bug | | [#450](https://github.com/mudler/vllm.cpp/issues/450) | `ENG-RELEASE-WINDOWS` | PR #446 archive-target repair lacks its usage projection | bug | | [#453](https://github.com/mudler/vllm.cpp/issues/453) | `ENG-RELEASE-WINDOWS` | PR #446 exact-range gate lacks closed creation evidence for its new Windows checkers | bug | diff --git a/.agents/specs/benchmarks-entry-cap.md b/.agents/specs/benchmarks-entry-cap.md new file mode 100644 index 000000000..07c44a87d --- /dev/null +++ b/.agents/specs/benchmarks-entry-cap.md @@ -0,0 +1,569 @@ +# The BENCHMARKS cap is a file cap, and a file cap is a lock + +Issue: [#460](https://github.com/mudler/vllm.cpp/issues/460). +Row: `ENG-RECORD-CONFLICT-SURFACES`. +Measured against `origin/main` `918c568a` on 2026-08-12. + +`docs/BENCHMARKS.md` is gated by a 45,000-character budget on the whole file. +The page measures 44,795 characters, so 205 characters are free. Adding a +measurement row therefore means deleting somebody else's row, and the documented +way to delete one, moving it byte-for-byte into `.agents/benchmark-record.md`, +is broken for exactly the rows that carry evidence links. + +This is the same defect this row already retired twice on 2026-08-11: the +`MAX_CHARS` budget in `scripts/check-now-current.py` and the `chars` key of +`STATUS_RATCHET` in `scripts/check-public-doc-tables.py`, both removed by +`87308dea` under #364 with the reasoning recorded in place. The scoreboard's own +`max_chars` was left standing in that pass. It is now the binding constraint on +every remaining roadmap measurement. + +AGENTS.md, Records, states the rule this spec applies: **cap the entry, never +the file**, and **a gate is what usually creates the lock: if a checker requires +every change to touch a shared file, that is the defect**. + +## Scope + +**In scope.** + +1. `max_chars` on `PageRules` in `scripts/check-public-doc-tables.py`, which + applies to `docs/BENCHMARKS.md` (45,000) and `docs/FEATURES.md` (30,000). +2. The rules that take over its obligation: a per-row character cap, a regrowth + guard on per-attempt headings at any depth, and, added for review finding F1, + a `_prose_paragraphs` that counts list items and blockquote lines so the + append-only wall the byte cap was also catching stays caught. The first two + are entry-scoped; the third is a count, and Risks says so plainly. +3. `check_links` in `scripts/check-agent-record.py`, so the archive path the + scoreboard points at works for a row that carries a relative link. +4. The mutation suites `tests/scripts/test_check_public_doc_tables.py` and + `tests/scripts/test_agent_record.py`. +5. The acceptance demonstration: the owed 35B canonical regrid row named by + #481 lands with nothing evicted. **Implemented as a test that adds the row to + the real page and drops it again, not as an edit to the page.** #481 is open + and rewrites the 35B row in place, so writing a second copy of the same fact + would duplicate a keyed row the moment both merge. The surface's ability to + accept the row is what this row owes; the row's content is #481's. + +**Out of scope, deliberately.** The `STATUS_RATCHET` keys kept by #364; the +required-section, canonical-section, prose-paragraph, paragraph-length and +cell-length rules, all of which are kept and none of which are widened; the +content of any existing row; `scripts/roll-benchmark-record.py`'s move logic; +and rebuilding the public scoreboard as a derived or globbed surface, which is +argued against in Design and deferred in Work breakdown. + +**Not a correctness change.** No product source, kernel, ABI or model path +moves. Nothing measured changes value. + +## Upstream chain + +**No vLLM counterpart. This is project infrastructure.** vLLM has no equivalent +of `docs/BENCHMARKS.md`, of `.agents/benchmark-record.md`, or of the checker +suite that gates them: they exist to serve this project's protocol, which vLLM +does not run. The authority for this change is AGENTS.md, Records, and the +precedent set by `87308dea` (#364) on the two sibling budgets. + +## Our baseline + +**The page has had no usable headroom for 25 commits.** Free characters against +the 45,000 cap at each of the last 25 commits that touched `docs/BENCHMARKS.md`, +with the page's table-row count: + +| commit | rows | chars | free | subject | +|---|---:|---:|---:|---| +| `918c568a` | 162 | 44,795 | 205 | measure(SPEC-DSPARK) fibonacci gap | +| `523b8a6f` | 162 | 44,826 | 174 | measure(SPEC-DSPARK) 5-rep interleaved | +| `887e04ff` | 163 | 44,579 | 421 | **docs(release): compact benchmark projection (#475)** | +| `1c9dbe08` | 163 | 44,931 | 69 | perf(SPEC-DSPARK) sync-free Markov chain | +| `bbc482a2` | 163 | 44,942 | 58 | merge: Whisper encoder FA-2 | +| `c5615cfe` | 164 | 44,692 | 308 | perf(SPEC-DSPARK) speculative verify | +| `4112ac8c` | 165 | 44,968 | 32 | fix(mm-speed) review findings | +| `425abf7c` | 163 | 44,936 | 64 | bench(cpu) x86_64 floor | +| `93613baa` | 165 | 44,964 | 36 | **docs(benchmarks): trim the Voxtral encoder row back inside** | +| `04b2b9fa` | 165 | **45,007** | **-7** | **merge: origin/main into row/MM-SPEED-ENC-FA2** | + +Three facts follow, and each of them is the thing AGENTS.md names. + +**The success mode is unsafe.** `04b2b9fa` is a clean automatic merge that +landed the page at 45,007 characters, 7 over the cap. Two PRs each paid for +their row by evicting a different one; the three-way merge applied both +additions and neither eviction cancelled the other. That is verbatim the +corollary in Records: "merging two such edits cleanly is worse than +conflicting". It has already happened here, in the tree, not in theory. + +**The eviction is real and it is winning.** Row count fell from 165 to 162 over +these 25 commits while the project gained measurements. Two commits, +`93613baa` and `887e04ff`, exist for no purpose but to pay rent: their subjects +are "trim the Voxtral encoder row back inside" and "compact benchmark +projection". + +**The payment mechanism does not work for the rows that carry evidence.** +`check_links` (`scripts/check-agent-record.py:599-610`) runs `LINK_RE.findall` +over the raw file with no fenced-span stripping and resolves every hit from +`source.parent`. Three of the 162 rows carry a `docs/`-relative link +(`bench-evidence/qwen35-4b-sm120-main-20260807.md`, +`bench-evidence/rpi5-a76-q8-dot-20260806.md`, +`bench-evidence/rpi5-a76-llamacpp-20260806.md`), and none of them can be moved +into `.agents/benchmark-record.md` byte-for-byte: the target resolves from +`.agents/` and dangles. #433 hit this and had to archive a shorter link-free row +instead. The subset of payable rows shrinks every time one is spent. + +**What the cap is actually still catching.** `_h2_headers` matches `## ` only, +so `### ` subsections are ungoverned by the canonical-section allowlist. The +live page carries six of them. An appended per-attempt `### ` section is +rejected today by nothing except the character budget. + +**Corrected 2026-08-12, review finding F1: that is not the only obligation.** +`_prose_paragraphs` also excludes every line starting with `-`, `*`, `>`, `|` or +`#`, so a bulleted or blockquoted wall of forensics is outside the paragraph +count, `MAX_PARAGRAPH_CHARS`, `MAX_CELL_CHARS`, `MAX_ROW_CHARS` and the heading +guard at once. The character budget is the only thing that ever caught it, which +means the replacement has to pick up **two** obligations, not one: the appended +subsection, and the append-only wall of list and quote lines. Design §3a is the +second. The mutation table in Risks is the measurement. + +## Port map + +**No upstream file to port from.** The local anchors this change edits: + +| Anchor | What changes | +|---|---| +| `scripts/check-public-doc-tables.py` | `max_chars` removed from `PageRules`; `MAX_ROW_CHARS` and `DATED_HEADING_RE` added; `page_errors` gains the two entry-scoped checks; `_prose_paragraphs` folds list items and blockquote lines in (`LIST_ITEM_RE`), so `max_prose_paragraphs` re-baselines to 21 on FEATURES and `STATUS_RATCHET["long_paragraphs"]` tightens 82 to 75 | +| `scripts/check-agent-record.py` | `check_links` gains `extract_links`, a pure fenced/inline-code-aware link scanner; `FENCE_RE` and `strip_code_spans` implement CommonMark's closing-fence rule; `link_base` becomes `link_bases` | +| `tests/scripts/test_check_public_doc_tables.py` | three `max_chars` tests replaced by entry-cap, regrowth and no-eviction tests | +| `tests/scripts/test_agent_record.py` | new `LinkExtraction` cases | +| `docs/BENCHMARKS.md` | **unchanged.** The owed row is added and dropped inside `test_the_shipped_page_can_accept_the_next_measurement_row`, so the surface is proven without writing a fact #481 already owns | + +## Design + +**Remove the file cap. Relocate its obligation to two entry-scoped rules.** + +*1. `max_chars` is deleted from `PageRules`, with the reason recorded in place.* +A byte budget on a shared page is a lock by construction: every addition is a +read-modify-write of one global, the conflict is the lucky outcome, and the +clean merge is the unsafe one. `87308dea` removed the two sibling budgets on +exactly this argument; this is the third, left behind in that pass. + +*2. `MAX_ROW_CHARS = 600` caps one table row.* This is the literal "cap the +entry": a measurement's cost is bounded locally, by the author of that +measurement, and never by deleting a row somebody else owns. It is the same +shape as `MAX_ENTRY_CHARS` in `check-now-current.py`. 600 is set from the live +pages: the longest row on `docs/BENCHMARKS.md` is 520 characters and on +`docs/FEATURES.md` 580. It is a genuine constraint, and a tighter one than what +it joins: `MAX_CELL_CHARS = 220` alone permits a five-column row of 1,100 +characters. + +*3. `DATED_HEADING_RE` rejects a per-attempt heading at any depth.* This is the +regrowth guard, the same shape as the `ROW_TABLE_LINE` guard `87308dea` added to +`check-now-current.py`, and it closes the `### ` hole the character cap was +covering. The shape is measured, not invented: of the **310 sections already +rolled into `.agents/benchmark-record.md`, 287 name a date in their heading** +(`2026-08-07`, `2026-07-31`, ...), which is what a per-attempt entry looks like +here. **Zero of the 36 live headings across the two public pages name a date** +(18 each, at every depth, `_headings`). So the guard fires on the first appended +DATED checkpoint section, at `##` or `###`, and never on a subject section. + +**What this guard is, and is not, tighter than.** It is strictly tighter than +the canonical-section allowlist it joins, which runs over `_h2_headers` and so +sees `## ` only: a DATED `### ` subsection was previously rejected by nothing +but the character budget. It is **not** tighter than the retired byte cap in +general, and the first revision of this spec, of the checker comment and of the +PR body all said that it was. That claim is false and was proven false by +mutation. An **undated** appended subsection passes this guard by construction, +because the guard fires on a *date*. See §3a, which is the rule that closes +that. + +*3a. `_prose_paragraphs` folds list items and blockquote lines in.* Added +2026-08-12 in response to review finding F1, which is the finding that showed +§3 alone does not discharge the stop condition. `_prose_paragraphs` excluded +every line starting with `-`, `*`, `>`, `|` or `#`, so a bulleted or quoted wall +was counted by **nothing**: not the paragraph count, not `MAX_PARAGRAPH_CHARS`, +not `MAX_CELL_CHARS`, not `MAX_ROW_CHARS`, and not the heading guard. Two +mutations measured it against the real checker with the entry cap in place: + +| mutant | page size | verdict before §3a | verdict after | +|---|---:|---|---| +| 3,000 appended bullet lines (the exact mutant `test_oversized_page_fails` used, which BASE rejects as `113833 chars, over the 45000-char scoreboard budget`) | 113,833 | **SURVIVED**, `[]` | CAUGHT, `prose paragraph of 194014 chars exceeds 700` | +| 500 appended UNDATED `### Attempt N` sections with bulleted forensics | 117,222 | **SURVIVED**, exit 0, 68/68 green | CAUGHT, `535 prose paragraphs, over the 35 budget` | +| 3,000 appended blockquote lines | 127,718 | **SURVIVED**, `[]` | CAUGHT, `prose paragraph of 83014 chars exceeds 700` | + +A contiguous run of list items folds into ONE paragraph rather than N, so a +legitimate short list costs one paragraph and a wall trips +`MAX_PARAGRAPH_CHARS`, while a run per appended section trips the paragraph +COUNT. `docs/BENCHMARKS.md` carries no list item and is unmoved at 35; +`docs/FEATURES.md` carries one and moves 20 to 21, so `max_prose_paragraphs` is +**re-baselined to 21**, which is a re-measurement under a larger counted +population and not slack. `docs/STATUS.md` shares the function and moves the +other way, 82 long paragraphs to 75, because its 29 list items now join +neighbouring paragraphs instead of splitting them; `STATUS_RATCHET` follows the +measurement **down** to 75 in the same change, since banking the 7 units would +be exactly the slack the ratchet exists to refuse. + +*4. `check_links` stops validating text that is not a link, and the archive +resolves what it archived.* Two changes, both narrow: + +- Fenced code blocks and inline code spans are stripped before `LINK_RE`. A + target inside a fence is not a link: CommonMark renders it as literal text, no + reader can follow it, and there is nothing for the checker's rule ("every link + resolves") to be about. Today the checker forbids any document in the tree + from *showing* a link in sample output, which is why #460's own reproduction + is a fence. +- **The pairing rule is CommonMark's**, added 2026-08-12 for review finding F2. + A closing fence must use the opener's character, be at least as long, and + carry nothing but whitespace after the marker; a line with an info string + opens a block and never closes one. The first revision closed on any line + matching `^\s*(```+|~~~+)`, which does not fail safe: it **inverts fence phase** + for the rest of the file. With the one unbalanced fence this tree already has, + the bare ` ``` ` at + `.agents/completed/state-events/0000-00/STATE-LEGACY-000001.md:17697` was + "closed" by the ` ```sh ` at `:17948`, and from there ordinary prose was + blanked, including a live reader-followable spec link at `:18297`. Under the + corrected rule that link is validated again and three lines of genuinely + fenced sample are not. +- `.agents/benchmark-record.md` resolves a target from `docs/` as well as from + `.agents/`. It is the declared archive of `docs/BENCHMARKS.md`, so content + moved into it verbatim was written against `docs/`. `link_base` already + carries one such special case for migrated legacy payloads; this generalises + it to `link_bases`, a tuple, and a target still has to exist under one of + them. Fence-stripping alone does not cover this case, because + `roll-benchmark-record.py` moves sections as live markdown, not fenced. + +**Why not (a) alone, the `check_links` fix.** It unblocks payment and leaves the +ratchet. Every measurement would still evict a row somebody else owns, and the +clean merge of two such payments would still land the page over budget, as +`04b2b9fa` did. It treats the symptom the issue was filed from and not the +defect the issue describes. + +**Why not (b), per-row files or a derived page.** The three admissible shapes in +Records govern *record* surfaces. `docs/BENCHMARKS.md` is not one: AGENTS.md, +Public documents, defines it as a projection whose purpose is to be one readable +page a user reaches from the README badge. "Derived at read time, so nobody +writes it" removes the lock only when the rendered artifact is not committed, +and GitHub renders committed markdown with no build step, so the generated page +would still be committed and still be a file every measurement PR writes. The +lock would move from the author to the generator, not die. It is also not what +is blocking: the binding pain today is the *cap*, not conflicts. Deferred as W4 +with the condition that would justify it. + +**Why not simply raise the cap.** Prohibited by the task, and the checker's own +comment already records why it does not work: the previous occupant of this line +"answered it by adding slack to the constant, which only postponed it to the +next cadence of parallel work". + +## Tests to port + +**No upstream tests exist.** These are written against this project's checkers. + +`tests/scripts/test_check_public_doc_tables.py`: + +- `test_the_shipped_page_can_accept_the_next_measurement_row`: **the acceptance + test.** It adds the owed 35B canonical regrid row to the REAL page, + immediately below the last row of the Open gaps table so markdown renders it + inside that table, and asserts every pre-existing row survives. The row is + added and dropped inside the test, so the page is not edited and #481 is not + collided with. It does NOT assert the grown page exceeds the retired budget: + review finding F3 showed that assertion stored a measurement of + `docs/BENCHMARKS.md` inside a test file, with 173 characters of margin, so any + PR compacting the page by more than that turned it red in a file it does not + own. `887e04ff` shrank the page by 280 and `93613baa` by 43. The + above-the-budget point belongs to the synthetic test below and is made there. +- `test_a_new_row_costs_no_eviction`: the same property on a synthetic page + sized past the retired 45,000 budget. RED on BASE, which reports + `48836 chars, over the 45000-char scoreboard budget`. +- `test_oversized_row_fails`: a single row past `MAX_ROW_CHARS` is rejected. + RED on BASE, which has no row cap. +- `test_the_row_cap_is_not_subsumed_by_the_cell_cap`: a row of legal cells whose + sum is illegal is rejected, so the entry cap adds a rule `MAX_CELL_CHARS` does + not already carry. +- `test_a_row_at_the_cap_is_allowed`: the boundary is inclusive. +- `test_a_dated_h2_is_rejected`, `test_a_dated_h3_is_rejected`, and + `test_a_dated_h3_is_rejected_on_the_feature_matrix_too`: an appended + per-attempt section fails at the first one, at either depth, on either page. + RED on BASE for `###`, which no rule covered. +- `test_a_new_subject_subsection_is_allowed` and `test_a_dated_row_is_still_allowed`: + the guard is not a section freeze and does not reach into rows. +- `test_a_date_inside_a_fence_is_not_a_heading`: sample output is not a section. +- `test_the_shipped_pages_carry_no_dated_heading`: the two live pages satisfy + the new guard as shipped. +- `test_no_page_carries_a_whole_file_size_budget`: the invariant behind this + row, held as a rule rather than as a habit. +- `test_a_wall_of_BULLETS_fails`, `test_a_wall_of_BLOCKQUOTES_fails` and + `test_appended_UNDATED_subsections_with_bullets_fail`: mutants M8, M8b and M7 + from Risks, each RED on the first revision of this row and on nothing before + it except the byte cap. +- `test_a_short_bullet_list_is_still_allowed` and + `test_a_bullet_run_is_ONE_paragraph_not_many`: the fold is a budget, not a ban, + and a run costs one paragraph rather than N, which is what keeps the count + budget off ordinary documents. +- `test_an_EMPHASIS_lead_wall_is_a_KNOWN_residue`: characterisation of M11. It + goes RED the day W8 lands, which is the point. +- `test_the_shipped_pages_sit_at_their_folded_paragraph_budget`: the two + paragraph budgets are pinned to what the pages measure under the fold, so + re-baselining FEATURES from 20 to 21 is a re-measurement and cannot become + slack. +- Replaced: `test_oversized_page_fails`, + `test_release_projection_fits_after_current_main_merge`'s `max_chars` + assertion, `test_the_two_pages_have_distinct_budgets`'s `max_chars` + comparison. Each is replaced by an assertion on the rule that took the + obligation over, never deleted outright. + +`tests/scripts/test_agent_record.py`, new `LinkExtraction`: + +- `test_fenced_link_is_not_extracted`, `test_tilde_fenced_link_is_not_extracted` + and `test_inline_code_link_is_not_extracted`: RED on BASE, which extracts all + three. +- `test_live_link_is_still_extracted`, `test_a_backticked_label_is_still_a_link`, + `test_link_after_a_closed_fence_is_still_extracted` and + `test_link_beside_an_inline_span_is_still_extracted`: the narrowing does not + swallow real links, including the `` [`name`](path) `` form this tree uses + everywhere. +- `test_stripping_preserves_line_and_column_positions`: spans are blanked, not + deleted, so every line and column offset survives. Review finding F6: this is + NOT evidence of anything today, because `check_links` reports no line numbers + at all. It is held so that a caller that does report positions cannot be + broken by this function, and the docstring now says so. +- `test_a_fence_with_an_INFO_STRING_does_not_close_a_block`, + `test_a_closing_fence_must_match_the_opener` and + `test_a_LONGER_closing_fence_does_close`: CommonMark's closing-fence rule, both + directions (F2). +- `test_prose_two_lines_below_a_closed_fence_is_still_scanned`: the live case, on + the real file that mis-paired. RED on the first revision of this row. +- `test_a_link_straddled_by_two_INLINE_SPANS_is_not_extracted`: two stray + backticks hide a target, matching the renderer, and the ordinary + `` [`name`](path) `` form is still a link (F5). +- `test_an_archived_row_with_a_docs_relative_link_is_accepted`: the #460 + reproduction, moved into the record as live markdown, resolves. RED on BASE + with `dangling link bench-evidence/rpi5-a76-q8-dot-20260806.md`. +- `test_an_archived_row_with_a_MISSING_link_still_dangles`: the second base is a + base, not an amnesty. +- `test_the_benchmark_record_also_resolves_from_docs` and the two `link_bases` + cases in `MigratedLegacyLinks`: every other file keeps single-base resolution. +- `test_the_tree_has_no_dangling_link`: the whole-tree run stays green. + +## Gates + +1. `python3 -m pytest tests/scripts/ -q`, unbounded, full run. +2. `python3 scripts/check-public-doc-tables.py`. +3. `python3 scripts/check-agent-record.py`. +4. `scripts/agent-preflight.sh --staged`. +5. `python3 scripts/check-pr-size.py` red-before/green-after harness on both + changed checkers: it reruns each HEAD test module against the BASE checker in + an isolated worktree and refuses the PR unless BASE goes red. +6. Acceptance: the live `docs/BENCHMARKS.md` plus the owed row is valid and + carries one more row and no fewer, with the row placed directly under the + last Open gaps row so it renders in that table. Measured on the merged tree: + **162 rows to 163, errors `[]`**, with every pre-existing row asserted still + present. The test no longer compares the page against the retired budget: see + Tests to port, and review finding F3. +7. The mutation set, run against three trees. Table in Risks. + +**No GPU. Nothing here measures.** + +## Dependencies + +- #364 / `87308dea`, which set the precedent and removed the two sibling + budgets, is on `main`. +- #481 is open and rewrites the 35B row on `docs/BENCHMARKS.md` in place. This + row does not edit the page at all, so the two cannot conflict, and #481 keeps + ownership of the "regrid owed" fact. That is why the acceptance demonstration + is a test rather than an edit. +- `origin/main` moved from `918c568a` to `e1087a88` mid-row (12 commits, + GATE-PIN-UNPINNED-SNAPSHOTS #471 and four SPEC-DSPARK measurements #442). + Merged, and every gate rerun on the merged tree. The measurements in Our + baseline are as taken at `918c568a` and are not restated. +- #481 also blocks W8 / #507, for the same reason it shapes the acceptance + demonstration: closing the emphasis-lead residue owes an edit to four + paragraphs on a page #481 holds open. +- Nothing else blocks. + +## Work breakdown + +| W | Work | State | +|---|---|---| +| W1 | This spec, committed alone | this commit | +| W2 | `check-public-doc-tables.py`: retire `max_chars`, add `MAX_ROW_CHARS` and `DATED_HEADING_RE`, with tests | in this PR | +| W3 | `check-agent-record.py`: fenced/inline-code-aware `extract_links`, `link_bases` for the archive, with tests | in this PR | +| W4 | Rebuild `docs/BENCHMARKS.md` as a derived index over per-row files | **DEFERRED.** Justified only if the page becomes a *conflict* hotspot after the cap is gone. Trigger: `git merge-tree` shows it conflicting in 3 or more concurrently open PRs, measured, as #364 measured its three surfaces. Not justified by the cap, which W2 removes. | +| W5 | Teach `roll-benchmark-record.py` to record the archived section's origin explicitly, rather than relying on W3's two-base resolution | **DEFERRED.** W3 makes the move work; W5 would make it self-describing. No blocker depends on it. | +| W6 | Make `_h2_headers` fence-aware, so the gate and the rollup agree on what a section is ([#495](https://github.com/mudler/vllm.cpp/issues/495)) | **DEFERRED, filed not fixed.** Found doing W2 and reproduced: `_h2_headers` is a bare `startswith("## ")` scan while `split_sections` tracks fences, so a heading-shaped line inside a fence is a section to the gate and not to the script the gate tells you to run. It changes what an existing gate counts, so it takes its own spec and red-before rather than riding along here. Neither page has a fenced heading today. W2's `_headings` is already fence-aware and is the natural basis for the repair. | +| W7 | Retire `MAX_README_CHARS` the same way ([#498](https://github.com/mudler/vllm.cpp/issues/498)) | **DEFERRED, filed not fixed.** `README.md` measures 29,965 of 30,000: **35 characters free**, tighter than any of the three budgets already retired. 13 of the last 20 commits touching it sat under 60 free, `031410e8` landed it 52 OVER, and `row/DOCS-README-BUDGET` (#161) is a whole merged row whose purpose was paying rent. Same defect, third checker (`check-readme-structure.py`), so it needs its own spec and its own mutation in `tests/scripts/test_check_readme_structure.py` rather than riding along here. | +| W8 | Close the EMPHASIS-lead paragraph residue ([#507](https://github.com/mudler/vllm.cpp/issues/507)) | **DEFERRED, filed not fixed.** §3a folds list items and blockquote lines into `_prose_paragraphs`, but a line opening with emphasis still starts with `*` and is still excluded, so a `**bold**`-led prose wall is unbounded (mutant M11). Replacing the bare `*` prefix with `LIST_ITEM_RE` closes it and immediately turns FOUR paragraphs already shipped on `docs/BENCHMARKS.md` red against `MAX_PARAGRAPH_CHARS`, at 717, 719, 748 and 1,084 characters, so it owes an edit to a page #481 holds open and it changes what an existing gate counts. Own spec, own red-before. `test_an_EMPHASIS_lead_wall_is_a_KNOWN_residue` pins the residue and goes red the day it lands. | + +## Risks/decisions + +**Risk: removing a size gate lets the page bloat.** Answered by measurement, not +assertion, and the first answer was wrong. What bloated the page to 11,405 lines +was per-attempt sections, and after W2 a DATED one fails at either heading +depth, where before only `##` was covered. But a bulleted or blockquoted wall, +with or without undated `###` headings above it, was caught by nothing at all +until §3a folded those lines into `_prose_paragraphs`. The full mutation set, +run against BASE `origin/main`, against the first revision of this row, and +against the revision that lands (`scratchpad/mutate.py`, reproduced in the PR +body): + +| mutant | BASE | first revision | landed | +|---|---|---|---| +| M1 dated `##` appended | CAUGHT | CAUGHT | CAUGHT | +| M1b dated `###` appended | CAUGHT | CAUGHT | CAUGHT | +| M2 one row over 600 | CAUGHT (by size) | CAUGHT | CAUGHT | +| M3 legal cells, illegal row | CAUGHT (by size) | CAUGHT | CAUGHT | +| M4 required section dropped | CAUGHT | CAUGHT | CAUGHT | +| M5 em-dash | CAUGHT | CAUGHT | CAUGHT | +| M6 200 legal rows, 599 chars each | CAUGHT (by size) | SURVIVED | **SURVIVED, intended** | +| M7 500 undated `###` + bullets | CAUGHT (by size) | **SURVIVED** | CAUGHT | +| M8 3,000 bullet lines | CAUGHT (by size) | **SURVIVED** | CAUGHT | +| M8b 3,000 blockquote lines | CAUGHT (by size) | **SURVIVED** | CAUGHT | +| M9 live post-fence link redirected to a missing target | CAUGHT | **SURVIVED** | CAUGHT | +| M10 archived row, missing `docs/`-relative link | CAUGHT | CAUGHT | CAUGHT | +| M11 500 `**bold**`-lead paragraphs | CAUGHT (by size) | SURVIVED | **SURVIVED, known residue** | + +Two mutants the byte cap caught are not caught after this row, and both are +stated rather than glossed. **M6 is the point of the row**: rows are the growth +mode of a keyed table, each is capped at 600, and no author pays for one by +deleting another's. **M11 is a defect**, filed as +[#507](https://github.com/mudler/vllm.cpp/issues/507) and deferred as W8: a line +opening with emphasis starts with `*` and is still excluded, so a `**bold**`-led +wall is unbounded. It is deferred, not dismissed, because closing it turns four +paragraphs already shipped on `docs/BENCHMARKS.md` red against +`MAX_PARAGRAPH_CHARS` (717, 719, 748 and 1,084 characters) and so owes an edit +to a page #481 holds open. `test_an_EMPHASIS_lead_wall_is_a_KNOWN_residue` pins +it and goes red the day it is closed. + +**Risk: `DATED_HEADING_RE` fires on a legitimate heading.** A pinned-date +subject would trip it, for example "vLLM 0.26.0 as of 2026-08-12". Measured +against both live pages: zero of 36 headings match, 18 per page at every depth. +If one is ever wanted, the date belongs in the row or the prose, which are +unaffected. The error message says so. + +**Risk: the paragraph budget is itself a whole-page count.** It is, and both +pages now sit exactly on it: `docs/BENCHMARKS.md` at 35 of 35 and +`docs/FEATURES.md` at 21 of 21. So this row removes the file lock on ROWS and +leaves one on PROSE. That is deliberate and is the same line #364 drew when it +deleted `chars` from `STATUS_RATCHET` and kept `long_paragraphs`: rows are how a +keyed table grows and prose is how it decays, so the count is a quality gate on +the decay mode rather than rent on the growth mode. It is still a real cost to +an author adding a paragraph, and it is now recorded in the `PageRules` +docstring so the next one meets it there instead of in CI. + +**Risk: `MAX_ROW_CHARS = 600` is tuned to current content.** It is, and that is +sound for an entry cap in a way it is not for a file cap: an author who needs a +longer row shortens their own row, and never anyone else's. The FEATURES margin +is thin (580 of 600). Accepted; the alternative, no row cap, leaves +`MAX_CELL_CHARS` permitting a 1,100-character row. + +**Risk: skipping fenced links narrows a checker.** It narrows it to what the +rule was always about. A fenced target is not a link under CommonMark, so no +reader can follow it and no rendering can dangle. Real links, including one on +the same line after a closed inline span, stay checked, and +`test_the_tree_has_no_dangling_link` holds the whole tree. + +Measured at merge `fdbc8ae6`, over the markdown files the checker scans: +**4,170 targets before the strip, 4,163 after**, so 7 stop being validated, and +every one of the 7 is a code sample. Three are inside the unclosed block at +`.agents/completed/state-events/0000-00/STATE-LEGACY-000001.md:17697` +(`porting.md`, `../tests/vt/test_ops_matmul_elem.cpp`, +`specs/accelerator-seam-audit.md`); the other four are the `path` placeholder in +this spec's own samples. **The loose fence rule the first revision shipped keeps +4,162 on this same tree, one FEWER than the CommonMark rule, and the one it +loses is not a sample**: it is the live prose link at `:18297`, lost to the +phase inversion F2 describes. The corrected pairing restores it. + +These totals move every time the tree gains a markdown file, so they are dated +to a SHA rather than treated as constants. Reproduce with `extract_links` and +`LINK_RE.findall` over `markdown_files()`; the RATIO that matters, and the thing +that does not move, is that the corrected rule validates strictly more than the +loose one and loses only samples. + +The spec's earlier figures, 4,109 before and 4,105 after with "all 4 are quoted +samples", were both stale and wrong about the count: the review re-measured 5 +lost, and the fifth was the live link. + +**Risk: two stray backticks hide a target that does not exist.** They do. This +form, which this spec itself carried until F5 found it: + +```text +`[`name`](path)` +``` + +is four backticks, so CommonMark reads it as a code span holding an open +bracket, the literal text name, and a second code span holding the rest, and +there is no link for the checker to validate. The checker agrees with the +renderer, which is the rule, but the consequence is that wrapping a dangling +link in backticks hides it. Pinned by +`test_a_link_straddled_by_two_INLINE_SPANS_is_not_extracted`, which asserts the +same answer for the double-backtick form an author should use instead, and +asserts that the ordinary `` [`name`](path) `` form is still a link. + +**Risk: an unbalanced fence blanks the rest of a file.** It does, and one file +in the tree has one: `.agents/specs/laguna-s21-scope-2026-07-30.md` ends on a +stray closing fence at its last line. It costs nothing today, because there is +no content after it, and the same behaviour already exists in +`_prose_paragraphs`, `_table_rows` and `split_sections`. Making an unbalanced +fence an error is a separate rule with a separate red-before, not part of this +change. + +**Risk: two-base resolution in the archive hides a genuine dangling link.** The +second base applies to one file, `.agents/benchmark-record.md`, the declared +archive of `docs/BENCHMARKS.md`, and the target must still exist under one of +the two bases. Every other file keeps single-base resolution. + +**Decision: `docs/FEATURES.md` loses its `max_chars` too.** The constant lives on +the shared `PageRules`, the argument is identical, FEATURES is at 29,740 of +30,000 with 260 characters free, and leaving it would leave a known lock armed +on the page that grows every time a feature ships. It gains the same two +entry-scoped rules. + +## Evidence + +- Char and row history: reproduce with `git show :docs/BENCHMARKS.md | wc -c` + over `git log --format=%h -25 -- docs/BENCHMARKS.md`. Table in Our baseline. +- `04b2b9fa` over the cap: `git show 04b2b9fa:docs/BENCHMARKS.md | wc -c` gives + 45,007. +- Heading-shape survey, re-measured on the merged tree after review finding F8 + found the first numbers unreproducible: at merge `fdbc8ae6`, **287 of 310** + archived section titles carry a date, and **0 of 36** live headings do, 18 per + page. The archive total climbs as rows land, so it is dated. Reproduce with + `_headings` and `DATED_HEADING_RE` over the two pages, and with + `grep -c '^## '` over `.agents/benchmark-record.md`. The spec previously said + 278 of 301 and 0 of 32; 32 was `_h2_headers`-shaped, not `_headings`-shaped. +- #460's reproduction, red before and green after W3. +- The full mutation set, three trees: BASE `origin/main`, the first revision of + this row, and the revision that lands. Table in Risks; harness reproduced in + the PR body. +- Link-extraction census at merge `fdbc8ae6`: 4,170 raw targets, 4,163 after + the strip, the 7 losses enumerated in Risks, against 4,162 for the loose fence + rule, whose extra loss is the live link. +- `check-pr-size.py`'s own red-before/green-after harness, for both checkers. +- **CI on this PR: a red `windows-msvc-*` is NOT this row's.** Both lanes are + `if: github.event_name == 'pull_request'` (`ci.yml:640`), so the lane + `scripts/main-baseline.py` reads never runs them, and it reports main GREEN + while they fail on every PR that reaches them. `main` does not compile under + MSVC: `tests/vt/test_cpu_isa_x86.cpp` lacks ``, and MSVC's + `` does not supply it transitively. Reproduced byte-for-byte on + `row/ENG-RELEASE-WINDOWS` @`673c2f3d` and here @`104d3f36`: same file, same + `__msvc_string_view.hpp(550,23) error C2027`, same target, same failing step. + This PR touches no `src/`, `include/`, `tests/`, `cmake/` or `.ps1` path, so + it cannot be the cause. Filed as + [#503](https://github.com/mudler/vllm.cpp/issues/503). + +## Stop conditions + +- **Stop if** removing `max_chars` cannot be shown to leave the append-log class + caught. The regrowth guard is the condition of the removal, not a nicety. + **This condition failed on the first revision and was met on the second.** + M7 and M8, both append-log-shaped, survived the dated-heading guard because it + fires on a *date*; §3a is what discharges the condition, and the three-tree + mutation table in Risks is the evidence. The residual survivor M11 is the same + class and is NOT claimed as caught: it is filed as #507, deferred as W8, and + pinned by a test. +- **Stop if** the entry cap or the heading guard fires on either live page as + shipped. That would mean the replacement is a different rule, not a relocated + one, and the page would owe an edit this row is not authorised to make. +- **Stop and return `NEEDS_DECISION`** if closing #460 turns out to require + rewriting an archived link, which would break the byte-for-byte guarantee the + archive exists to give. +- **Never** raise a cap to pass, and never delete an assertion to turn a gate + green. + +## Outcome + +Pending. Filled on `DONE` with the measured before/after, what was rejected, and +why each constant is set where it is. diff --git a/scripts/check-agent-record.py b/scripts/check-agent-record.py index 65336931f..4817336c2 100644 --- a/scripts/check-agent-record.py +++ b/scripts/check-agent-record.py @@ -431,6 +431,12 @@ ) STATE_RE = re.compile(r"`(" + "|".join(re.escape(state) for state in STATES) + r")`") LINK_RE = re.compile(r"\[[^\]]*\]\(([^)]+)\)") +# Group 1 is the run of fence characters, group 2 everything after it, which is +# the INFO STRING on an opening fence and must be empty on a closing one. Both +# groups are load-bearing: see strip_code_spans for the pairing rule and for the +# live file that mis-paired without it. +FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") +INLINE_CODE_RE = re.compile(r"`+[^`\n]*`+") CLAIM_RE = re.compile(r"CLAIM-[A-Za-z0-9_.-]+") LINE_FRAGMENT_RE = re.compile(r"L(\d+)(?:-L?(\d+))?") COMMIT_RE = re.compile(r"[0-9a-f]{7,40}") @@ -591,27 +597,108 @@ def parse_claim_rows(path: Path, errors: list[str]) -> list[ClaimRow]: return rows -def link_base(source: Path, text: str) -> Path: - """Resolve migrated legacy links from their original .agents/ location.""" +def strip_code_spans(text: str) -> str: + """Blank out fenced blocks and inline code, preserving line and column count. + + A target inside a code span is NOT a link: CommonMark renders it as literal + text, so no reader can follow it and there is nothing for "every link + resolves" to be about. Before 2026-08-12 this checker validated them anyway + (#460), which meant no document in the tree could SHOW a link in sample + output, and, worse, that a docs/BENCHMARKS.md row quoting its evidence link + could not be archived into .agents/ byte-for-byte. + + THE PAIRING RULE IS COMMONMARK'S, not "the next line that looks like a + fence". A closing fence must use the OPENER'S character, be at least as + long, and carry nothing but whitespace after the marker; a line with an info + string opens a block and never closes one. Getting that wrong does not fail + safe, it INVERTS fence phase for the rest of the file: with the one + unbalanced fence this tree already has, a bare ``` at + .agents/completed/state-events/0000-00/STATE-LEGACY-000001.md:17697 was + "closed" by the ```sh at :17948, and ordinary prose at :18297 was blanked, + so a live reader-followable link stopped being validated. Measured + tree-wide, the loose rule dropped 5 targets and the CommonMark rule drops 4. + + Blanking rather than deleting preserves every line and column offset. Note + that check_links reports no line numbers today, so this buys nothing yet; it + is kept so that a caller that does report positions cannot be broken by this + function, and test_stripping_preserves_line_and_column_positions holds it. + """ + out: list[str] = [] + fence: str | None = None + fence_len = 0 + for line in text.splitlines(): + marker = FENCE_RE.match(line) + if fence is None: + # CommonMark: a backtick opening fence's info string may not contain + # a backtick, which is what keeps `` `a` and `b` `` from opening one. + if marker is not None and not ( + marker.group(1)[0] == "`" and "`" in marker.group(2) + ): + fence = marker.group(1)[0] + fence_len = len(marker.group(1)) + out.append(" " * len(line)) + continue + out.append(INLINE_CODE_RE.sub(lambda m: " " * len(m.group(0)), line)) + continue + if ( + marker is not None + and marker.group(1)[0] == fence + and len(marker.group(1)) >= fence_len + and not marker.group(2).strip() + ): + fence = None + out.append(" " * len(line)) + return "\n".join(out) + + +def extract_links(text: str) -> list[str]: + """Return every link target a READER could follow in this document.""" + return LINK_RE.findall(strip_code_spans(text)) + + +def link_bases(source: Path, text: str) -> tuple[Path, ...]: + """Return every directory a relative link in this file may resolve from. + + Normally exactly one, the file's own directory. Two files are archives that + hold content written somewhere else and moved here verbatim, so a target in + them was authored against the ORIGINAL directory: migrated legacy + state-event payloads came from .agents/, and .agents/benchmark-record.md is + the declared archive of docs/BENCHMARKS.md (#460). + + BE PRECISE ABOUT WHAT THE SECOND BASE BUYS. It does NOT make the archived + copy clickable: a reader opening .agents/benchmark-record.md on GitHub and + clicking a docs/-relative target such as USAGE.md, BUILD.md or + bench-evidence/... gets a 404, because the browser resolves it against + .agents/. What it enforces is that the EVIDENCE STILL EXISTS in the tree + under one of the two declared bases, so archiving a row byte-for-byte cannot + silently orphan the file it points at. That is the property the archive + exists to give, and it is weaker than followability. Making the archived + copy followable means rewriting the target or recording its origin, which is + spec W5 and is deliberately not done here: rewriting a target would break + the byte-for-byte guarantee, and W5 records the origin instead. + """ if ( source.is_relative_to(AGENTS / "completed/state-events") and "" in text ): - return AGENTS - return source.parent + return (AGENTS,) + if source == AGENTS / "benchmark-record.md": + return (source.parent, ROOT / "docs") + return (source.parent,) def check_links(errors: list[str]) -> None: for source in markdown_files(): text = source.read_text(encoding="utf-8") - base = link_base(source, text) - for raw_target in LINK_RE.findall(text): + bases = link_bases(source, text) + for raw_target in extract_links(text): target = raw_target.strip().strip("<>") if not target or target.startswith(("http://", "https://", "mailto:")): continue target_path, _, fragment = target.partition("#") - resolved = (base / target_path).resolve() - if not resolved.exists(): + candidates = [(base / target_path).resolve() for base in bases] + resolved = next((c for c in candidates if c.exists()), None) + if resolved is None: errors.append(f"{source.relative_to(ROOT)}: dangling link {raw_target}") continue line_match = LINE_FRAGMENT_RE.fullmatch(fragment) diff --git a/scripts/check-public-doc-tables.py b/scripts/check-public-doc-tables.py index 3ac79f872..b9a60fca8 100755 --- a/scripts/check-public-doc-tables.py +++ b/scripts/check-public-doc-tables.py @@ -8,11 +8,16 @@ before the 2026-08-04 conversion), which is exactly what makes them unreadable to users. -This checker fails if either page loses a required user-facing section, grows -past its budget, accumulates sections or prose instead of rows, or stops -pointing at the record it relocates detail into. When BENCHMARKS.md fails -because sections accumulated, `scripts/roll-benchmark-record.py` moves them into -.agents/benchmark-record.md. +This checker fails if either page loses a required user-facing section, appends +a section or a per-attempt dated heading instead of updating a row, lets one +entry grow into a wall of prose, or stops pointing at the record it relocates +detail into. When BENCHMARKS.md fails because sections accumulated, +`scripts/roll-benchmark-record.py` moves them into .agents/benchmark-record.md. + +Nothing here budgets the WHOLE FILE. Adding a measurement row must never +require deleting a row someone else owns: see the MAX_ROW_CHARS comment for the +measurement behind that, and AGENTS.md, Records, "cap the entry, never the +file". The validation logic is the pure functions `benchmarks_errors(text)` and `features_errors(text)` so they are unit-testable and mutation-testable (see @@ -22,6 +27,7 @@ from __future__ import annotations +import re import sys from pathlib import Path @@ -37,6 +43,52 @@ # Likewise shared: a single paragraph past this is narrative, not a caption. MAX_PARAGRAPH_CHARS = 700 +# THE PER-PAGE `max_chars` BUDGET WAS REMOVED 2026-08-12 +# (ENG-RECORD-CONFLICT-SURFACES, #460). It was 45,000 for the scoreboard and +# 30,000 for the feature matrix, and both pages sat against it: BENCHMARKS.md +# measured 44,795 of 45,000 and FEATURES.md 29,740 of 30,000. A budget on a +# SHARED file makes every addition an eviction of somebody else's row, which is +# the corollary AGENTS.md Records states outright ("cap the entry, never the +# file"), and 87308dea already removed the two sibling budgets on this argument +# under #364: MAX_CHARS in check-now-current.py and the `chars` key of +# STATUS_RATCHET below. This was the third, left standing in that pass. +# +# The measured consequences, over the last 25 commits touching BENCHMARKS.md: +# free space ranged from 421 characters down to MINUS SEVEN; row count fell 165 +# to 162 while the project gained measurements; two commits (93613baa, 887e04ff) +# exist for no purpose but to pay rent; and 04b2b9fa is a CLEAN automatic merge +# that landed the page at 45,007 chars, over the cap, because two PRs each paid +# by evicting a different row and the three-way merge applied both additions and +# neither eviction. A gate whose success mode is unsafe is worse than no gate. +# +# MAX_ROW_CHARS and DATED_HEADING_RE carry the obligation between them, and both +# are ENTRY-scoped, so an author bounds their own row and never anyone else's. + +# One table row is one ENTRY, and this is its budget. Set from the shipped +# pages: the longest live row is 520 chars on BENCHMARKS.md and 580 on +# FEATURES.md. It is a real constraint and a tighter one than the cell cap it +# joins, which alone permits a five-column row of 1,100 characters. +MAX_ROW_CHARS = 600 + +# REGROWTH GUARD. What actually bloated BENCHMARKS.md to 11,405 lines was +# PER-ATTEMPT sections, appended one checkpoint at a time, and the page's own +# archive records their shape: 287 of the 310 sections already rolled into +# .agents/benchmark-record.md name a DATE in the heading. Zero of the 36 live +# headings across the two public pages do (18 each; reproduce with _headings). +# So a dated heading is the append-log entry, and it fails at the FIRST one. +# +# WHAT THIS GUARD IS AND IS NOT TIGHTER THAN. It is strictly tighter than the +# canonical-section allowlist it joins, which runs over _h2_headers and so +# matches "## " only: an appended DATED "### " subsection was rejected by +# nothing but the retired character budget, and is now rejected here at every +# depth. It is NOT tighter than the retired byte cap in general, and an earlier +# revision of this comment claimed that it was. An UNDATED appended subsection +# passes this guard by construction, because the guard fires on a date. What +# bounds that case is the paragraph budget, once _prose_paragraphs counts the +# bulleted forensics such a section carries: see the fold recorded there, and +# the two mutations that measured the gap before it existed. +DATED_HEADING_RE = re.compile(r"\b(?:19|20)\d{2}-\d{2}-\d{2}\b") + # The scoreboard must point at the record, and the record must exist, otherwise # "move it to the archive" silently loses the evidence. RECORD_LINK = ".agents/benchmark-record.md" @@ -49,6 +101,23 @@ class PageRules: headroom for genuinely new subjects but not for accumulated entries. That is the whole mechanism: growth that is new ROWS passes, growth that is new SECTIONS or new PROSE fails. + + NONE of these limits is a budget on the whole file: see the MAX_ROW_CHARS + comment above for why the per-page `max_chars` key was removed on + 2026-08-12. A limit here either counts a QUALITY defect (sections, + paragraphs) or bounds ONE entry, so adding a ROW never requires deleting a + row someone else owns. + + THAT SENTENCE IS TRUE OF ROWS, AND ONLY OF ROWS. `max_h2_sections` and + `max_prose_paragraphs` are still whole-page COUNTS, and both live pages sit + on the paragraph one: docs/BENCHMARKS.md measures 35 of 35 and + docs/FEATURES.md 21 of 21. So adding a PARAGRAPH does still cost somebody's + paragraph. That is deliberate rather than overlooked. Rows are the growth + mode of a keyed table and prose is the decay mode this checker exists to + stop, which is the same argument #364 used to keep `long_paragraphs` in + STATUS_RATCHET while deleting `chars`. It is a real cost, not a free one, + and it is recorded here so the next author meets it in the docstring instead + of in CI. """ def __init__( @@ -59,7 +128,6 @@ def __init__( required_sections: tuple[tuple[str, tuple[str, ...]], ...], max_h2_sections: int, max_prose_paragraphs: int, - max_chars: int, min_table_rows: int, required_links: tuple[str, ...], canonical_sections: tuple[str, ...] | None = None, @@ -69,7 +137,6 @@ def __init__( self.required_sections = required_sections self.max_h2_sections = max_h2_sections self.max_prose_paragraphs = max_prose_paragraphs - self.max_chars = max_chars self.min_table_rows = min_table_rows self.required_links = required_links # When set, EVERY H2 must match one of these substrings. This is the @@ -101,7 +168,6 @@ def is_canonical(self, title: str) -> bool: ), max_h2_sections=16, max_prose_paragraphs=35, - max_chars=45000, min_table_rows=40, required_links=(RECORD_LINK,), # The scoreboard's sections ARE its schema: a reference engine, a resource @@ -135,8 +201,12 @@ def is_canonical(self, title: str) -> bool: ("How to read this page", ("how to read", "legend", "reading")), ), max_h2_sections=20, - max_prose_paragraphs=20, - max_chars=30000, + # 21, not 20, since 2026-08-12: _prose_paragraphs now folds list items in, + # and this page carries one (the C-ABI capability note). Re-baselined to + # what the page measures under the new definition, exactly as the old number + # was pinned to what it measured under the old one. Nothing was widened: the + # population counted grew, the headroom did not. + max_prose_paragraphs=21, min_table_rows=60, required_links=("STATUS.md", "BENCHMARKS.md"), ) @@ -150,11 +220,37 @@ def _is_separator_row(cells: list[str]) -> bool: return all(set(cell) <= set("-: ") for cell in cells) -def _prose_paragraphs(text: str) -> list[tuple[int, str]]: - """Yield (start_line, paragraph) for prose only. +LIST_ITEM_RE = re.compile(r"^(?:[-*+](?:\s|$)|\d+[.)]\s)") + - Fenced code blocks, tables, headings, and list items are excluded: the rule - targets the narrative paragraph, not legitimate tables or code samples. +def _prose_paragraphs(text: str) -> list[tuple[int, str]]: + """Yield (start_line, paragraph) for narrative prose. + + Fenced code blocks, tables and headings are excluded: the rule targets the + narrative paragraph, not legitimate tables or code samples. + + LIST ITEMS AND BLOCKQUOTE LINES ARE PROSE HERE, folded into the paragraph + that runs through them, and that is a 2026-08-12 repair, not the original + behaviour (#460, review finding F1). Excluding them meant a bulleted or + quoted wall was counted by NOTHING: not this budget, not MAX_PARAGRAPH_CHARS, + not MAX_CELL_CHARS, not MAX_ROW_CHARS, and not the dated-heading guard, which + only sees a DATE. Two mutations proved it on the real checker: 3,000 appended + bullet lines took docs/BENCHMARKS.md to 113,833 characters and 500 appended + UNDATED "### Attempt N" sections with bulleted forensics took it to 117,222, + and both reported no errors. Folding makes a contiguous run one paragraph, so + the wall trips MAX_PARAGRAPH_CHARS, and a run per section trips the paragraph + COUNT. Neither live page gains a paragraph from the fold except + docs/FEATURES.md, which gains its one list item; see max_prose_paragraphs. + + KNOWN RESIDUE, deliberately left and filed, not silently kept: a line + beginning with EMPHASIS rather than a list marker ("**Protocol.** ...") is + still excluded, because it starts with "*". That is the accident this + exclusion always was, and closing it turns four paragraphs already shipped on + docs/BENCHMARKS.md red at 717, 719, 748 and 1,084 characters against + MAX_PARAGRAPH_CHARS = 700. Fixing it therefore owes an edit to a page #481 + holds open, so it takes its own spec and its own red-before rather than + riding along here: issue #507, spec W8. A prose wall led by "**bold**" is + UNBOUNDED until then. """ paragraphs: list[tuple[int, str]] = [] current: list[str] = [] @@ -175,12 +271,14 @@ def flush() -> None: continue if in_fence: continue + # "-", "+" and ">" are gone from this list entirely, which IS the fold: + # a list item or a quoted line now joins the paragraph running through + # it. Only "|", "#" and an emphasis-lead "*" still break a paragraph, + # and the last of those is the residue documented above, not a rule. is_prose = bool(stripped) and not ( stripped.startswith("|") or stripped.startswith("#") - or stripped.startswith("-") - or stripped.startswith("*") - or stripped.startswith(">") + or (stripped.startswith("*") and not LIST_ITEM_RE.match(stripped)) ) if is_prose: if not current: @@ -192,9 +290,13 @@ def flush() -> None: return paragraphs -def _table_rows(text: str) -> list[tuple[int, list[str]]]: - """Yield (line_number, cells) for every non-separator table row.""" - rows: list[tuple[int, list[str]]] = [] +def _table_rows(text: str) -> list[tuple[int, list[str], str]]: + """Yield (line_number, cells, raw_row) for every non-separator table row. + + The raw row travels with the cells because MAX_ROW_CHARS bounds the ENTRY, + which is the whole row, not any one of its cells. + """ + rows: list[tuple[int, list[str], str]] = [] in_fence = False for lineno, raw in enumerate(text.splitlines(), start=1): stripped = raw.strip() @@ -206,10 +308,32 @@ def _table_rows(text: str) -> list[tuple[int, list[str]]]: if stripped.startswith("|") and stripped.endswith("|"): cells = [c.strip() for c in stripped.strip("|").split("|")] if not _is_separator_row(cells): - rows.append((lineno, cells)) + rows.append((lineno, cells, stripped)) return rows +def _headings(text: str) -> list[tuple[int, str]]: + """Yield (line_number, title) for every ATX heading, fences excluded. + + Every DEPTH, unlike _h2_headers: the regrowth guard has to see the "### " + subsections the canonical-section allowlist never covered. + """ + headings: list[tuple[int, str]] = [] + in_fence = False + for lineno, raw in enumerate(text.splitlines(), start=1): + stripped = raw.strip() + if stripped.startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + if stripped.startswith("#"): + title = stripped.lstrip("#").strip() + if title: + headings.append((lineno, title)) + return headings + + def page_errors(text: str, rules: PageRules) -> list[str]: """Return human-readable problems with one keyed-table page.""" errors: list[str] = [] @@ -248,12 +372,18 @@ def page_errors(text: str, rules: PageRules) -> list[str]: "them (use commas, periods, parentheses, or hyphens)" ) - if len(text) > rules.max_chars: - errors.append( - f"{rules.name} is {len(text)} chars, over the {rules.max_chars}-char " - f"{rules.kind} budget; per-attempt detail belongs in the record and " - "the lifecycle surfaces, not here" - ) + # REGROWTH GUARD (see DATED_HEADING_RE). A dated heading is what a + # per-attempt entry looks like on this page, at any depth, and it fails at + # the FIRST one rather than once a count budget fills up. + for lineno, title in _headings(text): + if DATED_HEADING_RE.search(title): + errors.append( + f"line {lineno}: heading {title[:60]!r} names a date, so it is " + f"a PER-ATTEMPT entry; {rules.name} is a KEYED TABLE and a " + "checkpoint updates its ROW. Run " + "scripts/roll-benchmark-record.py --apply to move it verbatim " + f"into {RECORD_LINK}, and put the date in the row or the prose" + ) for link in rules.required_links: if link not in text: @@ -285,7 +415,7 @@ def page_errors(text: str, rules: PageRules) -> list[str]: f"{rules.min_table_rows} minimum; the {rules.kind} carries its " "content as keyed table rows, not as prose" ) - for lineno, cells in rows: + for lineno, cells, raw_row in rows: for cell in cells: if len(cell) > MAX_CELL_CHARS: errors.append( @@ -293,6 +423,16 @@ def page_errors(text: str, rules: PageRules) -> list[str]: f"{MAX_CELL_CHARS} (wall-of-prose smell; move the detail " f"to {RECORD_LINK})" ) + # THE ENTRY CAP (see MAX_ROW_CHARS). This bounds YOUR row, so the cost + # of a new measurement is paid by shortening it, never by deleting a + # row someone else owns. + if len(raw_row) > MAX_ROW_CHARS: + errors.append( + f"line {lineno}: table row of {len(raw_row)} chars exceeds the " + f"{MAX_ROW_CHARS}-char ENTRY budget; shorten THIS row and move " + f"its forensics to {RECORD_LINK}. The page itself has no budget " + "and never needs an unrelated row deleted to make space" + ) return errors @@ -342,7 +482,15 @@ def features_errors(text: str) -> list[str]: STATUS = ROOT / "docs/STATUS.md" STATUS_RATCHET = { "h2_sections": 11, - "long_paragraphs": 82, + # 75, down from 82 on 2026-08-12. Not a compaction of the page: + # _prose_paragraphs now folds list items and blockquote lines into the + # paragraph running through them, so docs/STATUS.md's 29 list items join + # neighbouring paragraphs instead of splitting them, and the page measures + # 75 long paragraphs where it measured 82. A ratchet is pinned to what the + # page measures, so it follows the measurement DOWN in the same change that + # moved it. Leaving 82 would have banked 7 units of slack this row did not + # earn. + "long_paragraphs": 75, "oversized_cells": 44, } STATUS_REQUIRED = ( @@ -373,7 +521,7 @@ def status_errors(text: str) -> list[str]: ), "oversized_cells": sum( 1 - for _, cells in _table_rows(text) + for _, cells, _raw in _table_rows(text) for cell in cells if len(cell) > MAX_CELL_CHARS ), diff --git a/tests/scripts/test_agent_record.py b/tests/scripts/test_agent_record.py index 8c9072cdc..bfbb37563 100644 --- a/tests/scripts/test_agent_record.py +++ b/tests/scripts/test_agent_record.py @@ -315,12 +315,180 @@ class MigratedLegacyLinks(unittest.TestCase): def test_legacy_payload_keeps_original_agents_relative_link_base(self) -> None: source = ROOT / ".agents/completed/state-events/0000-00/STATE-LEGACY-000001.md" text = "\n[spec](specs/example.md)" - self.assertEqual(agent_record.link_base(source, text), ROOT / ".agents") + self.assertEqual(agent_record.link_bases(source, text), (ROOT / ".agents",)) def test_post_cutover_event_links_remain_event_relative(self) -> None: source = ROOT / ".agents/completed/state-events/2026-08/STATE-20260808T120000-001.md" - self.assertEqual(agent_record.link_base(source, "[local](note.md)"), source.parent) + self.assertEqual( + agent_record.link_bases(source, "[local](note.md)"), (source.parent,) + ) + + +class LinkExtraction(unittest.TestCase): + """#460: what the checker calls a link must be a link a READER can follow. + + docs/BENCHMARKS.md is compacted by MOVING a superseded row into + .agents/benchmark-record.md byte-for-byte. Before this, any row carrying a + docs/-relative evidence link dangled the moment it was archived, whether it + was quoted inside a fence or moved as live markdown, so the documented + payment mechanism did not work for exactly the rows that carry evidence. + """ + def test_fenced_link_is_not_extracted(self) -> None: + text = "```text\n| row | [evidence](bench-evidence/x.md) |\n```\n" + self.assertEqual(agent_record.extract_links(text), []) + + def test_tilde_fenced_link_is_not_extracted(self) -> None: + text = "~~~console\n$ see [evidence](bench-evidence/x.md)\n~~~\n" + self.assertEqual(agent_record.extract_links(text), []) + + def test_inline_code_link_is_not_extracted(self) -> None: + self.assertEqual( + agent_record.extract_links("write `[label](target.md)` to link\n"), [] + ) + + def test_live_link_is_still_extracted(self) -> None: + self.assertEqual( + agent_record.extract_links("see [spec](specs/example.md) now\n"), + ["specs/example.md"], + ) + + def test_a_backticked_label_is_still_a_link(self) -> None: + # The overwhelmingly common form in this tree: [`name`](path). + self.assertEqual( + agent_record.extract_links("[`workflow.md`](workflow.md)\n"), + ["workflow.md"], + ) + + def test_link_after_a_closed_fence_is_still_extracted(self) -> None: + text = "```sh\nrun [x](nope.md)\n```\n\nreal [spec](specs/example.md)\n" + self.assertEqual(agent_record.extract_links(text), ["specs/example.md"]) + + def test_link_beside_an_inline_span_is_still_extracted(self) -> None: + text = "`VT_FLAG=1` and [spec](specs/example.md)\n" + self.assertEqual(agent_record.extract_links(text), ["specs/example.md"]) + + def test_a_fence_with_an_INFO_STRING_does_not_close_a_block(self) -> None: + """The F2 defect, minimised. + + CommonMark: a closing fence carries no info string. Treating ```sh as a + close INVERTS fence phase for the rest of the file, so prose after the + next real fence gets blanked and text inside a block gets validated. + The two assertions are the two halves of the inversion. First: with no + real close, everything after ```sh is still inside. Second, and this is + the half that bit the tree: the BARE fence is the close, so the link + below it is live and the one above it is not. Under the loose rule both + answers were exactly backwards. + """ + self.assertEqual( + agent_record.extract_links("```\nopen\n```sh\ninside [a](nope-a.md)\n"), + [], + ) + self.assertEqual( + agent_record.extract_links( + "```\nopen\n```sh\ninside [a](nope-a.md)\n```\n" + "after [b](specs/example.md)\n" + ), + ["specs/example.md"], + ) + + def test_a_closing_fence_must_match_the_opener(self) -> None: + # Wrong character, then too short: neither closes, so the link after it + # is still inside the block. + for closer in ("~~~", "``"): + with self.subTest(closer=closer): + text = f"````\ncode\n{closer}\n[x](nope.md)\n" + self.assertEqual(agent_record.extract_links(text), []) + + def test_a_LONGER_closing_fence_does_close(self) -> None: + text = "```\ncode\n`````\n\nreal [spec](specs/example.md)\n" + self.assertEqual(agent_record.extract_links(text), ["specs/example.md"]) + + def test_prose_two_lines_below_a_closed_fence_is_still_scanned(self) -> None: + """The live case, in the file that actually mis-paired. + + .agents/completed/state-events/0000-00/STATE-LEGACY-000001.md has an + unclosed fence at :17697. Under the loose rule the ```sh at :17948 + "closed" it, phase inverted, and the ordinary prose link at :18297 was + blanked: a reader-followable link silently stopped being validated. + """ + source = ROOT / ".agents/completed/state-events/0000-00/STATE-LEGACY-000001.md" + targets = agent_record.extract_links(source.read_text(encoding="utf-8")) + self.assertIn("specs/cpu-llamacpp-floor-remeasure-2026-07-22.md", targets) + + def test_a_link_straddled_by_two_INLINE_SPANS_is_not_extracted(self) -> None: + """F5: `[`name`](path)` is four backticks, and not a link. + + CommonMark reads it as the code span `[`, the literal text name, and the + code span `](path)`, so there is no link and the checker agrees. It + earns a test because the effect is to HIDE a target that does not exist: + this row's own spec carried the form and hid `path`. An author who wants + to SHOW the form writes it with a double-backtick delimiter, which is + also code and also correctly skipped; an author who wants a real link + writes it without the outer pair, which still resolves. + """ + self.assertEqual(agent_record.extract_links("`[`name`](path)`\n"), []) + self.assertEqual(agent_record.extract_links("`` [`name`](path) ``\n"), []) + self.assertEqual( + agent_record.extract_links("[`name`](specs/example.md)\n"), + ["specs/example.md"], + ) + + def test_stripping_preserves_line_and_column_positions(self) -> None: + # Blanked, not deleted, so every line and column offset survives. NOT + # evidence of anything today: check_links reports no line numbers at + # all. Held so a caller that does report them cannot be broken here. + text = "a\n```\nbbbb\n```\nc `dd` e\n" + stripped = agent_record.strip_code_spans(text) + self.assertEqual(len(stripped.splitlines()), len(text.splitlines())) + for original, blanked in zip(text.splitlines(), stripped.splitlines()): + self.assertEqual(len(original), len(blanked)) + + def test_the_benchmark_record_also_resolves_from_docs(self) -> None: + # It is the declared archive of docs/BENCHMARKS.md, so a row moved into + # it verbatim keeps its docs/-relative evidence link resolvable. + source = ROOT / ".agents/benchmark-record.md" + self.assertEqual( + agent_record.link_bases(source, ""), (source.parent, ROOT / "docs") + ) + + def test_an_archived_row_with_a_docs_relative_link_is_accepted(self) -> None: + record = ROOT / ".agents/benchmark-record.md" + original = record.read_text(encoding="utf-8") + moved = ( + "\n## Assembly vs compiler SDOT\n\n| Result | Evidence |\n|---|---|\n" + "| leaf wall | [assembly evidence]" + "(bench-evidence/rpi5-a76-q8-dot-20260806.md) |\n" + ) + errors: list[str] = [] + try: + record.write_text(original + moved, encoding="utf-8") + agent_record.check_links(errors) + finally: + record.write_text(original, encoding="utf-8") + self.assertEqual( + [e for e in errors if "rpi5-a76-q8-dot" in e], [], errors[:5] + ) + + def test_an_archived_row_with_a_MISSING_link_still_dangles(self) -> None: + # The second base is a base, not an amnesty. + record = ROOT / ".agents/benchmark-record.md" + original = record.read_text(encoding="utf-8") + errors: list[str] = [] + try: + record.write_text( + original + "\n[gone](bench-evidence/no-such-file-20260812.md)\n", + encoding="utf-8", + ) + agent_record.check_links(errors) + finally: + record.write_text(original, encoding="utf-8") + require(errors, r"dangling link bench-evidence/no-such-file-20260812\.md") + + def test_the_tree_has_no_dangling_link(self) -> None: + errors: list[str] = [] + agent_record.check_links(errors) + self.assertEqual(errors, []) diff --git a/tests/scripts/test_check_public_doc_tables.py b/tests/scripts/test_check_public_doc_tables.py index 030232e91..a350f4ffa 100644 --- a/tests/scripts/test_check_public_doc_tables.py +++ b/tests/scripts/test_check_public_doc_tables.py @@ -33,6 +33,13 @@ def _load(name: str, relative: str): roll_record = _load("roll_record", "scripts/roll-benchmark-record.py") +# The whole-file character budget retired on 2026-08-12 (#460). Used by exactly +# one test, on a SYNTHETIC page that test builds itself, so that the "a row no +# longer has to evict one" claim can name the size it is talking about. It must +# never be compared against a real page: that would store a measurement of one +# file inside another, which is the defect this row removed. +RETIRED_PAGE_BUDGET = 45000 + # A minimal scoreboard that satisfies every rule, used as the mutation baseline. # MIN_TABLE_ROWS forces a real table, so the baseline carries one. _ROWS = "\n".join(f"| Model {i} | {i}.0x |" for i in range(60)) @@ -164,7 +171,11 @@ def test_release_projection_fits_after_current_main_merge(self) -> None: ).stdout merged = _project_release_rows(main, text) self.assertEqual(_release_projection_errors(merged), []) - self.assertLess(len(merged), doc_tables.BENCHMARKS_RULES.max_chars) + # The whole-page size assertion this line used to carry went with + # `max_chars` on 2026-08-12 (#460). What has to hold after a merge is + # that every ENTRY is in budget, which the full check below asserts. + for row in _release_rows(merged): + self.assertLessEqual(len(row), doc_tables.MAX_ROW_CHARS) self.assertEqual(doc_tables.benchmarks_errors(merged), []) def test_shipped_record_exists(self) -> None: @@ -224,10 +235,269 @@ def test_em_dash_fails(self) -> None: errors = doc_tables.benchmarks_errors(mutated) self.assertTrue(any("em-dash" in e for e in errors), errors) - def test_oversized_page_fails(self) -> None: - mutated = VALID + "\n" + ("- a filler bullet line\n" * 3000) + # THE PAGE HAS NO SIZE BUDGET as of 2026-08-12 (#460), so the test that + # used to sit here, asserting a 45,000-char page fails, is gone. It is + # replaced, not deleted: the obligation it carried moved to the entry cap + # and the regrowth guard below, and the reason it had to move is + # test_a_new_row_costs_no_eviction, which the old rule made impossible. + + def test_a_new_row_costs_no_eviction(self) -> None: + """The acceptance test for #460: a measurement row lands on its own. + + A page already at the retired 45,000-char budget gains one row and + stays valid. Under the old rule this required deleting somebody else's + row, and a clean merge of two such payments landed the real page at + 45,007 chars (04b2b9fa), applying both additions and neither eviction. + """ + filler = "\n".join( + f"| Subject {i:04d} | {'measured, byte exact, ' * 6}{i}.0x |" + for i in range(300) + ) + big = VALID.replace(_ROWS, _ROWS + "\n" + filler) + self.assertGreater(len(big), RETIRED_PAGE_BUDGET) + self.assertEqual(doc_tables.benchmarks_errors(big), []) + + added = "| Qwen3.6-35B canonical regrid | PENDING, stale grid |" + grown = big.replace(filler, filler + "\n" + added) + self.assertEqual(doc_tables.benchmarks_errors(grown), []) + self.assertEqual(len(grown), len(big) + len(added) + 1) + + def test_the_shipped_page_can_accept_the_next_measurement_row(self) -> None: + """#460 on the REAL page: add a row, keep every existing row. + + The row added here is the 35B canonical regrid PR #481 records as owed. + It is added and dropped again inside this test, so the page is not + edited; what is asserted is that the surface would accept it. + + The row is inserted DIRECTLY BELOW the last row of the Open gaps table, + with no blank line between, so markdown renders it inside that table + rather than it merely satisfying _table_rows. An earlier revision + separated it by blank lines, which proved the checker claim but not the + placement. + + Nothing here compares the grown page against the retired 45,000-char + budget. That assertion would store a measurement of docs/BENCHMARKS.md + inside this test file, which is exactly what AGENTS.md, Records, forbids + and what #460 removed: it had 173 characters of margin, so any PR + compacting the page by more than that turned it red in a file it does + not own, and 887e04ff shrank the page by 280. The above-the-budget point + is made by test_a_new_row_costs_no_eviction, on a synthetic page that + test builds itself. + """ + text = (ROOT / "docs/BENCHMARKS.md").read_text(encoding="utf-8") + self.assertEqual(doc_tables.benchmarks_errors(text), []) + owed = ( + "| Qwen3.6-35B canonical regrid (`ROAD-V1-A`, #378) | " + "**STALE, no current number.** The canonical 0.918x-0.972x grid " + "predates 136 src commits, one worth +2.05% at c8; holding vLLM " + "fixed only IMPLIES c8 ~0.937, an estimate | Recapture the 6-point " + "c1-c32 grid on current main, both arms in one session, before any " + "35B residual is attributed |" + ) + # The median Open gaps row on this page is 328 chars and 205 were free, + # so a REALISTIC row was unaffordable, not just a pathological one. + self.assertGreater(len(owed), 328) + self.assertLessEqual(len(owed), doc_tables.MAX_ROW_CHARS) + + before, sep, after = text.partition("\n## Reproduce") + self.assertEqual(sep, "\n## Reproduce") + open_gaps = before.rstrip("\n") + self.assertTrue( + open_gaps.endswith("|"), + "the last thing before ## Reproduce is expected to be the final Open " + "gaps table row; if it is not, this test is inserting the owed row " + "somewhere markdown will not render it in that table", + ) + grown = f"{open_gaps}\n{owed}\n{sep}{after}" + self.assertNotEqual(grown, text) + self.assertIn(f"|\n{owed}\n", grown) + self.assertEqual(doc_tables.benchmarks_errors(grown), []) + # Nothing was evicted to make room. + for _, _, row in doc_tables._table_rows(text): + self.assertIn(row, grown) + + def test_oversized_row_fails(self) -> None: + # Cap the ENTRY: one row past budget is rejected, and the fix is to + # shorten THAT row. + wide = "| Subject | " + " | ".join(["x" * 200] * 4) + " |" + mutated = VALID.replace("| Thing | Pending |", wide) + errors = doc_tables.benchmarks_errors(mutated) + self.assertTrue(any("ENTRY budget" in e for e in errors), errors) + + def test_the_row_cap_is_not_subsumed_by_the_cell_cap(self) -> None: + # Every cell legal, the row illegal: without the row cap a five-column + # row of 1,100 chars passes. + cells = " | ".join(["y" * (doc_tables.MAX_CELL_CHARS - 1)] * 4) + mutated = VALID.replace("| Thing | Pending |", f"| {cells} |") + errors = doc_tables.benchmarks_errors(mutated) + self.assertEqual([e for e in errors if "wall-of-prose" in e], []) + self.assertTrue(any("ENTRY budget" in e for e in errors), errors) + + def test_a_row_at_the_cap_is_allowed(self) -> None: + # Exactly at MAX_ROW_CHARS, with every cell inside MAX_CELL_CHARS. + # "| a | b | c | d |" costs the four cells plus 13 characters. + budget = doc_tables.MAX_ROW_CHARS - 13 + widths = [budget // 4] * 4 + widths[0] += budget - sum(widths) + cells = ["z" * w for w in widths] + self.assertTrue(all(w <= doc_tables.MAX_CELL_CHARS for w in widths)) + row = "| " + " | ".join(cells) + " |" + self.assertEqual(len(row), doc_tables.MAX_ROW_CHARS) + mutated = VALID.replace("| Thing | Pending |", row) + self.assertEqual(doc_tables.benchmarks_errors(mutated), []) + + def test_a_dated_h2_is_rejected(self) -> None: + mutated = VALID + "\n## vLLM re-grid 2026-08-12\n\nA paragraph.\n" errors = doc_tables.benchmarks_errors(mutated) - self.assertTrue(any("scoreboard budget" in e for e in errors), errors) + self.assertTrue(any("PER-ATTEMPT entry" in e for e in errors), errors) + + def test_a_dated_h3_is_rejected(self) -> None: + # The hole the retired byte cap was silently covering: the canonical + # allowlist runs over "## " only, so this was caught by nothing else. + mutated = VALID.replace( + "## How we measure", + "### Qwen3.6-27B by concurrency, 2026-08-12 rerun\n\n" + "| Point | Ratio |\n|---|---|\n| c1 | 0.9x |\n\n## How we measure", + ) + errors = doc_tables.benchmarks_errors(mutated) + self.assertTrue(any("PER-ATTEMPT entry" in e for e in errors), errors) + self.assertTrue(any("names a date" in e for e in errors), errors) + + def test_a_dated_h3_is_rejected_on_the_feature_matrix_too(self) -> None: + mutated = VALID_FEATURES + "\n### Coverage sweep 2026-08-12\n\nText.\n" + errors = doc_tables.features_errors(mutated) + self.assertTrue(any("PER-ATTEMPT entry" in e for e in errors), errors) + + def test_a_new_subject_subsection_is_allowed(self) -> None: + # The guard is not a section freeze: a genuinely new subject passes. + mutated = VALID.replace( + "## How we measure", + "### Laguna-S-2.1 (NVFP4)\n\n" + "| Point | Ratio |\n|---|---|\n| c1 | 1.03x |\n\n## How we measure", + ) + self.assertEqual(doc_tables.benchmarks_errors(mutated), []) + + def test_a_date_inside_a_fence_is_not_a_heading(self) -> None: + # Sample output that happens to contain a heading-shaped line is not a + # section, so the regrowth guard must not fire on it. + mutated = VALID.replace( + "vllm-bench run", "vllm-bench run\n### historical 2026-08-04 output" + ) + self.assertEqual(doc_tables.benchmarks_errors(mutated), []) + + def test_a_dated_row_is_still_allowed(self) -> None: + # The date belongs in the ROW. Only the heading form is the append log. + mutated = VALID.replace( + "| Thing | Pending |", "| Thing | Pending, captured 2026-08-12 |" + ) + self.assertEqual(doc_tables.benchmarks_errors(mutated), []) + + def test_the_shipped_pages_carry_no_dated_heading(self) -> None: + for page in ("docs/BENCHMARKS.md", "docs/FEATURES.md"): + text = (ROOT / page).read_text(encoding="utf-8") + dated = [ + title + for _, title in doc_tables._headings(text) + if doc_tables.DATED_HEADING_RE.search(title) + ] + self.assertEqual(dated, [], page) + + def test_no_page_carries_a_whole_file_size_budget(self) -> None: + # The invariant behind #460, held as a rule rather than as a habit: a + # budget on a shared file makes every addition an eviction. + for rules in (doc_tables.BENCHMARKS_RULES, doc_tables.FEATURES_RULES): + self.assertFalse(hasattr(rules, "max_chars"), rules.name) + self.assertNotIn("chars", doc_tables.STATUS_RATCHET) + + def test_a_wall_of_BULLETS_fails(self) -> None: + """M8: the exact mutant the retired byte cap used to catch. + + 3,000 appended bullet lines. Under `max_chars` this was rejected as + "113833 chars, over the 45000-char scoreboard budget". Between the entry + cap landing and the 2026-08-12 fold it was rejected by NOTHING: + _prose_paragraphs excluded every line starting with "-", so the wall was + outside the paragraph count, MAX_PARAGRAPH_CHARS, MAX_CELL_CHARS, + MAX_ROW_CHARS and the dated-heading guard at once. It is the append-log + class the spec's stop condition names, so it is the condition of the + removal, not a nicety. + """ + wall = "\n".join( + f"- attempt {i}: lever measured neutral, reverted, see the record" + for i in range(3000) + ) + errors = doc_tables.benchmarks_errors(VALID + "\n\n" + wall + "\n") + self.assertTrue(any("prose paragraph of" in e for e in errors), errors[:3]) + + def test_a_wall_of_BLOCKQUOTES_fails(self) -> None: + # Same channel, the other excluded prefix. + wall = "\n".join(f"> quoted forensic note {i}" for i in range(3000)) + errors = doc_tables.benchmarks_errors(VALID + "\n\n" + wall + "\n") + self.assertTrue(any("prose paragraph of" in e for e in errors), errors[:3]) + + def test_appended_UNDATED_subsections_with_bullets_fail(self) -> None: + """M7: the dated-heading guard fires on a DATE, so it never saw this. + + 500 appended UNDATED "### Attempt N" sections carrying bulleted + forensics measured 117,222 characters with the checker exiting 0 and all + 68 tests green. The heading guard cannot reach it, by construction. The + paragraph COUNT does, once the bullets are counted. + """ + appended = "\n".join( + f"### Attempt {i}\n\n- lever {i} measured neutral on a warm box\n" + f"- reverted, probe kept\n" + for i in range(500) + ) + errors = doc_tables.benchmarks_errors(VALID + "\n" + appended) + self.assertTrue(any("prose paragraphs" in e for e in errors), errors[:3]) + + def test_a_short_bullet_list_is_still_allowed(self) -> None: + # The fold is a budget, not a ban: a legitimate short list passes. + mutated = VALID.replace( + "Greedy, closed loop, three interleaved repetitions.", + "Greedy, closed loop.\n\n- three interleaved repetitions\n" + "- one flock holder\n- cold leg discarded\n", + ) + self.assertEqual(doc_tables.benchmarks_errors(mutated), []) + + def test_a_bullet_run_is_ONE_paragraph_not_many(self) -> None: + # Folding must not turn every legitimate list item into a paragraph, or + # the count budget would fire on ordinary documents instead of walls. + text = "intro line\n\n- a\n- b\n- c\n\ntail line\n" + self.assertEqual( + [para for _, para in doc_tables._prose_paragraphs(text)], + ["intro line", "- a - b - c", "tail line"], + ) + + def test_an_EMPHASIS_lead_wall_is_a_KNOWN_residue(self) -> None: + """Characterisation, not approval: issue #507, spec W8. + + A line beginning with emphasis still starts with "*" and is still + excluded, so a wall led by "**bold**" is unbounded. Closing it turns + four paragraphs already shipped on docs/BENCHMARKS.md red against + MAX_PARAGRAPH_CHARS (717, 719, 748, 1,084 chars), which owes an edit to + a page #481 holds open, so it takes its own spec and red-before. + + This test goes RED the day that lands. That is the point: the residue is + recorded where the next author will meet it, and closing it is a + deliberate act rather than a silent one. + """ + wall = "\n\n".join(f"**Attempt {i}.** " + "x " * 200 for i in range(500)) + self.assertEqual(doc_tables.benchmarks_errors(VALID + "\n\n" + wall + "\n"), []) + + def test_the_shipped_pages_sit_at_their_folded_paragraph_budget(self) -> None: + # The budgets are pinned to what the pages MEASURE under the fold, which + # is what makes re-baselining FEATURES from 20 to 21 a re-measurement + # and not slack. If either page shrinks, these numbers come down. + for page, rules, expected in ( + ("docs/BENCHMARKS.md", doc_tables.BENCHMARKS_RULES, 35), + ("docs/FEATURES.md", doc_tables.FEATURES_RULES, 21), + ): + with self.subTest(page=page): + text = (ROOT / page).read_text(encoding="utf-8") + self.assertEqual( + len(doc_tables._prose_paragraphs(text)), expected + ) + self.assertEqual(rules.max_prose_paragraphs, expected) def test_missing_record_link_fails(self) -> None: mutated = VALID.replace( @@ -358,10 +628,16 @@ def test_em_dash_fails(self) -> None: def test_the_two_pages_have_distinct_budgets(self) -> None: # A regression guard on the refactor: FEATURES must not silently - # inherit the scoreboard's looser limits. + # inherit the scoreboard's looser limits. The `max_chars` comparison + # this used to lead with went with the key itself on 2026-08-12 (#460); + # the remaining per-page limits still have to differ. + self.assertNotEqual( + doc_tables.FEATURES_RULES.max_prose_paragraphs, + doc_tables.BENCHMARKS_RULES.max_prose_paragraphs, + ) self.assertNotEqual( - doc_tables.FEATURES_RULES.max_chars, - doc_tables.BENCHMARKS_RULES.max_chars, + doc_tables.FEATURES_RULES.min_table_rows, + doc_tables.BENCHMARKS_RULES.min_table_rows, ) self.assertNotEqual( doc_tables.FEATURES_RULES.required_sections, @@ -668,7 +944,12 @@ def test_the_status_ratchet_only_ever_moves_down(self) -> None: """ ceiling = { "h2_sections": 11, - "long_paragraphs": 82, + # 75, down from 82 on 2026-08-12: _prose_paragraphs now folds list + # items in, so docs/STATUS.md measures 75 long paragraphs where it + # measured 82, and the ratchet followed the measurement DOWN in the + # same change. Lowered here in the same commit, which is what this + # test's own message demands. + "long_paragraphs": 75, "oversized_cells": 44, } self.assertEqual(