diff --git a/.bully.yml b/.bully.yml index 6c94d00..24821fa 100644 --- a/.bully.yml +++ b/.bully.yml @@ -94,3 +94,13 @@ rules: scope: ["pipeline/pipeline.py", "pipeline/analyzer.py"] severity: error pattern: "subprocess.run($$$, shell=True, $$$)" + + ruff-clean: + description: > + Python files must pass `ruff check`. --force-exclude makes ruff + respect pyproject's extend-exclude even when a single file is + passed, so test fixtures under bench/fixtures/ are skipped. + engine: script + scope: "*.py" + severity: error + script: "ruff check --force-exclude {file}" diff --git a/README.md b/README.md index 0c6591b..cf34956 100644 --- a/README.md +++ b/README.md @@ -243,3 +243,32 @@ MIT. See [LICENSE](LICENSE). ## Contributing Issues and PRs welcome. Please read [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) before participating, [SECURITY.md](SECURITY.md) for how to report vulnerabilities, and [CHANGELOG.md](CHANGELOG.md) for release notes. + +## Test Bench + +Bully ships with a local bench for watching its own speed and input-token cost over time. Two modes: + +### Mode A — fixture suite (regression trend) + +```bash +bully bench # run all bench/fixtures/, append to bench/history.jsonl +bully bench --compare # diff the last two runs +bully bench --no-tokens # skip Anthropic API call, use char-count proxy +bully bench --json # emit the raw run record on stdout +``` + +Results are written to `bench/history.jsonl`, one line per run. Commit a fresh run alongside changes that touch `pipeline/pipeline.py` to make speed/token impact visible in PRs. + +### Mode B — config cost analysis + +```bash +bully bench --config path/to/.bully.yml +``` + +Reports the input-token cost of the given config per invocation: floor tokens, per-rule marginal cost (sorted), diff scaling at 1/10/100/1000 added lines, and per-scope grouping. Useful for deciding whether a rule or rule pack earns its keep. + +### Real token counts + +Both modes use Anthropic's `messages/count_tokens` endpoint when `ANTHROPIC_API_KEY` is set and the optional `anthropic` SDK is installed (`pip install -e ".[bench]"`). Without either, both modes fall back to a `len(json.dumps(payload))` proxy and tag the output `method: proxy`. + +The bench does not make real model calls — only `count_tokens`, which is free and does not spend credits. diff --git a/bench/fixtures/01-script-only-small-diff/config.yml b/bench/fixtures/01-script-only-small-diff/config.yml new file mode 100644 index 0000000..3f4dbcd --- /dev/null +++ b/bench/fixtures/01-script-only-small-diff/config.yml @@ -0,0 +1,7 @@ +rules: + no-print: + description: Disallow bare print() in production Python code + engine: script + scope: "**/*.py" + severity: warning + script: "grep -n 'print(' {file} && exit 1 || exit 0" diff --git a/bench/fixtures/01-script-only-small-diff/fixture.json b/bench/fixtures/01-script-only-small-diff/fixture.json new file mode 100644 index 0000000..4ef6e3d --- /dev/null +++ b/bench/fixtures/01-script-only-small-diff/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "01-script-only-small-diff", + "description": "One script rule, small Python edit", + "file_path": "bench/fixtures/01-script-only-small-diff/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,3 +1,4 @@\n def main():\n+ x = 1\n return 0\n" +} diff --git a/bench/fixtures/01-script-only-small-diff/target.py b/bench/fixtures/01-script-only-small-diff/target.py new file mode 100644 index 0000000..8dca9fc --- /dev/null +++ b/bench/fixtures/01-script-only-small-diff/target.py @@ -0,0 +1,3 @@ +def main(): + x = 1 + return 0 diff --git a/bench/fixtures/02-ast-only-small-diff/config.yml b/bench/fixtures/02-ast-only-small-diff/config.yml new file mode 100644 index 0000000..4bc8943 --- /dev/null +++ b/bench/fixtures/02-ast-only-small-diff/config.yml @@ -0,0 +1,8 @@ +rules: + no-var: + description: Prefer let/const over var in TypeScript + engine: ast + scope: "**/*.ts" + severity: warning + language: ts + pattern: "var $X = $Y" diff --git a/bench/fixtures/02-ast-only-small-diff/fixture.json b/bench/fixtures/02-ast-only-small-diff/fixture.json new file mode 100644 index 0000000..d25c59a --- /dev/null +++ b/bench/fixtures/02-ast-only-small-diff/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "02-ast-only-small-diff", + "description": "One ast rule, small TypeScript edit", + "file_path": "bench/fixtures/02-ast-only-small-diff/target.ts", + "edit_type": "Edit", + "diff": "--- a/target.ts\n+++ b/target.ts\n@@ -1,2 +1,3 @@\n function main() {\n+ const x = 1;\n }\n" +} diff --git a/bench/fixtures/02-ast-only-small-diff/target.ts b/bench/fixtures/02-ast-only-small-diff/target.ts new file mode 100644 index 0000000..124eeff --- /dev/null +++ b/bench/fixtures/02-ast-only-small-diff/target.ts @@ -0,0 +1,3 @@ +function main() { + const x = 1; +} diff --git a/bench/fixtures/03-semantic-only-small-diff/config.yml b/bench/fixtures/03-semantic-only-small-diff/config.yml new file mode 100644 index 0000000..83e7462 --- /dev/null +++ b/bench/fixtures/03-semantic-only-small-diff/config.yml @@ -0,0 +1,9 @@ +rules: + no-hardcoded-secrets: + description: > + Do not hardcode API keys, tokens, passwords, or other secret credentials + in source code. Values that look like credentials should be loaded from + environment variables or a secret manager. + engine: semantic + scope: "**/*.py" + severity: error diff --git a/bench/fixtures/03-semantic-only-small-diff/fixture.json b/bench/fixtures/03-semantic-only-small-diff/fixture.json new file mode 100644 index 0000000..310e56f --- /dev/null +++ b/bench/fixtures/03-semantic-only-small-diff/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "03-semantic-only-small-diff", + "description": "One semantic rule, small Python edit (no violation)", + "file_path": "bench/fixtures/03-semantic-only-small-diff/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,2 +1,3 @@\n def main():\n+ api_url = 'https://example.com'\n pass\n" +} diff --git a/bench/fixtures/03-semantic-only-small-diff/target.py b/bench/fixtures/03-semantic-only-small-diff/target.py new file mode 100644 index 0000000..b9b377b --- /dev/null +++ b/bench/fixtures/03-semantic-only-small-diff/target.py @@ -0,0 +1,3 @@ +def main(): + api_url = 'https://example.com' + pass diff --git a/bench/fixtures/04-mixed-engines/config.yml b/bench/fixtures/04-mixed-engines/config.yml new file mode 100644 index 0000000..73dbc98 --- /dev/null +++ b/bench/fixtures/04-mixed-engines/config.yml @@ -0,0 +1,19 @@ +rules: + no-print: + description: Disallow bare print() in production Python code + engine: script + scope: "**/*.py" + severity: warning + script: "grep -n 'print(' {file} && exit 1 || exit 0" + no-eval: + description: Disallow eval() usage + engine: ast + scope: "**/*.py" + severity: error + language: python + pattern: "eval($X)" + no-hardcoded-secrets: + description: Do not hardcode API keys, tokens, or passwords. + engine: semantic + scope: "**/*.py" + severity: error diff --git a/bench/fixtures/04-mixed-engines/fixture.json b/bench/fixtures/04-mixed-engines/fixture.json new file mode 100644 index 0000000..0f55fea --- /dev/null +++ b/bench/fixtures/04-mixed-engines/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "04-mixed-engines", + "description": "One rule per engine, moderate Python edit", + "file_path": "bench/fixtures/04-mixed-engines/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,4 +1,8 @@\n import os\n \n def main():\n+ x = os.getenv('X')\n+ y = {'a': 1, 'b': 2}\n+ for k, v in y.items():\n+ pass\n return 0\n" +} diff --git a/bench/fixtures/04-mixed-engines/target.py b/bench/fixtures/04-mixed-engines/target.py new file mode 100644 index 0000000..3e3c05d --- /dev/null +++ b/bench/fixtures/04-mixed-engines/target.py @@ -0,0 +1,8 @@ +import os + +def main(): + x = os.getenv('X') + y = {'a': 1, 'b': 2} + for k, v in y.items(): + pass + return 0 diff --git a/bench/fixtures/05-big-extends-chain/config.yml b/bench/fixtures/05-big-extends-chain/config.yml new file mode 100644 index 0000000..73fecd0 --- /dev/null +++ b/bench/fixtures/05-big-extends-chain/config.yml @@ -0,0 +1,8 @@ +extends: ["./parent.yml"] +rules: + local-rule: + description: A rule defined at the leaf + engine: script + scope: "**/*.py" + severity: warning + script: "exit 0" diff --git a/bench/fixtures/05-big-extends-chain/fixture.json b/bench/fixtures/05-big-extends-chain/fixture.json new file mode 100644 index 0000000..e24cc04 --- /dev/null +++ b/bench/fixtures/05-big-extends-chain/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "05-big-extends-chain", + "description": "Three-level extends chain; stresses parser + skip walker", + "file_path": "bench/fixtures/05-big-extends-chain/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1 +1,2 @@\n+x = 1\n y = 2\n" +} diff --git a/bench/fixtures/05-big-extends-chain/grandparent.yml b/bench/fixtures/05-big-extends-chain/grandparent.yml new file mode 100644 index 0000000..68a3c39 --- /dev/null +++ b/bench/fixtures/05-big-extends-chain/grandparent.yml @@ -0,0 +1,7 @@ +rules: + grandparent-rule: + description: A rule defined three levels up the extends chain + engine: script + scope: "**/*.py" + severity: warning + script: "exit 0" diff --git a/bench/fixtures/05-big-extends-chain/parent.yml b/bench/fixtures/05-big-extends-chain/parent.yml new file mode 100644 index 0000000..11e5017 --- /dev/null +++ b/bench/fixtures/05-big-extends-chain/parent.yml @@ -0,0 +1,8 @@ +extends: ["./grandparent.yml"] +rules: + parent-rule: + description: A rule defined two levels up + engine: script + scope: "**/*.py" + severity: warning + script: "exit 0" diff --git a/bench/fixtures/05-big-extends-chain/target.py b/bench/fixtures/05-big-extends-chain/target.py new file mode 100644 index 0000000..2c18545 --- /dev/null +++ b/bench/fixtures/05-big-extends-chain/target.py @@ -0,0 +1,2 @@ +x = 1 +y = 2 diff --git a/bench/fixtures/06-many-semantic-rules/config.yml b/bench/fixtures/06-many-semantic-rules/config.yml new file mode 100644 index 0000000..863920a --- /dev/null +++ b/bench/fixtures/06-many-semantic-rules/config.yml @@ -0,0 +1,101 @@ +rules: + sem-01: + description: Do not hardcode API keys, tokens, passwords, or other secret credentials in source code. + engine: semantic + scope: "**/*.py" + severity: error + sem-02: + description: Avoid broad exception handlers that swallow errors silently. + engine: semantic + scope: "**/*.py" + severity: warning + sem-03: + description: Avoid mutable default arguments in function signatures. + engine: semantic + scope: "**/*.py" + severity: warning + sem-04: + description: Use context managers for file and resource handling. + engine: semantic + scope: "**/*.py" + severity: warning + sem-05: + description: Do not use assert for input validation in production code paths. + engine: semantic + scope: "**/*.py" + severity: error + sem-06: + description: Avoid globally-mutable state; prefer explicit dependency injection. + engine: semantic + scope: "**/*.py" + severity: warning + sem-07: + description: Do not catch Exception or BaseException without re-raising. + engine: semantic + scope: "**/*.py" + severity: warning + sem-08: + description: Prefer f-strings over .format() and %-formatting. + engine: semantic + scope: "**/*.py" + severity: warning + sem-09: + description: Do not log sensitive values (passwords, tokens, PII). + engine: semantic + scope: "**/*.py" + severity: error + sem-10: + description: Avoid running subprocess with shell=True unless the command is shlex-quoted. + engine: semantic + scope: "**/*.py" + severity: error + sem-11: + description: Prefer pathlib.Path over os.path for new code. + engine: semantic + scope: "**/*.py" + severity: warning + sem-12: + description: Do not use print() for operational logging; use the logging module. + engine: semantic + scope: "**/*.py" + severity: warning + sem-13: + description: Avoid time.sleep() in request handlers. + engine: semantic + scope: "**/*.py" + severity: warning + sem-14: + description: Prefer list/dict/set comprehensions over map+filter when it improves clarity. + engine: semantic + scope: "**/*.py" + severity: warning + sem-15: + description: Do not silently drop exceptions in background tasks. + engine: semantic + scope: "**/*.py" + severity: error + sem-16: + description: Check for file existence with Path.exists() rather than try/except FileNotFoundError when not needed. + engine: semantic + scope: "**/*.py" + severity: warning + sem-17: + description: Do not commit debugging breakpoint()/pdb.set_trace() calls. + engine: semantic + scope: "**/*.py" + severity: error + sem-18: + description: Prefer dataclasses over manually-defined __init__/__eq__/__repr__ for data-holding classes. + engine: semantic + scope: "**/*.py" + severity: warning + sem-19: + description: Use type hints on public function signatures. + engine: semantic + scope: "**/*.py" + severity: warning + sem-20: + description: Do not hard-code environment-specific paths; use configuration. + engine: semantic + scope: "**/*.py" + severity: warning diff --git a/bench/fixtures/06-many-semantic-rules/fixture.json b/bench/fixtures/06-many-semantic-rules/fixture.json new file mode 100644 index 0000000..773bbde --- /dev/null +++ b/bench/fixtures/06-many-semantic-rules/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "06-many-semantic-rules", + "description": "20 semantic rules, small Python edit; stresses payload size", + "file_path": "bench/fixtures/06-many-semantic-rules/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,2 +1,3 @@\n import os\n+path = os.path.join('/tmp', 'f')\n x = 1\n" +} diff --git a/bench/fixtures/06-many-semantic-rules/target.py b/bench/fixtures/06-many-semantic-rules/target.py new file mode 100644 index 0000000..627aaf7 --- /dev/null +++ b/bench/fixtures/06-many-semantic-rules/target.py @@ -0,0 +1,3 @@ +import os +path = os.path.join('/tmp', 'f') +x = 1 diff --git a/bench/fixtures/07-large-diff/config.yml b/bench/fixtures/07-large-diff/config.yml new file mode 100644 index 0000000..0a590e5 --- /dev/null +++ b/bench/fixtures/07-large-diff/config.yml @@ -0,0 +1,6 @@ +rules: + no-hardcoded-secrets: + description: Do not hardcode API keys, tokens, or passwords. + engine: semantic + scope: "**/*.py" + severity: error diff --git a/bench/fixtures/07-large-diff/fixture.json b/bench/fixtures/07-large-diff/fixture.json new file mode 100644 index 0000000..9ecfe18 --- /dev/null +++ b/bench/fixtures/07-large-diff/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "07-large-diff", + "description": "500-line diff, one semantic rule; scales semantic payload", + "file_path": "bench/fixtures/07-large-diff/target.py", + "edit_type": "Edit", + "diff": "--- a/bench/fixtures/07-large-diff/target.py\n+++ b/bench/fixtures/07-large-diff/target.py\n@@ -1 +1,501 @@\n def main():\n+ line_0 = 0\n+ line_1 = 1\n+ line_2 = 2\n+ line_3 = 3\n+ line_4 = 4\n+ line_5 = 5\n+ line_6 = 6\n+ line_7 = 7\n+ line_8 = 8\n+ line_9 = 9\n+ line_10 = 10\n+ line_11 = 11\n+ line_12 = 12\n+ line_13 = 13\n+ line_14 = 14\n+ line_15 = 15\n+ line_16 = 16\n+ line_17 = 17\n+ line_18 = 18\n+ line_19 = 19\n+ line_20 = 20\n+ line_21 = 21\n+ line_22 = 22\n+ line_23 = 23\n+ line_24 = 24\n+ line_25 = 25\n+ line_26 = 26\n+ line_27 = 27\n+ line_28 = 28\n+ line_29 = 29\n+ line_30 = 30\n+ line_31 = 31\n+ line_32 = 32\n+ line_33 = 33\n+ line_34 = 34\n+ line_35 = 35\n+ line_36 = 36\n+ line_37 = 37\n+ line_38 = 38\n+ line_39 = 39\n+ line_40 = 40\n+ line_41 = 41\n+ line_42 = 42\n+ line_43 = 43\n+ line_44 = 44\n+ line_45 = 45\n+ line_46 = 46\n+ line_47 = 47\n+ line_48 = 48\n+ line_49 = 49\n+ line_50 = 50\n+ line_51 = 51\n+ line_52 = 52\n+ line_53 = 53\n+ line_54 = 54\n+ line_55 = 55\n+ line_56 = 56\n+ line_57 = 57\n+ line_58 = 58\n+ line_59 = 59\n+ line_60 = 60\n+ line_61 = 61\n+ line_62 = 62\n+ line_63 = 63\n+ line_64 = 64\n+ line_65 = 65\n+ line_66 = 66\n+ line_67 = 67\n+ line_68 = 68\n+ line_69 = 69\n+ line_70 = 70\n+ line_71 = 71\n+ line_72 = 72\n+ line_73 = 73\n+ line_74 = 74\n+ line_75 = 75\n+ line_76 = 76\n+ line_77 = 77\n+ line_78 = 78\n+ line_79 = 79\n+ line_80 = 80\n+ line_81 = 81\n+ line_82 = 82\n+ line_83 = 83\n+ line_84 = 84\n+ line_85 = 85\n+ line_86 = 86\n+ line_87 = 87\n+ line_88 = 88\n+ line_89 = 89\n+ line_90 = 90\n+ line_91 = 91\n+ line_92 = 92\n+ line_93 = 93\n+ line_94 = 94\n+ line_95 = 95\n+ line_96 = 96\n+ line_97 = 97\n+ line_98 = 98\n+ line_99 = 99\n+ line_100 = 100\n+ line_101 = 101\n+ line_102 = 102\n+ line_103 = 103\n+ line_104 = 104\n+ line_105 = 105\n+ line_106 = 106\n+ line_107 = 107\n+ line_108 = 108\n+ line_109 = 109\n+ line_110 = 110\n+ line_111 = 111\n+ line_112 = 112\n+ line_113 = 113\n+ line_114 = 114\n+ line_115 = 115\n+ line_116 = 116\n+ line_117 = 117\n+ line_118 = 118\n+ line_119 = 119\n+ line_120 = 120\n+ line_121 = 121\n+ line_122 = 122\n+ line_123 = 123\n+ line_124 = 124\n+ line_125 = 125\n+ line_126 = 126\n+ line_127 = 127\n+ line_128 = 128\n+ line_129 = 129\n+ line_130 = 130\n+ line_131 = 131\n+ line_132 = 132\n+ line_133 = 133\n+ line_134 = 134\n+ line_135 = 135\n+ line_136 = 136\n+ line_137 = 137\n+ line_138 = 138\n+ line_139 = 139\n+ line_140 = 140\n+ line_141 = 141\n+ line_142 = 142\n+ line_143 = 143\n+ line_144 = 144\n+ line_145 = 145\n+ line_146 = 146\n+ line_147 = 147\n+ line_148 = 148\n+ line_149 = 149\n+ line_150 = 150\n+ line_151 = 151\n+ line_152 = 152\n+ line_153 = 153\n+ line_154 = 154\n+ line_155 = 155\n+ line_156 = 156\n+ line_157 = 157\n+ line_158 = 158\n+ line_159 = 159\n+ line_160 = 160\n+ line_161 = 161\n+ line_162 = 162\n+ line_163 = 163\n+ line_164 = 164\n+ line_165 = 165\n+ line_166 = 166\n+ line_167 = 167\n+ line_168 = 168\n+ line_169 = 169\n+ line_170 = 170\n+ line_171 = 171\n+ line_172 = 172\n+ line_173 = 173\n+ line_174 = 174\n+ line_175 = 175\n+ line_176 = 176\n+ line_177 = 177\n+ line_178 = 178\n+ line_179 = 179\n+ line_180 = 180\n+ line_181 = 181\n+ line_182 = 182\n+ line_183 = 183\n+ line_184 = 184\n+ line_185 = 185\n+ line_186 = 186\n+ line_187 = 187\n+ line_188 = 188\n+ line_189 = 189\n+ line_190 = 190\n+ line_191 = 191\n+ line_192 = 192\n+ line_193 = 193\n+ line_194 = 194\n+ line_195 = 195\n+ line_196 = 196\n+ line_197 = 197\n+ line_198 = 198\n+ line_199 = 199\n+ line_200 = 200\n+ line_201 = 201\n+ line_202 = 202\n+ line_203 = 203\n+ line_204 = 204\n+ line_205 = 205\n+ line_206 = 206\n+ line_207 = 207\n+ line_208 = 208\n+ line_209 = 209\n+ line_210 = 210\n+ line_211 = 211\n+ line_212 = 212\n+ line_213 = 213\n+ line_214 = 214\n+ line_215 = 215\n+ line_216 = 216\n+ line_217 = 217\n+ line_218 = 218\n+ line_219 = 219\n+ line_220 = 220\n+ line_221 = 221\n+ line_222 = 222\n+ line_223 = 223\n+ line_224 = 224\n+ line_225 = 225\n+ line_226 = 226\n+ line_227 = 227\n+ line_228 = 228\n+ line_229 = 229\n+ line_230 = 230\n+ line_231 = 231\n+ line_232 = 232\n+ line_233 = 233\n+ line_234 = 234\n+ line_235 = 235\n+ line_236 = 236\n+ line_237 = 237\n+ line_238 = 238\n+ line_239 = 239\n+ line_240 = 240\n+ line_241 = 241\n+ line_242 = 242\n+ line_243 = 243\n+ line_244 = 244\n+ line_245 = 245\n+ line_246 = 246\n+ line_247 = 247\n+ line_248 = 248\n+ line_249 = 249\n+ line_250 = 250\n+ line_251 = 251\n+ line_252 = 252\n+ line_253 = 253\n+ line_254 = 254\n+ line_255 = 255\n+ line_256 = 256\n+ line_257 = 257\n+ line_258 = 258\n+ line_259 = 259\n+ line_260 = 260\n+ line_261 = 261\n+ line_262 = 262\n+ line_263 = 263\n+ line_264 = 264\n+ line_265 = 265\n+ line_266 = 266\n+ line_267 = 267\n+ line_268 = 268\n+ line_269 = 269\n+ line_270 = 270\n+ line_271 = 271\n+ line_272 = 272\n+ line_273 = 273\n+ line_274 = 274\n+ line_275 = 275\n+ line_276 = 276\n+ line_277 = 277\n+ line_278 = 278\n+ line_279 = 279\n+ line_280 = 280\n+ line_281 = 281\n+ line_282 = 282\n+ line_283 = 283\n+ line_284 = 284\n+ line_285 = 285\n+ line_286 = 286\n+ line_287 = 287\n+ line_288 = 288\n+ line_289 = 289\n+ line_290 = 290\n+ line_291 = 291\n+ line_292 = 292\n+ line_293 = 293\n+ line_294 = 294\n+ line_295 = 295\n+ line_296 = 296\n+ line_297 = 297\n+ line_298 = 298\n+ line_299 = 299\n+ line_300 = 300\n+ line_301 = 301\n+ line_302 = 302\n+ line_303 = 303\n+ line_304 = 304\n+ line_305 = 305\n+ line_306 = 306\n+ line_307 = 307\n+ line_308 = 308\n+ line_309 = 309\n+ line_310 = 310\n+ line_311 = 311\n+ line_312 = 312\n+ line_313 = 313\n+ line_314 = 314\n+ line_315 = 315\n+ line_316 = 316\n+ line_317 = 317\n+ line_318 = 318\n+ line_319 = 319\n+ line_320 = 320\n+ line_321 = 321\n+ line_322 = 322\n+ line_323 = 323\n+ line_324 = 324\n+ line_325 = 325\n+ line_326 = 326\n+ line_327 = 327\n+ line_328 = 328\n+ line_329 = 329\n+ line_330 = 330\n+ line_331 = 331\n+ line_332 = 332\n+ line_333 = 333\n+ line_334 = 334\n+ line_335 = 335\n+ line_336 = 336\n+ line_337 = 337\n+ line_338 = 338\n+ line_339 = 339\n+ line_340 = 340\n+ line_341 = 341\n+ line_342 = 342\n+ line_343 = 343\n+ line_344 = 344\n+ line_345 = 345\n+ line_346 = 346\n+ line_347 = 347\n+ line_348 = 348\n+ line_349 = 349\n+ line_350 = 350\n+ line_351 = 351\n+ line_352 = 352\n+ line_353 = 353\n+ line_354 = 354\n+ line_355 = 355\n+ line_356 = 356\n+ line_357 = 357\n+ line_358 = 358\n+ line_359 = 359\n+ line_360 = 360\n+ line_361 = 361\n+ line_362 = 362\n+ line_363 = 363\n+ line_364 = 364\n+ line_365 = 365\n+ line_366 = 366\n+ line_367 = 367\n+ line_368 = 368\n+ line_369 = 369\n+ line_370 = 370\n+ line_371 = 371\n+ line_372 = 372\n+ line_373 = 373\n+ line_374 = 374\n+ line_375 = 375\n+ line_376 = 376\n+ line_377 = 377\n+ line_378 = 378\n+ line_379 = 379\n+ line_380 = 380\n+ line_381 = 381\n+ line_382 = 382\n+ line_383 = 383\n+ line_384 = 384\n+ line_385 = 385\n+ line_386 = 386\n+ line_387 = 387\n+ line_388 = 388\n+ line_389 = 389\n+ line_390 = 390\n+ line_391 = 391\n+ line_392 = 392\n+ line_393 = 393\n+ line_394 = 394\n+ line_395 = 395\n+ line_396 = 396\n+ line_397 = 397\n+ line_398 = 398\n+ line_399 = 399\n+ line_400 = 400\n+ line_401 = 401\n+ line_402 = 402\n+ line_403 = 403\n+ line_404 = 404\n+ line_405 = 405\n+ line_406 = 406\n+ line_407 = 407\n+ line_408 = 408\n+ line_409 = 409\n+ line_410 = 410\n+ line_411 = 411\n+ line_412 = 412\n+ line_413 = 413\n+ line_414 = 414\n+ line_415 = 415\n+ line_416 = 416\n+ line_417 = 417\n+ line_418 = 418\n+ line_419 = 419\n+ line_420 = 420\n+ line_421 = 421\n+ line_422 = 422\n+ line_423 = 423\n+ line_424 = 424\n+ line_425 = 425\n+ line_426 = 426\n+ line_427 = 427\n+ line_428 = 428\n+ line_429 = 429\n+ line_430 = 430\n+ line_431 = 431\n+ line_432 = 432\n+ line_433 = 433\n+ line_434 = 434\n+ line_435 = 435\n+ line_436 = 436\n+ line_437 = 437\n+ line_438 = 438\n+ line_439 = 439\n+ line_440 = 440\n+ line_441 = 441\n+ line_442 = 442\n+ line_443 = 443\n+ line_444 = 444\n+ line_445 = 445\n+ line_446 = 446\n+ line_447 = 447\n+ line_448 = 448\n+ line_449 = 449\n+ line_450 = 450\n+ line_451 = 451\n+ line_452 = 452\n+ line_453 = 453\n+ line_454 = 454\n+ line_455 = 455\n+ line_456 = 456\n+ line_457 = 457\n+ line_458 = 458\n+ line_459 = 459\n+ line_460 = 460\n+ line_461 = 461\n+ line_462 = 462\n+ line_463 = 463\n+ line_464 = 464\n+ line_465 = 465\n+ line_466 = 466\n+ line_467 = 467\n+ line_468 = 468\n+ line_469 = 469\n+ line_470 = 470\n+ line_471 = 471\n+ line_472 = 472\n+ line_473 = 473\n+ line_474 = 474\n+ line_475 = 475\n+ line_476 = 476\n+ line_477 = 477\n+ line_478 = 478\n+ line_479 = 479\n+ line_480 = 480\n+ line_481 = 481\n+ line_482 = 482\n+ line_483 = 483\n+ line_484 = 484\n+ line_485 = 485\n+ line_486 = 486\n+ line_487 = 487\n+ line_488 = 488\n+ line_489 = 489\n+ line_490 = 490\n+ line_491 = 491\n+ line_492 = 492\n+ line_493 = 493\n+ line_494 = 494\n+ line_495 = 495\n+ line_496 = 496\n+ line_497 = 497\n+ line_498 = 498\n+ line_499 = 499\n" +} \ No newline at end of file diff --git a/bench/fixtures/07-large-diff/target.py b/bench/fixtures/07-large-diff/target.py new file mode 100644 index 0000000..ca1dc31 --- /dev/null +++ b/bench/fixtures/07-large-diff/target.py @@ -0,0 +1 @@ +def main(): diff --git a/bench/fixtures/08-auto-generated-skip/bundle.min.js b/bench/fixtures/08-auto-generated-skip/bundle.min.js new file mode 100644 index 0000000..00df555 --- /dev/null +++ b/bench/fixtures/08-auto-generated-skip/bundle.min.js @@ -0,0 +1 @@ +var a=2 diff --git a/bench/fixtures/08-auto-generated-skip/config.yml b/bench/fixtures/08-auto-generated-skip/config.yml new file mode 100644 index 0000000..9b6e212 --- /dev/null +++ b/bench/fixtures/08-auto-generated-skip/config.yml @@ -0,0 +1,7 @@ +rules: + noop: + description: A rule that should never be evaluated because skip shortcircuits first + engine: script + scope: "**/*.js" + severity: warning + script: "exit 0" diff --git a/bench/fixtures/08-auto-generated-skip/fixture.json b/bench/fixtures/08-auto-generated-skip/fixture.json new file mode 100644 index 0000000..b14f78d --- /dev/null +++ b/bench/fixtures/08-auto-generated-skip/fixture.json @@ -0,0 +1,7 @@ +{ + "name": "08-auto-generated-skip", + "description": "File matches SKIP_PATTERNS (*.min.js); pipeline should short-circuit", + "file_path": "bench/fixtures/08-auto-generated-skip/bundle.min.js", + "edit_type": "Edit", + "diff": "--- a/bundle.min.js\n+++ b/bundle.min.js\n@@ -1 +1 @@\n-var a=1\n+var a=2\n" +} diff --git a/bench/history.jsonl b/bench/history.jsonl new file mode 100644 index 0000000..072b437 --- /dev/null +++ b/bench/history.jsonl @@ -0,0 +1 @@ +{"ts": "2026-04-17T18:06:32Z", "git_sha": "a4ebc06", "git_dirty": true, "python_version": "3.9.6", "anthropic_sdk_version": null, "machine": "darwin-23.6.0", "fixtures": [{"name": "01-script-only-small-diff", "description": "One script rule, small Python edit", "wall_ms_p50": 5.104042, "wall_ms_p95": 5.8833498, "phases_ms": {"skip_check": 0.249166, "trust_gate": 0.002, "parse_config": 0.094708, "filter_rules": 0.00825, "script_exec": 4.587875, "ast_exec": 0.000292, "semantic_build": 0.000292}, "cold_start_ms": 48.94475, "tokens": {"input": 0, "method": "n/a-no-semantic-rules"}}, {"name": "02-ast-only-small-diff", "description": "One ast rule, small TypeScript edit", "wall_ms_p50": 7.551583, "wall_ms_p95": 7.7790168, "phases_ms": {"skip_check": 0.239125, "trust_gate": 0.002375, "parse_config": 0.099292, "filter_rules": 0.008125, "script_exec": 0.00025, "ast_exec": 7.017292, "semantic_build": 0.000375}, "cold_start_ms": 52.670583, "tokens": {"input": 0, "method": "n/a-no-semantic-rules"}}, {"name": "03-semantic-only-small-diff", "description": "One semantic rule, small Python edit (no violation)", "wall_ms_p50": 0.341042, "wall_ms_p95": 0.34400000000000003, "phases_ms": {"skip_check": 0.150708, "trust_gate": 0.00125, "parse_config": 0.079, "filter_rules": 0.006958, "script_exec": 0.000209, "ast_exec": 0.000167, "semantic_build": 0.004917}, "cold_start_ms": 44.313875, "tokens": {"input": 1601, "method": "proxy"}}, {"name": "04-mixed-engines", "description": "One rule per engine, moderate Python edit", "wall_ms_p50": 12.792958, "wall_ms_p95": 14.2772166, "phases_ms": {"skip_check": 0.273625, "trust_gate": 0.002208, "parse_config": 0.134583, "filter_rules": 0.013541, "script_exec": 4.88525, "ast_exec": 7.253709, "semantic_build": 0.022708}, "cold_start_ms": 59.831958, "tokens": {"input": 1527, "method": "proxy"}}, {"name": "05-big-extends-chain", "description": "Three-level extends chain; stresses parser + skip walker", "wall_ms_p50": 9.617083, "wall_ms_p95": 10.24385, "phases_ms": {"skip_check": 0.510542, "trust_gate": 0.001917, "parse_config": 0.339708, "filter_rules": 0.013208, "script_exec": 8.433916, "ast_exec": 0.000333, "semantic_build": 0.000209}, "cold_start_ms": 54.553125, "tokens": {"input": 0, "method": "n/a-no-semantic-rules"}}, {"name": "06-many-semantic-rules", "description": "20 semantic rules, small Python edit; stresses payload size", "wall_ms_p50": 1.03975, "wall_ms_p95": 1.0824416, "phases_ms": {"skip_check": 0.449916, "trust_gate": 0.001375, "parse_config": 0.390167, "filter_rules": 0.045834, "script_exec": 0.00025, "ast_exec": 0.000166, "semantic_build": 0.060083}, "cold_start_ms": 45.03475, "tokens": {"input": 3780, "method": "proxy"}}, {"name": "07-large-diff", "description": "500-line diff, one semantic rule; scales semantic payload", "wall_ms_p50": 0.528292, "wall_ms_p95": 0.543158, "phases_ms": {"skip_check": 0.1445, "trust_gate": 0.001291, "parse_config": 0.084959, "filter_rules": 0.00675, "script_exec": 0.000209, "ast_exec": 0.000167, "semantic_build": 0.16875}, "cold_start_ms": 45.428917, "tokens": {"input": 11736, "method": "proxy"}}, {"name": "08-auto-generated-skip", "description": "File matches SKIP_PATTERNS (*.min.js); pipeline should short-circuit", "wall_ms_p50": 0.1675, "wall_ms_p95": 0.1737916, "phases_ms": {"skip_check": 0.125833}, "cold_start_ms": 43.669291, "tokens": {"input": 0, "method": "n/a-no-semantic-rules"}}], "aggregates": {"total_wall_ms_p50": 37.14225, "total_cold_start_ms": 394.44724900000006, "total_input_tokens": 18644, "tokens_method": "proxy"}} diff --git a/docs/superpowers/plans/2026-04-17-test-bench.md b/docs/superpowers/plans/2026-04-17-test-bench.md new file mode 100644 index 0000000..660a177 --- /dev/null +++ b/docs/superpowers/plans/2026-04-17-test-bench.md @@ -0,0 +1,2251 @@ +# Bully Test Bench Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a two-mode local bench for measuring bully's speed and input-token cost — a fixture suite that appends to a versioned history log, plus a config-cost analyzer that takes any `.bully.yml` and reports per-invocation token breakdown. + +**Architecture:** New `pipeline/bench.py` module holds both modes. Adds a tiny optional `phase_timer` parameter to `run_pipeline` so the bench can time phases without duplicating the pipeline loop. Fixtures live in `bench/fixtures//` as paired `config.yml` + `fixture.json` files so the production `.bully.yml` parser exercises them end-to-end. Real Anthropic `messages/count_tokens` when available; `len(json.dumps(...))` proxy otherwise. Pipeline stays stdlib-only; `anthropic` is an optional dep. + +**Tech Stack:** Python 3.10+, stdlib, `anthropic` (optional), `pytest` + `hypothesis` (already dev deps). + +**Reference spec:** `docs/superpowers/specs/2026-04-17-test-bench-design.md` + +--- + +## File Structure + +**Create:** +- `pipeline/bench.py` — both modes + helpers (fixture loader, token counter, phase timer, history writer) +- `pipeline/tests/test_bench.py` — unit + integration tests for bench +- `bench/fixtures//config.yml` — 8 fixture configs +- `bench/fixtures//fixture.json` — 8 fixture metadata files + +**Modify:** +- `pipeline/pipeline.py` — add optional `phase_timer` parameter to `run_pipeline`; add `bench` subcommand dispatch in `main()` +- `pyproject.toml` — add `[project.optional-dependencies] bench = ["anthropic>=0.40"]` +- `README.md` — add a "Bench" section at the end + +**Not created (runtime-generated):** +- `bench/history.jsonl` — created by first `bully bench` run; checked into git incrementally per run +- `bench/fixtures/__init__.py` — not needed; bench enumerates the directory + +--- + +## Task 1: Pipeline phase-timer hook (zero-cost by default) + +**Files:** +- Modify: `pipeline/pipeline.py` (add import + small edits inside `run_pipeline`) +- Test: `pipeline/tests/test_bench.py` (new file) + +- [ ] **Step 1: Write the failing test** + +Create `pipeline/tests/test_bench.py`: + +```python +"""Bench harness tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import pipeline as pl + + +def test_phase_timer_default_is_noop_and_pipeline_still_runs(tmp_path, monkeypatch): + """run_pipeline works unchanged when no phase_timer is passed.""" + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " - id: trivial\n" + " description: trivial\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + target = tmp_path / "x.py" + target.write_text("x = 1\n") + result = pl.run_pipeline(str(cfg), str(target), diff="") + assert result["status"] == "pass" + + +def test_phase_timer_records_each_phase(tmp_path, monkeypatch): + """When a phase_timer is passed, it's called for each phase.""" + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " - id: trivial\n" + " description: trivial\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + target = tmp_path / "x.py" + target.write_text("x = 1\n") + + seen: list[str] = [] + + class Recorder: + def __call__(self, name): + seen.append(name) + return self + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + pl.run_pipeline(str(cfg), str(target), diff="", phase_timer=Recorder()) + assert "skip_check" in seen + assert "parse_config" in seen + assert "filter_rules" in seen + assert "script_exec" in seen +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest pipeline/tests/test_bench.py -v` +Expected: FAIL — `run_pipeline` has no `phase_timer` keyword argument. + +- [ ] **Step 3: Add the phase-timer hook to `run_pipeline`** + +In `pipeline/pipeline.py`, near the top of the file (after existing imports), add a no-op timer: + +```python +class _NoopPhaseTimer: + """Default phase timer: every call is a no-op context manager.""" + + def __call__(self, name: str) -> "_NoopPhaseTimer": + return self + + def __enter__(self) -> "_NoopPhaseTimer": + return self + + def __exit__(self, *a) -> bool: + return False + + +_NOOP_PHASE_TIMER = _NoopPhaseTimer() +``` + +Then modify `run_pipeline`'s signature (around line 1434) to accept the hook: + +```python +def run_pipeline( + config_path: str, + file_path: str, + diff: str, + rule_filter: set[str] | None = None, + *, + include_skipped: bool = False, + phase_timer=_NOOP_PHASE_TIMER, +) -> dict: +``` + +Wrap the existing phases in `run_pipeline` with `with phase_timer():` blocks. Phase names to use, in order: + +- `skip_check` wraps the `effective_skip_patterns(config_path)` call + `_path_matches_skip` check (the block that currently starts around line 1458) +- `trust_gate` wraps the `_trust_status(config_path)` call (around line 1467) +- `parse_config` wraps the `parse_config(config_path)` call (around line 1483) +- `filter_rules` wraps `filter_rules(rules, file_path)` (around line 1484) and the rule_filter narrow immediately after +- `script_exec` wraps the `for rule in script_rules:` loop (around line 1550) +- `ast_exec` wraps the `if ast_rules:` block (around line 1557) +- `semantic_build` wraps the can't-match filter loop + semantic payload construction (lines 1581 through the end of the semantic section) + +The wrapping pattern for each is: + +```python +with phase_timer("parse_config"): + rules = parse_config(config_path) +``` + +Important: early returns inside a phase (e.g., the skip-gate returning a `"skipped"` dict) work correctly because the context manager's `__exit__` runs on the way out. Don't restructure the early-return logic. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest pipeline/tests/test_bench.py -v` +Expected: PASS, both tests. + +- [ ] **Step 5: Run the full existing test suite to catch regressions** + +Run: `pytest pipeline/tests -x -q` +Expected: all existing tests pass (the default `_NOOP_PHASE_TIMER` is a no-op). + +- [ ] **Step 6: Commit** + +```bash +git add pipeline/pipeline.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add optional phase_timer hook to run_pipeline + +Bench harness needs to measure per-phase timing without duplicating the +pipeline loop. Default is a zero-cost no-op so normal hook invocations +are unaffected. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 2: Bench module skeleton + optional `anthropic` dep + +**Files:** +- Create: `pipeline/bench.py` +- Modify: `pyproject.toml` +- Modify: `pipeline/pipeline.py` (wire `bully bench` subcommand) +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing test** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_bench_cli_entrypoint_exists(): + """`bully bench --help` exits cleanly (implies argparse wiring is in place).""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "pipeline.pipeline", "bench", "--help"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "bench" in (result.stdout + result.stderr).lower() + + +def test_bench_module_imports(): + """bench module loads without requiring anthropic.""" + from pipeline import bench + + assert hasattr(bench, "main") +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py::test_bench_module_imports pipeline/tests/test_bench.py::test_bench_cli_entrypoint_exists -v` +Expected: FAIL — `pipeline.bench` doesn't exist; `bully bench` isn't wired. + +- [ ] **Step 3: Create `pipeline/bench.py`** + +```python +""" +Bully Test Bench + +Two modes: + bully bench -- run fixture suite, append to bench/history.jsonl + bully bench --config -- analyze token cost of any .bully.yml + +Stdlib-only except for the optional `anthropic` import, which is gated +behind API-key presence and falls back to a char-count proxy. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="bully bench", + description="Measure bully's speed and input-token cost.", + ) + parser.add_argument( + "--config", + help="Path to a .bully.yml; enables Mode B (config cost analysis).", + ) + parser.add_argument( + "--compare", + action="store_true", + help="Mode A only: diff the last two runs in bench/history.jsonl.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Emit machine-readable JSON instead of a formatted table.", + ) + parser.add_argument( + "--no-tokens", + action="store_true", + help="Skip Anthropic API call; use char-count proxy for token counts.", + ) + parser.add_argument( + "--fixtures-dir", + default="bench/fixtures", + help="Directory of fixture subdirectories (default: bench/fixtures).", + ) + parser.add_argument( + "--history", + default="bench/history.jsonl", + help="Path to history JSONL (default: bench/history.jsonl).", + ) + + args = parser.parse_args(argv) + + # Stub: subsequent tasks will dispatch to mode_a / mode_b / compare. + print("bench: not yet implemented", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +- [ ] **Step 4: Wire `bully bench` into the main CLI** + +In `pipeline/pipeline.py`, inside `main()`, find the block that checks subcommand flags (starts around line 2222 with `if args.trust:`). Add a check for `bench` **before** the validation-only blocks but after the simple ones. The cleanest location is right after the `if args.hook_mode:` check (around line 2241). + +Add: + +```python + if args.bench: + from pipeline.bench import main as bench_main + sys.exit(bench_main(args.bench_args)) +``` + +Then wire the argparse side. Find `_parse_args` in the same file (grep for `def _parse_args`). Add a subparser-style flag. Since the existing parser is flat (flags only, no subparsers), do the simplest thing: add a `--bench` passthrough. But `bully bench ...` is already friendlier than `bully --bench ...`, so do this: + +At the very top of `main()` (before `_parse_args(sys.argv[1:])`), add a bench short-circuit: + +```python +def main() -> None: + # Short-circuit: `bully bench ...` dispatches to the bench CLI directly, + # bypassing the main parser (which uses a flat flag model). + if len(sys.argv) >= 2 and sys.argv[1] == "bench": + from pipeline.bench import main as bench_main + sys.exit(bench_main(sys.argv[2:])) + + args = _parse_args(sys.argv[1:]) + ... +``` + +Do **not** add a `--bench` flag to `_parse_args`. Keep the dispatch surgical. + +- [ ] **Step 5: Add optional `bench` dependencies to pyproject.toml** + +Modify `pyproject.toml`: in the `[project.optional-dependencies]` section, add a `bench` group. The section should look like: + +```toml +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "ruff>=0.8.0", + "shellcheck-py>=0.10.0", + "pre-commit>=3.7", + "hypothesis>=6.100", +] +bench = ["anthropic>=0.40"] +``` + +- [ ] **Step 6: Update setuptools packages list** + +Still in `pyproject.toml`, the existing `[tool.setuptools]` section has `packages = ["pipeline"]`. No change needed — `pipeline.bench` is already covered. + +- [ ] **Step 7: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v` +Expected: all tests in this task pass. + +- [ ] **Step 8: Commit** + +```bash +git add pipeline/bench.py pipeline/pipeline.py pyproject.toml pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add bench module skeleton and CLI entry point + +`bully bench` now dispatches to pipeline.bench. Flags parsed but not yet +implemented -- subsequent tasks fill in fixture loading, token counting, +and both modes. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 3: Fixture loader + +**Files:** +- Modify: `pipeline/bench.py` +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing test** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_load_fixture_reads_config_and_metadata(tmp_path): + """load_fixture returns a Fixture with config_path + metadata.""" + from pipeline.bench import load_fixture + + fx_dir = tmp_path / "my-fixture" + fx_dir.mkdir() + (fx_dir / "config.yml").write_text( + "rules:\n" + " - id: r1\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (fx_dir / "fixture.json").write_text( + '{"name": "my-fixture", "description": "x", ' + '"file_path": "src/a.py", "edit_type": "Edit", "diff": "--- a\\n"}' + ) + fx = load_fixture(fx_dir) + assert fx.name == "my-fixture" + assert fx.file_path == "src/a.py" + assert fx.edit_type == "Edit" + assert fx.diff.startswith("--- a") + assert fx.config_path.name == "config.yml" + assert fx.config_path.exists() + + +def test_load_fixture_rejects_missing_files(tmp_path): + """load_fixture raises when either expected file is missing.""" + from pipeline.bench import FixtureError, load_fixture + + fx_dir = tmp_path / "bad" + fx_dir.mkdir() + # Missing both files. + import pytest + with pytest.raises(FixtureError, match="config.yml"): + load_fixture(fx_dir) + + +def test_load_fixture_rejects_malformed_json(tmp_path): + """load_fixture raises a clear error on malformed metadata JSON.""" + from pipeline.bench import FixtureError, load_fixture + + fx_dir = tmp_path / "bad-json" + fx_dir.mkdir() + (fx_dir / "config.yml").write_text("rules: []\n") + (fx_dir / "fixture.json").write_text("{not json") + import pytest + with pytest.raises(FixtureError, match="fixture.json"): + load_fixture(fx_dir) + + +def test_discover_fixtures_lists_all_subdirs(tmp_path): + """discover_fixtures returns a sorted list of fixture directories.""" + from pipeline.bench import discover_fixtures + + for name in ["zeta", "alpha", "mu"]: + d = tmp_path / name + d.mkdir() + (d / "config.yml").write_text("rules: []\n") + (d / "fixture.json").write_text( + '{"name": "' + name + '", "description": "", ' + '"file_path": "a.py", "edit_type": "Edit", "diff": ""}' + ) + result = discover_fixtures(tmp_path) + assert [f.name for f in result] == ["alpha", "mu", "zeta"] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py::test_load_fixture_reads_config_and_metadata -v` +Expected: FAIL — `load_fixture` not defined. + +- [ ] **Step 3: Add the fixture loader to `pipeline/bench.py`** + +Add to `pipeline/bench.py`, above `main()`: + +```python +import json +from dataclasses import dataclass + + +class FixtureError(Exception): + """Raised when a fixture directory is malformed.""" + + +@dataclass(frozen=True) +class Fixture: + name: str + description: str + file_path: str + edit_type: str + diff: str + config_path: Path + + @property + def dir(self) -> Path: + return self.config_path.parent + + +def load_fixture(fixture_dir: Path) -> Fixture: + """Load a fixture from `/config.yml` + `/fixture.json`.""" + fixture_dir = Path(fixture_dir) + cfg = fixture_dir / "config.yml" + meta = fixture_dir / "fixture.json" + if not cfg.is_file(): + raise FixtureError(f"missing config.yml in {fixture_dir}") + if not meta.is_file(): + raise FixtureError(f"missing fixture.json in {fixture_dir}") + try: + data = json.loads(meta.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + raise FixtureError(f"malformed fixture.json in {fixture_dir}: {e}") from e + + required = ("name", "description", "file_path", "edit_type", "diff") + for key in required: + if key not in data: + raise FixtureError(f"fixture.json in {fixture_dir} missing field {key!r}") + + return Fixture( + name=data["name"], + description=data["description"], + file_path=data["file_path"], + edit_type=data["edit_type"], + diff=data["diff"], + config_path=cfg, + ) + + +def discover_fixtures(root: Path) -> list[Fixture]: + """Load every fixture subdirectory under `root`, sorted by name.""" + root = Path(root) + if not root.is_dir(): + return [] + out: list[Fixture] = [] + for child in sorted(root.iterdir()): + if not child.is_dir(): + continue + out.append(load_fixture(child)) + return out +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v` +Expected: all four new tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/bench.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add bench fixture loader + +Fixtures live in bench/fixtures//{config.yml,fixture.json}. +load_fixture validates both files exist and the metadata has required +fields; discover_fixtures returns all subdirectories sorted by name. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 4: Token-counting helper with proxy fallback + +**Files:** +- Modify: `pipeline/bench.py` +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_count_tokens_proxy_when_no_api_key(monkeypatch): + """count_tokens falls back to char-count when no API key is present.""" + from pipeline.bench import count_tokens + + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + payload = {"file": "x.py", "diff": "hello", "evaluate": []} + count, method = count_tokens(payload, system="sys prompt") + assert method == "proxy" + assert count == len( + __import__("json").dumps(payload) + ) + len("sys prompt") + + +def test_count_tokens_proxy_when_anthropic_missing(monkeypatch): + """If `anthropic` is not importable, fall back to proxy even with key set.""" + from pipeline import bench + + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.setattr(bench, "_import_anthropic", lambda: None) + + payload = {"x": 1} + count, method = bench.count_tokens(payload, system="s") + assert method == "proxy" + + +def test_count_tokens_api_path(monkeypatch): + """With API key + anthropic client, call messages.count_tokens.""" + from pipeline import bench + + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + + class FakeResp: + input_tokens = 42 + + class FakeMessages: + def count_tokens(self, **kwargs): + assert kwargs["model"] == bench.BENCH_MODEL + assert kwargs["system"] == "s" + assert "content" in kwargs["messages"][0] + return FakeResp() + + class FakeClient: + def __init__(self): + self.messages = FakeMessages() + + class FakeAnthropic: + Anthropic = FakeClient + + monkeypatch.setattr(bench, "_import_anthropic", lambda: FakeAnthropic) + + count, method = bench.count_tokens({"a": 1}, system="s") + assert count == 42 + assert method == "count_tokens" + + +def test_load_evaluator_system_prompt_strips_frontmatter(tmp_path, monkeypatch): + """System prompt is read from agents/bully-evaluator.md without frontmatter.""" + from pipeline import bench + + agents_dir = tmp_path / "agents" + agents_dir.mkdir() + (agents_dir / "bully-evaluator.md").write_text( + "---\n" + "name: bully-evaluator\n" + "model: sonnet\n" + "---\n" + "\n" + "You are the evaluator. Apply each rule.\n" + ) + monkeypatch.setattr(bench, "_repo_root", lambda: tmp_path) + + text = bench.load_evaluator_system_prompt() + assert text.startswith("You are the evaluator") + assert "---" not in text +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py -v -k "count_tokens or evaluator_system"` +Expected: FAIL — `count_tokens`, `_import_anthropic`, `BENCH_MODEL`, `load_evaluator_system_prompt` not defined. + +- [ ] **Step 3: Add token-counting and prompt-loading helpers** + +Add to `pipeline/bench.py`, above `main()`: + +```python +import os + + +BENCH_MODEL = "claude-sonnet-4-6" + + +def _repo_root() -> Path: + """Return the project root (directory holding the `agents/` dir). + + Assumes bench.py lives at /pipeline/bench.py. + """ + return Path(__file__).resolve().parent.parent + + +def _import_anthropic(): + """Import and return the anthropic module, or None if unavailable.""" + try: + import anthropic # type: ignore[import-not-found] + except ImportError: + return None + return anthropic + + +def load_evaluator_system_prompt() -> str: + """Load the bully-evaluator system prompt from agents/bully-evaluator.md. + + Strips the YAML frontmatter (everything between the first `---` pair). + """ + path = _repo_root() / "agents" / "bully-evaluator.md" + text = path.read_text(encoding="utf-8") + if text.startswith("---"): + # Find the closing frontmatter delimiter. + rest = text[3:] + end = rest.find("\n---") + if end != -1: + text = rest[end + 4 :] # past "\n---" + return text.lstrip("\n") + + +def count_tokens(payload: dict, *, system: str, use_api: bool = True) -> tuple[int, str]: + """Count input tokens for the given bully-evaluator payload. + + Returns (token_count, method) where method is 'count_tokens' or 'proxy'. + + Uses the Anthropic `messages/count_tokens` endpoint when + ANTHROPIC_API_KEY is set AND the anthropic SDK is importable AND + use_api is True. Falls back to `len(json.dumps(payload)) + len(system)`. + """ + api_key = os.environ.get("ANTHROPIC_API_KEY") + anthropic = _import_anthropic() if use_api else None + if use_api and api_key and anthropic is not None: + try: + client = anthropic.Anthropic(api_key=api_key) + resp = client.messages.count_tokens( + model=BENCH_MODEL, + system=system, + messages=[{"role": "user", "content": json.dumps(payload)}], + ) + return int(resp.input_tokens), "count_tokens" + except Exception: + # Any API failure -> proxy. Bench must not crash on transient errors. + pass + return len(json.dumps(payload)) + len(system), "proxy" +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v -k "count_tokens or evaluator_system"` +Expected: all four tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add pipeline/bench.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add token counter with Anthropic SDK + proxy fallback + +count_tokens uses messages/count_tokens when an API key and the SDK are +available; otherwise returns len(json.dumps(payload)) + len(system) and +tags the result 'proxy'. System prompt is loaded from the real +bully-evaluator agent file so counts match production. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 5: Phase-timer implementation + single-fixture runner + +**Files:** +- Modify: `pipeline/bench.py` +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_phasetimer_records_all_phase_durations(): + """PhaseTimer records a list of (name, ns) for each phase invocation.""" + from pipeline.bench import PhaseTimer + import time + + pt = PhaseTimer() + with pt("a"): + time.sleep(0.001) + with pt("b"): + time.sleep(0.002) + results = pt.results_ns() + assert "a" in results + assert "b" in results + # Each phase tracked at least one sample. + assert len(results["a"]) == 1 + assert len(results["b"]) == 1 + assert results["a"][0] > 0 + assert results["b"][0] > results["a"][0] - 500_000 # loose ordering sanity + + +def test_run_fixture_returns_structured_result(tmp_path, monkeypatch): + """run_fixture returns per-phase median/p95 plus cold-start and tokens.""" + from pipeline.bench import Fixture, run_fixture + + fx_dir = tmp_path / "fx" + fx_dir.mkdir() + cfg = fx_dir / "config.yml" + cfg.write_text( + "rules:\n" + " - id: r1\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (fx_dir / "fixture.json").write_text( + '{"name": "fx", "description": "x", "file_path": "a.py",' + ' "edit_type": "Edit", "diff": ""}' + ) + target = tmp_path / "a.py" + target.write_text("x = 1\n") + + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + monkeypatch.chdir(tmp_path) + + fx = Fixture( + name="fx", + description="x", + file_path=str(target), + edit_type="Edit", + diff="", + config_path=cfg, + ) + result = run_fixture(fx, iterations=3, use_api=False, skip_cold_start=True) + assert result["name"] == "fx" + assert "wall_ms_p50" in result + assert "wall_ms_p95" in result + assert "phases_ms" in result + assert "skip_check" in result["phases_ms"] + assert "parse_config" in result["phases_ms"] + assert result["tokens"]["method"] == "proxy" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py -v -k "phasetimer or run_fixture"` +Expected: FAIL — `PhaseTimer` and `run_fixture` not defined. + +- [ ] **Step 3: Add `PhaseTimer` and `run_fixture` to `pipeline/bench.py`** + +Add to `pipeline/bench.py`: + +```python +import statistics +import subprocess +import time + + +class PhaseTimer: + """Callable that records elapsed time for each named phase. + + Usage: + pt = PhaseTimer() + with pt("parse_config"): + ... + pt.results_ns() # {"parse_config": [12345, ...], ...} + """ + + def __init__(self) -> None: + self._samples: dict[str, list[int]] = {} + self._current: str | None = None + self._start_ns: int = 0 + + def __call__(self, name: str) -> "PhaseTimer": + self._current = name + return self + + def __enter__(self) -> "PhaseTimer": + self._start_ns = time.perf_counter_ns() + return self + + def __exit__(self, *a) -> bool: + elapsed = time.perf_counter_ns() - self._start_ns + assert self._current is not None + self._samples.setdefault(self._current, []).append(elapsed) + self._current = None + return False + + def results_ns(self) -> dict[str, list[int]]: + return dict(self._samples) + + +def _percentile(values: list[float], pct: float) -> float: + """Return the `pct`th percentile (0..100) by linear interpolation.""" + if not values: + return 0.0 + s = sorted(values) + if len(s) == 1: + return s[0] + k = (len(s) - 1) * (pct / 100.0) + lo = int(k) + hi = min(lo + 1, len(s) - 1) + frac = k - lo + return s[lo] + (s[hi] - s[lo]) * frac + + +def run_fixture( + fx: "Fixture", + *, + iterations: int = 5, + use_api: bool = True, + skip_cold_start: bool = False, +) -> dict: + """Run one fixture: warm + N timed + cold-start + token count. + + Returns a per-fixture result dict suitable for the history JSONL. + """ + # Import here to avoid a circular import at module load. + from pipeline import pipeline as pl + + cfg_path = str(fx.config_path) + + # Bundled fixtures are trusted by construction; short-circuit the + # trust gate so the bench doesn't require `bully trust` on every + # fixture config. Safe because fixtures ship in-repo. + os.environ["BULLY_TRUST_ALL"] = "1" + + # Warm run (discarded). + pl.run_pipeline(cfg_path, fx.file_path, fx.diff) + + # Timed runs. + wall_samples_ns: list[int] = [] + phase_samples_ns: dict[str, list[int]] = {} + for _ in range(iterations): + pt = PhaseTimer() + t0 = time.perf_counter_ns() + pl.run_pipeline(cfg_path, fx.file_path, fx.diff, phase_timer=pt) + wall_samples_ns.append(time.perf_counter_ns() - t0) + for name, samples in pt.results_ns().items(): + # Sum of this phase for this iteration (phases may re-enter). + phase_samples_ns.setdefault(name, []).append(sum(samples)) + + wall_ms = [ns / 1_000_000 for ns in wall_samples_ns] + phases_ms = { + name: statistics.median([ns / 1_000_000 for ns in samples]) + for name, samples in phase_samples_ns.items() + } + + # Cold-start: one subprocess invocation, wall-clock only. Use the + # default CLI path (not --hook-mode) so the subprocess doesn't block + # waiting on a Claude Code tool-hook payload. + cold_start_ms: float | None = None + if not skip_cold_start: + pipeline_py = Path(pl.__file__) + t0 = time.perf_counter_ns() + subprocess.run( + [ + sys.executable, str(pipeline_py), + "--config", cfg_path, + "--file", fx.file_path, + "--diff", fx.diff, + ], + capture_output=True, + text=True, + timeout=30, + ) + cold_start_ms = (time.perf_counter_ns() - t0) / 1_000_000 + + # Tokens: build the real semantic payload and count. + rules = pl.parse_config(cfg_path) + matching = pl.filter_rules(rules, fx.file_path) + passed = [r.id for r in matching if r.engine in ("script", "ast")] + semantic = [r for r in matching if r.engine == "semantic"] + if semantic: + payload = pl.build_semantic_payload( + fx.file_path, fx.diff, passed, semantic + ) + system = load_evaluator_system_prompt() + tokens, method = count_tokens(payload["_evaluator_input"], + system=system, use_api=use_api) + else: + tokens, method = 0, "n/a-no-semantic-rules" + + return { + "name": fx.name, + "description": fx.description, + "wall_ms_p50": statistics.median(wall_ms), + "wall_ms_p95": _percentile(wall_ms, 95), + "phases_ms": phases_ms, + "cold_start_ms": cold_start_ms, + "tokens": {"input": tokens, "method": method}, + } +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v -k "phasetimer or run_fixture"` +Expected: both tests pass. + +- [ ] **Step 5: Run the full bench test file to catch any regressions** + +Run: `pytest pipeline/tests/test_bench.py -v` +Expected: all bench tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add pipeline/bench.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add PhaseTimer and single-fixture runner + +PhaseTimer implements the pipeline hook protocol and records per-phase +elapsed time. run_fixture runs one fixture N times in-process, samples +cold-start once via subprocess, and builds the real semantic payload to +count tokens. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 6: Mode A — run all fixtures + write history + +**Files:** +- Modify: `pipeline/bench.py` +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_run_mode_a_writes_history_line(tmp_path, monkeypatch): + """Mode A writes one JSONL line per run with fixture results + aggregates.""" + from pipeline.bench import run_mode_a + + # Build two fixtures. + fixtures_root = tmp_path / "fixtures" + for name in ("a", "b"): + d = fixtures_root / name + d.mkdir(parents=True) + (d / "config.yml").write_text( + "rules:\n" + " - id: r\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (d / "fixture.json").write_text( + '{"name": "' + name + '", "description": "x",' + ' "file_path": "x.py", "edit_type": "Edit", "diff": ""}' + ) + (tmp_path / "x.py").write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + + history_path = tmp_path / "history.jsonl" + rc = run_mode_a( + fixtures_dir=fixtures_root, + history_path=history_path, + use_api=False, + iterations=2, + skip_cold_start=True, + emit_json=True, + ) + assert rc == 0 + assert history_path.is_file() + lines = history_path.read_text().strip().splitlines() + assert len(lines) == 1 + record = json.loads(lines[0]) + assert "ts" in record + assert "fixtures" in record + assert len(record["fixtures"]) == 2 + assert {f["name"] for f in record["fixtures"]} == {"a", "b"} + assert "aggregates" in record + assert "total_wall_ms_p50" in record["aggregates"] + + +def test_run_mode_a_errors_when_no_fixtures(tmp_path, capsys): + """Mode A returns non-zero and prints an error when no fixtures exist.""" + from pipeline.bench import run_mode_a + + history_path = tmp_path / "h.jsonl" + rc = run_mode_a( + fixtures_dir=tmp_path / "missing", + history_path=history_path, + use_api=False, + ) + assert rc != 0 + err = capsys.readouterr().err + assert "no fixtures" in err.lower() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py -v -k "mode_a"` +Expected: FAIL — `run_mode_a` not defined. + +- [ ] **Step 3: Add `run_mode_a` and helpers** + +Add to `pipeline/bench.py`: + +```python +import platform +from datetime import datetime, timezone + + +def _git_sha() -> str | None: + """Best-effort current git SHA; None if git unavailable or not a repo.""" + try: + r = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=2, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + if r.returncode != 0: + return None + return r.stdout.strip() or None + + +def _git_dirty() -> bool: + """True iff there are uncommitted changes.""" + try: + r = subprocess.run( + ["git", "status", "--porcelain"], + capture_output=True, + text=True, + timeout=2, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + return bool(r.stdout.strip()) + + +def _anthropic_sdk_version() -> str | None: + mod = _import_anthropic() + return getattr(mod, "__version__", None) if mod else None + + +def run_mode_a( + *, + fixtures_dir: Path, + history_path: Path, + use_api: bool = True, + iterations: int = 5, + skip_cold_start: bool = False, + emit_json: bool = False, +) -> int: + """Run every fixture in `fixtures_dir`, append a record to `history_path`.""" + fixtures = discover_fixtures(fixtures_dir) + if not fixtures: + sys.stderr.write(f"bench: no fixtures found in {fixtures_dir}\n") + return 1 + + results = [] + for fx in fixtures: + results.append( + run_fixture( + fx, + iterations=iterations, + use_api=use_api, + skip_cold_start=skip_cold_start, + ) + ) + + total_wall = sum(r["wall_ms_p50"] for r in results) + total_cold = sum( + r["cold_start_ms"] for r in results if r.get("cold_start_ms") is not None + ) or None + total_tokens = sum(r["tokens"]["input"] for r in results) + methods = {r["tokens"]["method"] for r in results if r["tokens"]["input"]} + record = { + "ts": datetime.now(timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z"), + "git_sha": _git_sha(), + "git_dirty": _git_dirty(), + "python_version": platform.python_version(), + "anthropic_sdk_version": _anthropic_sdk_version(), + "machine": f"{platform.system().lower()}-{platform.release()}", + "fixtures": results, + "aggregates": { + "total_wall_ms_p50": total_wall, + "total_cold_start_ms": total_cold, + "total_input_tokens": total_tokens, + "tokens_method": next(iter(methods)) if len(methods) == 1 else "mixed", + }, + } + history_path.parent.mkdir(parents=True, exist_ok=True) + with history_path.open("a", encoding="utf-8") as f: + f.write(json.dumps(record) + "\n") + + if emit_json: + print(json.dumps(record, indent=2)) + else: + _print_mode_a_summary(record) + return 0 + + +def _print_mode_a_summary(record: dict) -> None: + """Render a human-readable summary of a Mode A record to stdout.""" + print(f"bench run @ {record['ts']} (sha={record['git_sha'] or '?'})") + print(f" python={record['python_version']} machine={record['machine']}") + print() + print(f" {'fixture':<32} {'wall_p50_ms':>12} {'cold_ms':>10} " + f"{'tokens':>9} method") + for r in record["fixtures"]: + cold = r.get("cold_start_ms") + cold_str = f"{cold:.1f}" if isinstance(cold, (int, float)) else "-" + print( + f" {r['name']:<32} {r['wall_ms_p50']:>12.2f} {cold_str:>10} " + f"{r['tokens']['input']:>9} {r['tokens']['method']}" + ) + agg = record["aggregates"] + print() + print(f" totals: wall_p50={agg['total_wall_ms_p50']:.1f}ms " + f"cold={agg['total_cold_start_ms'] or 0:.1f}ms " + f"tokens={agg['total_input_tokens']} ({agg['tokens_method']})") +``` + +- [ ] **Step 4: Wire Mode A into `main()` (default path when `--config` is absent)** + +Replace the stub at the bottom of `main()` in `pipeline/bench.py`: + +```python + # Dispatch. + if args.config: + sys.stderr.write("bench: mode B (--config) not yet implemented\n") + return 1 + if args.compare: + sys.stderr.write("bench: --compare not yet implemented\n") + return 1 + return run_mode_a( + fixtures_dir=Path(args.fixtures_dir), + history_path=Path(args.history), + use_api=not args.no_tokens, + emit_json=args.json, + ) +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v -k "mode_a"` +Expected: both tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add pipeline/bench.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add Mode A: fixture suite runner + history writer + +`bully bench` now enumerates bench/fixtures/*, runs each fixture, and +appends one JSONL line to bench/history.jsonl with per-fixture phase +timings + aggregate totals. Human-readable summary to stdout by +default; --json emits the raw record. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 7: Mode A --compare + +**Files:** +- Modify: `pipeline/bench.py` +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing test** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_compare_reports_deltas_between_last_two_runs(tmp_path, capsys): + """--compare prints a delta table for the last two history entries.""" + from pipeline.bench import run_compare + + history = tmp_path / "h.jsonl" + older = { + "ts": "2026-04-15T10:00:00Z", + "git_sha": "aaa", + "fixtures": [ + {"name": "a", "wall_ms_p50": 10.0, "tokens": {"input": 100, + "method": "count_tokens"}}, + ], + "aggregates": { + "total_wall_ms_p50": 10.0, + "total_input_tokens": 100, + }, + } + newer = { + "ts": "2026-04-17T12:00:00Z", + "git_sha": "bbb", + "fixtures": [ + {"name": "a", "wall_ms_p50": 15.0, "tokens": {"input": 120, + "method": "count_tokens"}}, + ], + "aggregates": { + "total_wall_ms_p50": 15.0, + "total_input_tokens": 120, + }, + } + history.write_text(json.dumps(older) + "\n" + json.dumps(newer) + "\n") + + rc = run_compare(history_path=history) + assert rc == 0 + out = capsys.readouterr().out + assert "aaa" in out and "bbb" in out + assert "+5.00" in out or "+50.0" in out # wall delta + assert "+20" in out # token delta + + +def test_compare_fails_when_fewer_than_two_runs(tmp_path, capsys): + """--compare needs at least two runs to produce a delta.""" + from pipeline.bench import run_compare + + history = tmp_path / "h.jsonl" + history.write_text(json.dumps({"ts": "t", "fixtures": [], "aggregates": {}}) + "\n") + + rc = run_compare(history_path=history) + assert rc != 0 + err = capsys.readouterr().err + assert "two runs" in err.lower() or "2 runs" in err.lower() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py -v -k "compare"` +Expected: FAIL — `run_compare` not defined. + +- [ ] **Step 3: Add `run_compare`** + +Add to `pipeline/bench.py`: + +```python +def run_compare(*, history_path: Path) -> int: + """Print deltas between the last two runs in history_path.""" + if not history_path.is_file(): + sys.stderr.write(f"bench: history file not found: {history_path}\n") + return 1 + lines = [ + json.loads(line) + for line in history_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if len(lines) < 2: + sys.stderr.write( + "bench: --compare needs at least two runs in history\n" + ) + return 1 + + older, newer = lines[-2], lines[-1] + print(f"comparing {older.get('git_sha') or '?'} -> " + f"{newer.get('git_sha') or '?'}") + print(f" {older['ts']} -> {newer['ts']}") + print() + + fx_by_name = {f["name"]: f for f in newer.get("fixtures", [])} + old_by_name = {f["name"]: f for f in older.get("fixtures", [])} + all_names = sorted(set(fx_by_name) | set(old_by_name)) + + print(f" {'fixture':<32} {'Δ wall_ms':>12} {'Δ tokens':>12}") + for name in all_names: + new_fx = fx_by_name.get(name, {}) + old_fx = old_by_name.get(name, {}) + dw = new_fx.get("wall_ms_p50", 0) - old_fx.get("wall_ms_p50", 0) + dt = new_fx.get("tokens", {}).get("input", 0) - old_fx.get( + "tokens", {} + ).get("input", 0) + sign_w = "+" if dw >= 0 else "" + sign_t = "+" if dt >= 0 else "" + print(f" {name:<32} {sign_w}{dw:>11.2f} {sign_t}{dt:>11}") + + agg_new = newer.get("aggregates", {}) + agg_old = older.get("aggregates", {}) + dw_tot = agg_new.get("total_wall_ms_p50", 0) - agg_old.get( + "total_wall_ms_p50", 0 + ) + dt_tot = agg_new.get("total_input_tokens", 0) - agg_old.get( + "total_input_tokens", 0 + ) + print() + print(f" totals: Δwall={dw_tot:+.2f}ms Δtokens={dt_tot:+}") + return 0 +``` + +- [ ] **Step 4: Wire `--compare` into `main()`** + +In `pipeline/bench.py`, replace the compare stub in `main()`: + +```python + if args.compare: + return run_compare(history_path=Path(args.history)) +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v -k "compare"` +Expected: both tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add pipeline/bench.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add bench --compare: diff last two runs in history + +Per-fixture + aggregate deltas for wall time and input tokens. Fails +clearly when history has fewer than two runs. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 8: Mode B — config cost analysis + +**Files:** +- Modify: `pipeline/bench.py` +- Test: `pipeline/tests/test_bench.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_mode_b_reports_floor_and_per_rule(tmp_path, monkeypatch): + """Mode B computes floor, per-rule marginal, and diff scaling.""" + from pipeline.bench import run_mode_b + + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " - id: sem-long\n" + " description: 'A somewhat long description that should cost more tokens than a short one'\n" + " engine: semantic\n" + " scope: '**/*.py'\n" + " severity: error\n" + " - id: sem-short\n" + " description: short\n" + " engine: semantic\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " - id: script-only\n" + " description: scripted check\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: error\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + + result = run_mode_b(config_path=cfg, use_api=False, emit_json=True) + assert result["returncode"] == 0 + report = result["report"] + assert report["floor_tokens"] > 0 + assert len(report["per_rule"]) == 2 # two semantic rules + long_cost = next( + r["tokens"] for r in report["per_rule"] if r["id"] == "sem-long" + ) + short_cost = next( + r["tokens"] for r in report["per_rule"] if r["id"] == "sem-short" + ) + assert long_cost > short_cost + assert "diff_scaling" in report + sizes = [row["added_lines"] for row in report["diff_scaling"]] + assert sizes == [1, 10, 100, 1000] + # Monotonically non-decreasing as diff grows. + totals = [row["total_tokens"] for row in report["diff_scaling"]] + assert totals == sorted(totals) + # Script rule listed separately as zero-cost model-wise. + assert any(r["id"] == "script-only" for r in report["deterministic_rules"]) + + +def test_mode_b_handles_config_with_no_semantic_rules(tmp_path, monkeypatch): + """Empty-semantic config reports floor=0 and empty per-rule.""" + from pipeline.bench import run_mode_b + + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " - id: scripted\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + + result = run_mode_b(config_path=cfg, use_api=False, emit_json=True) + assert result["returncode"] == 0 + report = result["report"] + assert report["floor_tokens"] == 0 + assert report["per_rule"] == [] + assert len(report["deterministic_rules"]) == 1 + + +def test_mode_b_errors_when_config_missing(tmp_path, capsys): + """Missing config path yields a clear error.""" + from pipeline.bench import run_mode_b + + result = run_mode_b(config_path=tmp_path / "nope.yml", use_api=False) + assert result["returncode"] != 0 + err = capsys.readouterr().err + assert "not found" in err.lower() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest pipeline/tests/test_bench.py -v -k "mode_b"` +Expected: FAIL — `run_mode_b` not defined. + +- [ ] **Step 3: Add `run_mode_b`** + +Add to `pipeline/bench.py`: + +```python +def _synth_diff(added_lines: int, file_path: str = "src/synth.py") -> str: + """Build a unified diff that adds `added_lines` new lines to a file.""" + body = "".join(f"+line_{i}\n" for i in range(added_lines)) + return ( + f"--- a/{file_path}\n" + f"+++ b/{file_path}\n" + f"@@ -0,0 +1,{added_lines} @@\n" + + body + ) + + +def run_mode_b( + *, + config_path: Path, + use_api: bool = True, + emit_json: bool = False, +) -> dict: + """Analyze a .bully.yml's input-token cost. + + Returns {"returncode": int, "report": {...} | None}. + """ + from pipeline import pipeline as pl + + if not config_path.is_file(): + sys.stderr.write(f"bench: config not found: {config_path}\n") + return {"returncode": 1, "report": None} + + try: + rules = pl.parse_config(str(config_path)) + except pl.ConfigError as e: + sys.stderr.write(f"bench: config error: {e}\n") + return {"returncode": 1, "report": None} + + semantic_rules = [r for r in rules if r.engine == "semantic"] + deterministic = [r for r in rules if r.engine in ("script", "ast")] + system = load_evaluator_system_prompt() + + example_file = "src/example.py" + floor_payload = pl.build_semantic_payload( + example_file, "", [], [] + )["_evaluator_input"] + if not semantic_rules: + floor_tokens, method = 0, "n/a-no-semantic-rules" + else: + floor_tokens, method = count_tokens( + floor_payload, system=system, use_api=use_api + ) + + per_rule: list[dict] = [] + for r in semantic_rules: + payload = pl.build_semantic_payload( + example_file, "", [], [r] + )["_evaluator_input"] + tokens, _ = count_tokens(payload, system=system, use_api=use_api) + per_rule.append( + {"id": r.id, "description": r.description, + "tokens": tokens - floor_tokens} + ) + per_rule.sort(key=lambda x: x["tokens"], reverse=True) + + diff_scaling: list[dict] = [] + for size in (1, 10, 100, 1000): + payload = pl.build_semantic_payload( + example_file, _synth_diff(size, example_file), [], semantic_rules + )["_evaluator_input"] + tokens, _ = count_tokens(payload, system=system, use_api=use_api) + diff_scaling.append({"added_lines": size, "total_tokens": tokens}) + + scopes: dict[str, int] = {} + for r in semantic_rules: + payload = pl.build_semantic_payload( + example_file, "", [], [r] + )["_evaluator_input"] + tokens, _ = count_tokens(payload, system=system, use_api=use_api) + for glob in r.scope: + scopes[glob] = scopes.get(glob, 0) + (tokens - floor_tokens) + scope_rows = sorted( + ({"scope": g, "tokens": t} for g, t in scopes.items()), + key=lambda x: x["tokens"], + reverse=True, + ) + + report = { + "config": str(config_path.resolve()), + "method": method, + "floor_tokens": floor_tokens, + "per_rule": per_rule, + "diff_scaling": diff_scaling, + "scope_groups": scope_rows, + "deterministic_rules": [ + {"id": r.id, "engine": r.engine} for r in deterministic + ], + } + if emit_json: + print(json.dumps(report, indent=2)) + else: + _print_mode_b_report(report) + return {"returncode": 0, "report": report} + + +def _print_mode_b_report(report: dict) -> None: + print(f"config: {report['config']}") + print(f"method: {report['method']}") + print() + print(f"floor tokens (per dispatch): {report['floor_tokens']}") + print() + if report["per_rule"]: + print(f" {'rule':<30} {'tokens':>8}") + for row in report["per_rule"]: + print(f" {row['id']:<30} {row['tokens']:>8}") + else: + print(" no semantic rules") + print() + print("diff scaling (all semantic rules loaded):") + print(f" {'added_lines':<14} {'total_tokens':>12}") + for row in report["diff_scaling"]: + print(f" {row['added_lines']:<14} {row['total_tokens']:>12}") + print() + if report["deterministic_rules"]: + print("deterministic rules (0 model tokens):") + for row in report["deterministic_rules"]: + print(f" {row['id']} ({row['engine']})") +``` + +- [ ] **Step 4: Wire Mode B into `main()`** + +Replace the Mode B stub in `main()`: + +```python + if args.config: + result = run_mode_b( + config_path=Path(args.config), + use_api=not args.no_tokens, + emit_json=args.json, + ) + return result["returncode"] +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest pipeline/tests/test_bench.py -v -k "mode_b"` +Expected: all three tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add pipeline/bench.py pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add Mode B: config cost analysis + +`bully bench --config ` parses a .bully.yml and reports floor +tokens per dispatch, per-rule marginal cost (sorted), diff scaling at +1/10/100/1000 added lines, and a scope-grouped breakdown. Script and +ast rules are listed separately as zero model-token cost. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 9: Author the 8 bench fixtures + +**Files:** +- Create: `bench/fixtures/01-script-only-small-diff/config.yml` +- Create: `bench/fixtures/01-script-only-small-diff/fixture.json` +- Create: `bench/fixtures/02-ast-only-small-diff/config.yml` +- Create: `bench/fixtures/02-ast-only-small-diff/fixture.json` +- Create: `bench/fixtures/03-semantic-only-small-diff/config.yml` +- Create: `bench/fixtures/03-semantic-only-small-diff/fixture.json` +- Create: `bench/fixtures/04-mixed-engines/config.yml` +- Create: `bench/fixtures/04-mixed-engines/fixture.json` +- Create: `bench/fixtures/05-big-extends-chain/{parent-a.yml,parent-b.yml,config.yml,fixture.json}` +- Create: `bench/fixtures/06-many-semantic-rules/config.yml` +- Create: `bench/fixtures/06-many-semantic-rules/fixture.json` +- Create: `bench/fixtures/07-large-diff/config.yml` +- Create: `bench/fixtures/07-large-diff/fixture.json` +- Create: `bench/fixtures/08-auto-generated-skip/config.yml` +- Create: `bench/fixtures/08-auto-generated-skip/fixture.json` + +- [ ] **Step 1: Create fixture 01 — script-only-small-diff** + +`bench/fixtures/01-script-only-small-diff/config.yml`: + +```yaml +rules: + - id: no-print + description: Disallow bare print() in production Python code + engine: script + scope: "**/*.py" + severity: warning + script: "grep -n 'print(' {file} && exit 1 || exit 0" +``` + +`bench/fixtures/01-script-only-small-diff/fixture.json`: + +```json +{ + "name": "01-script-only-small-diff", + "description": "One script rule, small Python edit", + "file_path": "bench/fixtures/01-script-only-small-diff/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,3 +1,4 @@\n def main():\n+ x = 1\n return 0\n" +} +``` + +Also create `bench/fixtures/01-script-only-small-diff/target.py`: + +```python +def main(): + x = 1 + return 0 +``` + +- [ ] **Step 2: Create fixture 02 — ast-only-small-diff** + +`bench/fixtures/02-ast-only-small-diff/config.yml`: + +```yaml +rules: + - id: no-var + description: Prefer let/const over var in TypeScript + engine: ast + scope: "**/*.ts" + severity: warning + language: ts + pattern: "var $X = $Y" +``` + +`bench/fixtures/02-ast-only-small-diff/fixture.json`: + +```json +{ + "name": "02-ast-only-small-diff", + "description": "One ast rule, small TypeScript edit", + "file_path": "bench/fixtures/02-ast-only-small-diff/target.ts", + "edit_type": "Edit", + "diff": "--- a/target.ts\n+++ b/target.ts\n@@ -1,2 +1,3 @@\n function main() {\n+ const x = 1;\n }\n" +} +``` + +`bench/fixtures/02-ast-only-small-diff/target.ts`: + +```ts +function main() { + const x = 1; +} +``` + +- [ ] **Step 3: Create fixture 03 — semantic-only-small-diff** + +`bench/fixtures/03-semantic-only-small-diff/config.yml`: + +```yaml +rules: + - id: no-hardcoded-secrets + description: > + Do not hardcode API keys, tokens, passwords, or other secret credentials + in source code. Values that look like credentials should be loaded from + environment variables or a secret manager. + engine: semantic + scope: "**/*.py" + severity: error +``` + +`bench/fixtures/03-semantic-only-small-diff/fixture.json`: + +```json +{ + "name": "03-semantic-only-small-diff", + "description": "One semantic rule, small Python edit (no violation)", + "file_path": "bench/fixtures/03-semantic-only-small-diff/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,2 +1,3 @@\n def main():\n+ api_url = 'https://example.com'\n pass\n" +} +``` + +`bench/fixtures/03-semantic-only-small-diff/target.py`: + +```python +def main(): + api_url = 'https://example.com' + pass +``` + +- [ ] **Step 4: Create fixture 04 — mixed-engines** + +`bench/fixtures/04-mixed-engines/config.yml`: + +```yaml +rules: + - id: no-print + description: Disallow bare print() in production Python code + engine: script + scope: "**/*.py" + severity: warning + script: "grep -n 'print(' {file} && exit 1 || exit 0" + - id: no-eval + description: Disallow eval() usage + engine: ast + scope: "**/*.py" + severity: error + language: python + pattern: "eval($X)" + - id: no-hardcoded-secrets + description: Do not hardcode API keys, tokens, or passwords. + engine: semantic + scope: "**/*.py" + severity: error +``` + +`bench/fixtures/04-mixed-engines/fixture.json`: + +```json +{ + "name": "04-mixed-engines", + "description": "One rule per engine, moderate Python edit", + "file_path": "bench/fixtures/04-mixed-engines/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,4 +1,8 @@\n import os\n \n def main():\n+ x = os.getenv('X')\n+ y = {'a': 1, 'b': 2}\n+ for k, v in y.items():\n+ pass\n return 0\n" +} +``` + +`bench/fixtures/04-mixed-engines/target.py`: + +```python +import os + +def main(): + x = os.getenv('X') + y = {'a': 1, 'b': 2} + for k, v in y.items(): + pass + return 0 +``` + +- [ ] **Step 5: Create fixture 05 — big-extends-chain** + +`bench/fixtures/05-big-extends-chain/grandparent.yml`: + +```yaml +rules: + - id: grandparent-rule + description: A rule defined three levels up the extends chain + engine: script + scope: "**/*.py" + severity: warning + script: "exit 0" +``` + +`bench/fixtures/05-big-extends-chain/parent.yml`: + +```yaml +extends: grandparent.yml +rules: + - id: parent-rule + description: A rule defined two levels up + engine: script + scope: "**/*.py" + severity: warning + script: "exit 0" +``` + +`bench/fixtures/05-big-extends-chain/config.yml`: + +```yaml +extends: parent.yml +rules: + - id: local-rule + description: A rule defined at the leaf + engine: script + scope: "**/*.py" + severity: warning + script: "exit 0" +``` + +`bench/fixtures/05-big-extends-chain/fixture.json`: + +```json +{ + "name": "05-big-extends-chain", + "description": "Three-level extends chain; stresses parser + skip walker", + "file_path": "bench/fixtures/05-big-extends-chain/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1 +1,2 @@\n+x = 1\n y = 2\n" +} +``` + +`bench/fixtures/05-big-extends-chain/target.py`: + +```python +x = 1 +y = 2 +``` + +- [ ] **Step 6: Create fixture 06 — many-semantic-rules** + +`bench/fixtures/06-many-semantic-rules/config.yml` — 20 semantic rules with varied descriptions: + +```yaml +rules: + - id: sem-01 + description: Do not hardcode API keys, tokens, passwords, or other secret credentials in source code. + engine: semantic + scope: "**/*.py" + severity: error + - id: sem-02 + description: Avoid broad exception handlers that swallow errors silently. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-03 + description: Avoid mutable default arguments in function signatures. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-04 + description: Use context managers for file and resource handling. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-05 + description: Do not use assert for input validation in production code paths. + engine: semantic + scope: "**/*.py" + severity: error + - id: sem-06 + description: Avoid globally-mutable state; prefer explicit dependency injection. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-07 + description: Do not catch Exception or BaseException without re-raising. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-08 + description: Prefer f-strings over .format() and %-formatting. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-09 + description: Do not log sensitive values (passwords, tokens, PII). + engine: semantic + scope: "**/*.py" + severity: error + - id: sem-10 + description: Avoid running subprocess with shell=True unless the command is shlex-quoted. + engine: semantic + scope: "**/*.py" + severity: error + - id: sem-11 + description: Prefer pathlib.Path over os.path for new code. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-12 + description: Do not use print() for operational logging; use the logging module. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-13 + description: Avoid time.sleep() in request handlers. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-14 + description: Prefer list/dict/set comprehensions over map+filter when it improves clarity. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-15 + description: Do not silently drop exceptions in background tasks. + engine: semantic + scope: "**/*.py" + severity: error + - id: sem-16 + description: Check for file existence with Path.exists() rather than try/except FileNotFoundError when not needed. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-17 + description: Do not commit debugging breakpoint()/pdb.set_trace() calls. + engine: semantic + scope: "**/*.py" + severity: error + - id: sem-18 + description: Prefer dataclasses over manually-defined __init__/__eq__/__repr__ for data-holding classes. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-19 + description: Use type hints on public function signatures. + engine: semantic + scope: "**/*.py" + severity: warning + - id: sem-20 + description: Do not hard-code environment-specific paths; use configuration. + engine: semantic + scope: "**/*.py" + severity: warning +``` + +`bench/fixtures/06-many-semantic-rules/fixture.json`: + +```json +{ + "name": "06-many-semantic-rules", + "description": "20 semantic rules, small Python edit; stresses payload size", + "file_path": "bench/fixtures/06-many-semantic-rules/target.py", + "edit_type": "Edit", + "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,2 +1,3 @@\n import os\n+path = os.path.join('/tmp', 'f')\n x = 1\n" +} +``` + +`bench/fixtures/06-many-semantic-rules/target.py`: + +```python +import os +path = os.path.join('/tmp', 'f') +x = 1 +``` + +- [ ] **Step 7: Create fixture 07 — large-diff** + +`bench/fixtures/07-large-diff/config.yml`: + +```yaml +rules: + - id: no-hardcoded-secrets + description: Do not hardcode API keys, tokens, or passwords. + engine: semantic + scope: "**/*.py" + severity: error +``` + +For the large diff, generate a 500-line diff via Python one-liner in the fixture itself. Because fixture.json must hold the diff inline, use a script to generate it: + +```bash +python3 -c " +import json +added = ''.join(f'+ line_{i} = {i}\n' for i in range(500)) +header = '--- a/bench/fixtures/07-large-diff/target.py\n+++ b/bench/fixtures/07-large-diff/target.py\n@@ -1 +1,501 @@\n def main():\n' +print(json.dumps({ + 'name': '07-large-diff', + 'description': '500-line diff, one semantic rule; scales semantic payload', + 'file_path': 'bench/fixtures/07-large-diff/target.py', + 'edit_type': 'Edit', + 'diff': header + added, +}, indent=2)) +" > bench/fixtures/07-large-diff/fixture.json +``` + +And the target file: + +`bench/fixtures/07-large-diff/target.py`: + +```python +def main(): +``` + +- [ ] **Step 8: Create fixture 08 — auto-generated-skip** + +`bench/fixtures/08-auto-generated-skip/config.yml`: + +```yaml +rules: + - id: noop + description: A rule that should never be evaluated because skip shortcircuits first + engine: script + scope: "**/*.js" + severity: warning + script: "exit 0" +``` + +`bench/fixtures/08-auto-generated-skip/fixture.json`: + +```json +{ + "name": "08-auto-generated-skip", + "description": "File matches SKIP_PATTERNS (*.min.js); pipeline should short-circuit", + "file_path": "bench/fixtures/08-auto-generated-skip/bundle.min.js", + "edit_type": "Edit", + "diff": "--- a/bundle.min.js\n+++ b/bundle.min.js\n@@ -1 +1 @@\n-var a=1\n+var a=2\n" +} +``` + +`bench/fixtures/08-auto-generated-skip/bundle.min.js`: + +``` +var a=2 +``` + +- [ ] **Step 9: Run end-to-end bench against the new fixtures** + +Run: `python3 -m pipeline.pipeline bench --no-tokens` +Expected: emits a summary table, appends one line to `bench/history.jsonl`, exits 0. + +- [ ] **Step 10: Add an integration test that runs the real fixtures** + +Append to `pipeline/tests/test_bench.py`: + +```python +def test_real_fixtures_complete_successfully(tmp_path, monkeypatch): + """All authored fixtures under bench/fixtures/ run without errors.""" + from pipeline.bench import run_mode_a + + repo_root = Path(__file__).resolve().parent.parent.parent + fixtures_dir = repo_root / "bench" / "fixtures" + if not fixtures_dir.is_dir(): + import pytest + pytest.skip("fixtures directory not present") + + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + monkeypatch.chdir(repo_root) + history = tmp_path / "history.jsonl" + + rc = run_mode_a( + fixtures_dir=fixtures_dir, + history_path=history, + use_api=False, + iterations=2, + skip_cold_start=True, + ) + assert rc == 0 + record = json.loads(history.read_text().strip().splitlines()[-1]) + names = {f["name"] for f in record["fixtures"]} + assert len(names) >= 8 + # Auto-generated skip fixture should short-circuit (wall time near zero). + skip_fx = next( + f for f in record["fixtures"] if "auto-generated-skip" in f["name"] + ) + assert skip_fx["wall_ms_p50"] < 10.0 +``` + +- [ ] **Step 11: Run the integration test** + +Run: `pytest pipeline/tests/test_bench.py::test_real_fixtures_complete_successfully -v` +Expected: PASS. + +- [ ] **Step 12: Commit** + +```bash +git add bench/fixtures pipeline/tests/test_bench.py +git commit -m "$(cat <<'EOF' +Add 8 bench fixtures covering script/ast/semantic engines + +Fixtures exercise: single-engine smoke (01-03), mixed-engine path (04), +multi-level extends chain (05), payload size scaling (06, 07), and the +auto-generated skip short-circuit (08). Integration test runs all of +them against run_mode_a. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 10: Document the bench in README + +**Files:** +- Modify: `README.md` + +- [ ] **Step 1: Read the current README tail** + +Run: `wc -l README.md` +Record the final line number; the new section appends at end. + +- [ ] **Step 2: Append a bench section** + +Append to `README.md`: + +```markdown + +## Test Bench + +Bully ships with a local bench for watching its own speed and input-token cost over time. Two modes: + +### Mode A — fixture suite (regression trend) + +```bash +bully bench # run all bench/fixtures/, append to bench/history.jsonl +bully bench --compare # diff the last two runs +bully bench --no-tokens # skip Anthropic API call, use char-count proxy +bully bench --json # emit the raw run record on stdout +``` + +Results are written to `bench/history.jsonl`, one line per run. Commit a fresh run alongside changes that touch `pipeline/pipeline.py` to make speed/token impact visible in PRs. + +### Mode B — config cost analysis + +```bash +bully bench --config path/to/.bully.yml +``` + +Reports the input-token cost of the given config per invocation: floor tokens, per-rule marginal cost (sorted), diff scaling at 1/10/100/1000 added lines, and per-scope grouping. Useful for deciding whether a rule or rule pack earns its keep. + +### Real token counts + +Both modes use Anthropic's `messages/count_tokens` endpoint when `ANTHROPIC_API_KEY` is set and the optional `anthropic` SDK is installed (`pip install -e ".[bench]"`). Without either, both modes fall back to a `len(json.dumps(payload))` proxy and tag the output `method: proxy`. + +The bench does not make real model calls — only `count_tokens`, which is free and does not spend credits. +``` + +- [ ] **Step 3: Verify the README renders coherently** + +Run: `tail -50 README.md` +Expected: the bench section is present and the preceding section still closes cleanly. + +- [ ] **Step 4: Commit** + +```bash +git add README.md +git commit -m "$(cat <<'EOF' +Document the bench in README + +Covers Mode A (fixture suite + history.jsonl), Mode B (config cost +analysis), and the ANTHROPIC_API_KEY / optional anthropic SDK +requirement for real token counts. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Self-Review Checklist + +Before marking this plan complete: + +**Spec coverage:** +- [x] Two modes (A fixture, B config) — Tasks 6, 7, 8 +- [x] Fixture format (config.yml + fixture.json) — Task 9 +- [x] Phase-timer hook in pipeline — Task 1 +- [x] Token counting with count_tokens + proxy fallback — Task 4 +- [x] History JSONL with git sha, python version, aggregates — Task 6 +- [x] --compare mode — Task 7 +- [x] --no-tokens / --json flags — Tasks 2, 6, 8 +- [x] 8 hand-authored fixtures — Task 9 +- [x] Optional anthropic dep in pyproject — Task 2 +- [x] Trust-gate bypass via `BULLY_TRUST_ALL` env — used in Tasks 5, 6, 9 +- [x] Failure modes (missing config, missing fixture, no API key) — Tasks 3, 4, 6, 8 + +**Placeholder scan:** no TBD/TODO/"add appropriate error handling"/"similar to above" patterns. All code is spelled out. + +**Type consistency:** `Fixture` dataclass defined in Task 3 is used with the same field names in Tasks 5, 6, 9. `count_tokens` signature `(payload, *, system, use_api)` is consistent in Tasks 4, 5, 8. + +**Scope:** Single implementation plan. No decomposition needed. diff --git a/docs/superpowers/specs/2026-04-17-test-bench-design.md b/docs/superpowers/specs/2026-04-17-test-bench-design.md new file mode 100644 index 0000000..8213025 --- /dev/null +++ b/docs/superpowers/specs/2026-04-17-test-bench-design.md @@ -0,0 +1,254 @@ +# Bully Test Bench — Design + +**Date:** 2026-04-17 +**Status:** Approved, ready for implementation plan +**Owner:** Chris Arter + +## Goal + +A local-only bench for measuring two things: + +1. **Tool-level speed and input-token cost of `bully` itself**, captured per run into a versioned log so regressions and trends are visible over time. +2. **Per-config input-token cost estimate** for users who want to know what running `bully` will cost them in tokens before they adopt or extend a config. + +## Non-goals + +- CI integration or PR regression gates +- End-to-end cost (output tokens, real Sonnet round-trips) — input-token approximation is sufficient +- Multi-machine normalization or statistical comparison across environments +- Benchmarking of user codebases — the bench operates on self-contained fixtures or a single config file + +## Two modes + +### Mode A — Fixture bench (`bully bench`) + +Runs a fixed suite of `(config, file_path, diff)` fixtures through `run_pipeline` and records per-phase timings + payload input-token counts. Appends one line to `bench/history.jsonl` per run. + +**Purpose:** Chris's regression watch. Each commit can include a fresh bench line so trends are visible via `git log`. + +### Mode B — Config cost analysis (`bully bench --config `) + +Given any `.bully.yml`, compute a deterministic breakdown of what that config costs in input tokens per invocation. No fixtures, no history, no repetition — this is a one-shot report. + +**Purpose:** Users answering "will this config be expensive?" before adopting a rule pack or adding rules. + +## CLI surface + +Added to `pipeline/pipeline.py` argument parser, following existing subcommand patterns: + +``` +bully bench # Mode A, uses bench/fixtures/*.json +bully bench --config path/to/.bully.yml # Mode B +bully bench --json # Emit machine-readable output on stdout +bully bench --no-tokens # Skip Anthropic API call, use char-count proxy +bully bench --compare # Mode A only: diff latest two runs in history.jsonl +``` + +Default invocation (`bully bench`) writes a human-readable summary to stdout and appends to `bench/history.jsonl`. + +## Fixture format (Mode A) + +**Location:** `bench/fixtures//`, one directory per fixture. Two files per fixture: + +- `config.yml` — a real `.bully.yml` using the existing format the parser already understands +- `fixture.json` — metadata: `{name, description, file_path, edit_type, diff}` + +**Why two files:** the config parser is hand-rolled for a YAML-ish format; serializing a dict back into it would require a mini-writer that's easy to get wrong. Using real `.bully.yml` files means fixtures exercise the production parser end-to-end and stay human-editable. + +**Example `bench/fixtures/script-only-small-diff/config.yml`:** + +```yaml +rules: + - id: no-print + description: Disallow print() in production code + engine: script + scope: "**/*.py" + severity: error + script: "grep -n 'print(' {file} && exit 1 || exit 0" +``` + +**Example `bench/fixtures/script-only-small-diff/fixture.json`:** + +```json +{ + "name": "script-only-small-diff", + "description": "One script rule firing on a small Python edit", + "file_path": "src/app.py", + "edit_type": "Edit", + "diff": "--- a/src/app.py\n+++ b/src/app.py\n@@ -1,3 +1,4 @@\n def main():\n+ print('hello')\n return 0\n" +} +``` + +**Seed fixtures to author (~8):** + +1. `script-only-small-diff` — single script rule, small diff +2. `ast-only-small-diff` — single ast rule, small diff +3. `semantic-only-small-diff` — single semantic rule, small diff +4. `mixed-engines` — script + ast + semantic, mixed engines, medium diff +5. `big-extends-chain` — config that extends 3 levels deep +6. `many-semantic-rules` — 20 semantic rules (stress-test payload size) +7. `large-diff` — 500-line diff, single semantic rule (scaling check) +8. `auto-generated-skip` — file path matches `SKIP_PATTERNS`, should short-circuit + +## Harness design (`pipeline/bench.py`) + +New module, stdlib-only except for the optional `anthropic` import (gated). + +### Per-fixture execution (Mode A) + +For each fixture: + +1. **Stage config** — copy `config.yml` into a tempdir; set up an isolated `.bully/` directory so real telemetry isn't polluted. +2. **Pre-trust the staged config** — mark it trusted so the pipeline's trust gate doesn't short-circuit. Exact mechanism (writing the hash directly to the trust store, or invoking the existing trust-subcommand entry point) is an implementation-plan detail. +3. **Warm run** — one call to `run_pipeline` discarded (primes Python bytecode, filesystem cache). +4. **Timed runs** — N=5 calls. For each, wrap phases with `time.perf_counter_ns`: + - `parse_config` (includes extends chain) + - `effective_skip_patterns` + path-skip check + - `filter_rules` (scope-glob matching) + - Per-engine execution (script, ast) + - Semantic payload build + `_can_match_diff` filter +5. **Aggregate**: median and p95 per phase across the N=5 runs. +6. **Cold-start sample** — single `subprocess.run([sys.executable, str(pipeline_py), "--hook-mode"], input=...)` for this fixture, wall-clock only. Reports the Python startup cost real hooks pay. +7. **Token count** — build the semantic payload (if any semantic rules dispatched); call `count_tokens(payload)`. Record `tokens.input` and `tokens.method` (one of `"count_tokens"`, `"proxy"`). + +### Phase-timing mechanism + +Rather than monkey-patching `run_pipeline`, add a thin instrumentation layer: a `BenchTimer` context manager that `pipeline.run_pipeline` accepts as an optional hook parameter. Default is None (no overhead in normal runs). Bench passes in a timer that records each phase. + +This requires a small surgical change to `run_pipeline` to call into the timer at phase boundaries. The alternative — copy the pipeline loop into `bench.py` — duplicates logic and drifts. Better to take the tiny hook. + +### Token counting helper + +```python +def count_tokens(payload: dict, *, use_api: bool = True) -> tuple[int, str]: + """Return (token_count, method).""" +``` + +- If `use_api` and `ANTHROPIC_API_KEY` env var set and `anthropic` importable: + call `client.messages.count_tokens(model=, system=..., messages=[{"role": "user", "content": json.dumps(payload)}])` → `(n, "count_tokens")`. +- Else: `(len(json.dumps(payload)), "proxy")`. + +`system` prompt is the exact bully-evaluator system prompt, loaded from `agents/bully-evaluator.md` (frontmatter stripped). Same prompt real runs use, so token counts match reality. + +**Model ID:** pinned to `claude-sonnet-4-6` for now (matches today's production subagent dispatch). Exposed as a config constant at the top of `bench.py` so it can be updated alongside future model bumps without hunting through the code. + +### Config cost mode (Mode B) + +Given a config path: + +1. Parse the config. Warn if it doesn't exist or has `ConfigError`. +2. **Floor tokens**: build a semantic payload with `evaluate: []`, empty diff, `file: ""`. Count tokens. This is the fixed cost of any dispatch. +3. **Per-rule marginal cost**: for each semantic rule, build a payload containing just that one rule. Report `tokens - floor` as that rule's contribution. Sort descending, output as table. +4. **Diff scaling**: synthesize added-line diffs of sizes 1, 10, 100, 1000. For each, count tokens of a payload containing all semantic rules in the config. Report a small table. +5. **Script/ast summary**: count them, note "0 model tokens; local subprocess cost only." Include per-rule latency estimates only if the config has historical data in `.bully/log.jsonl`. +6. **Scope grouping**: group semantic rules by declared `scope` globs; report per-scope totals. + +Output: a formatted plain-text report. `--json` flag emits the structured data. + +## History record shape (Mode A) + +One line appended to `bench/history.jsonl` per run: + +```json +{ + "ts": "2026-04-17T14:32:10Z", + "git_sha": "6f66843", + "git_dirty": false, + "python_version": "3.12.3", + "anthropic_sdk_version": "0.40.0", + "machine": "darwin-23.6.0", + "fixtures": [ + { + "name": "script-only-small-diff", + "wall_ms_p50": 12.3, + "wall_ms_p95": 14.1, + "phases_ms": { + "parse_config": 2.1, + "skip_check": 0.3, + "filter_rules": 0.5, + "script_exec": 8.2, + "ast_exec": 0.0, + "semantic_build": 0.1 + }, + "cold_start_ms": 58.4, + "tokens": {"input": 0, "method": "n/a-no-semantic-rules"} + } + ], + "aggregates": { + "total_wall_ms_p50": 142.7, + "total_cold_start_ms": 465.2, + "total_input_tokens": 8420, + "tokens_method": "count_tokens" + } +} +``` + +Fields are flat enough to grep/awk but nested where it helps human reading. JSONL keeps one-line-per-run so `git diff` is clean. + +## Dependencies + +- Pipeline (`pipeline/pipeline.py`): stays stdlib-only. The phase-timer hook parameter is optional; default None is zero overhead. +- Bench (`pipeline/bench.py`): stdlib-only for mode logic. `anthropic` is an **optional** dep, imported lazily inside `count_tokens`, gated on API key presence. +- `pyproject.toml`: add an `[project.optional-dependencies] bench = ["anthropic>=0.40"]` entry. Install with `pip install -e ".[bench]"`. +- If `anthropic` isn't installed or no API key, the bench still runs, falls back to proxy, and tags the output accordingly. + +## File layout + +``` +pipeline/bench.py # New — harness and both modes +pipeline/pipeline.py # Small edit — add optional phase-timer hook +bench/fixtures//config.yml # New — 8 hand-authored fixtures (real .bully.yml) +bench/fixtures//fixture.json # New — paired metadata (diff, file_path, edit_type) +bench/history.jsonl # New — append-only run log, committed +pipeline/tests/test_bench.py # New — unit + integration tests +pyproject.toml # Edit — add optional `bench` extras +docs/superpowers/specs/2026-04-17-test-bench-design.md # This spec +``` + +## Testing strategy + +Unit tests in `pipeline/tests/test_bench.py`: + +- `count_tokens` with mocked `anthropic` client returns correct shape +- `count_tokens` with no API key falls back to proxy +- `count_tokens` with `anthropic` not importable falls back to proxy +- Fixture loader validates shape; rejects missing fields +- Phase-timer hook records each phase correctly +- History JSONL writer produces one line per run, parseable +- Config-cost report produces expected breakdown for a fixture config +- `--compare` diffs two adjacent runs correctly + +Integration tests: + +- Run full Mode A against the authored fixtures; assert output has all expected keys and plausible numbers (wall_ms > 0, tokens > 0 for semantic fixtures, etc.) +- Run Mode B against `examples/rules/django.yml`; assert floor > 0, per-rule table non-empty, diff scaling monotonically increasing + +Keep the bench test suite fast (under ~5s). Mock the Anthropic client — actual API calls are not part of the automated test suite. + +## Failure modes and handling + +| Condition | Behavior | +|---|---| +| No `ANTHROPIC_API_KEY` | Fall back to proxy, tag `tokens.method: "proxy"` | +| `anthropic` not installed | Same as above | +| `--config` path missing | Print error, exit 1 | +| `--config` has `ConfigError` | Print line-anchored error, exit 1 | +| Fixture file malformed JSON | Print path + parse error, exit 1 | +| Fixture has no matching rules | Report empty phase timings + 0 tokens, don't crash | +| `git` not available (for `git_sha`) | Record `"git_sha": null`, continue | + +## Open questions resolved + +- **Fixture source:** hand-authored, ~8 canonical, I'll write them. +- **Token definition:** real Anthropic `messages/count_tokens` when available; `len(json.dumps())` proxy fallback. Output tokens out of scope. +- **History format:** JSONL, one line per run, committed to repo at `bench/history.jsonl`. +- **Granularity:** per-fixture row with phase breakdown inside, plus aggregates. +- **Repetition:** N=5 per fixture, median + p95 reported. One discarded warm run. One subprocess sample for cold-start. +- **Output tokens:** deferred. Input-token approximation is sufficient for the current goal. + +## Future extensions (explicitly not in scope now) + +- `--full` flag: real Sonnet round-trip for output-token calibration +- CI integration / regression gates +- Normalized "relative to baseline" comparison view +- Historical trend plot rendered to SVG diff --git a/pipeline/bench.py b/pipeline/bench.py new file mode 100644 index 0000000..2b03e2b --- /dev/null +++ b/pipeline/bench.py @@ -0,0 +1,627 @@ +""" +Bully Test Bench + +Two modes: + bully bench -- run fixture suite, append to bench/history.jsonl + bully bench --config -- analyze token cost of any .bully.yml + +Stdlib-only except for the optional `anthropic` import, which is gated +behind API-key presence and falls back to a char-count proxy. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + + +class FixtureError(Exception): + """Raised when a fixture directory is malformed.""" + + +@dataclass(frozen=True) +class Fixture: + name: str + description: str + file_path: str + edit_type: str + diff: str + config_path: Path + + @property + def dir(self) -> Path: + return self.config_path.parent + + +def load_fixture(fixture_dir: Path) -> Fixture: + """Load a fixture from `/config.yml` + `/fixture.json`.""" + fixture_dir = Path(fixture_dir) + cfg = fixture_dir / "config.yml" + meta = fixture_dir / "fixture.json" + if not cfg.is_file(): + raise FixtureError(f"missing config.yml in {fixture_dir}") + if not meta.is_file(): + raise FixtureError(f"missing fixture.json in {fixture_dir}") + try: + data = json.loads(meta.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + raise FixtureError(f"malformed fixture.json in {fixture_dir}: {e}") from e + + required = ("name", "description", "file_path", "edit_type", "diff") + for key in required: + if key not in data: + raise FixtureError(f"fixture.json in {fixture_dir} missing field {key!r}") + + return Fixture( + name=data["name"], + description=data["description"], + file_path=data["file_path"], + edit_type=data["edit_type"], + diff=data["diff"], + config_path=cfg, + ) + + +def discover_fixtures(root: Path) -> list[Fixture]: + """Load every fixture subdirectory under `root`, sorted by name.""" + root = Path(root) + if not root.is_dir(): + return [] + out: list[Fixture] = [] + for child in sorted(root.iterdir()): + if not child.is_dir(): + continue + out.append(load_fixture(child)) + return out + + +BENCH_MODEL = "claude-sonnet-4-6" + + +def _repo_root() -> Path: + """Return the project root (directory holding the `agents/` dir). + + Assumes bench.py lives at /pipeline/bench.py. + """ + return Path(__file__).resolve().parent.parent + + +def _import_anthropic(): + """Import and return the anthropic module, or None if unavailable.""" + try: + import anthropic # type: ignore[import-not-found] + except ImportError: + return None + return anthropic + + +def load_evaluator_system_prompt() -> str: + """Load the bully-evaluator system prompt from agents/bully-evaluator.md. + + Strips the YAML frontmatter (everything between the first `---` pair). + """ + path = _repo_root() / "agents" / "bully-evaluator.md" + text = path.read_text(encoding="utf-8") + if text.startswith("---"): + # Find the closing frontmatter delimiter. + rest = text[3:] + end = rest.find("\n---") + if end != -1: + text = rest[end + 4 :] # past "\n---" + return text.lstrip("\n") + + +def count_tokens(payload: dict, *, system: str, use_api: bool = True) -> tuple[int, str]: + """Count input tokens for the given bully-evaluator payload. + + Returns (token_count, method) where method is 'count_tokens' or 'proxy'. + + Uses the Anthropic `messages/count_tokens` endpoint when + ANTHROPIC_API_KEY is set AND the anthropic SDK is importable AND + use_api is True. Falls back to `len(json.dumps(payload)) + len(system)`. + """ + api_key = os.environ.get("ANTHROPIC_API_KEY") + anthropic = _import_anthropic() if use_api else None + if use_api and api_key and anthropic is not None: + try: + client = anthropic.Anthropic(api_key=api_key) + resp = client.messages.count_tokens( + model=BENCH_MODEL, + system=system, + messages=[{"role": "user", "content": json.dumps(payload)}], + ) + return int(resp.input_tokens), "count_tokens" + except Exception: + # Any API failure -> proxy. Bench must not crash on transient errors. + pass + return len(json.dumps(payload)) + len(system), "proxy" + + +class PhaseTimer: + """Callable that records elapsed time for each named phase. + + Usage: + pt = PhaseTimer() + with pt("parse_config"): + ... + pt.results_ns() # {"parse_config": [12345, ...], ...} + """ + + def __init__(self) -> None: + self._samples: dict[str, list[int]] = {} + self._current: str | None = None + self._start_ns: int = 0 + + def __call__(self, name: str) -> PhaseTimer: + self._current = name + return self + + def __enter__(self) -> PhaseTimer: + self._start_ns = time.perf_counter_ns() + return self + + def __exit__(self, *a) -> bool: + elapsed = time.perf_counter_ns() - self._start_ns + assert self._current is not None + self._samples.setdefault(self._current, []).append(elapsed) + self._current = None + return False + + def results_ns(self) -> dict[str, list[int]]: + return dict(self._samples) + + +def _percentile(values: list[float], pct: float) -> float: + """Return the `pct`th percentile (0..100) by linear interpolation.""" + if not values: + return 0.0 + s = sorted(values) + if len(s) == 1: + return s[0] + k = (len(s) - 1) * (pct / 100.0) + lo = int(k) + hi = min(lo + 1, len(s) - 1) + frac = k - lo + return s[lo] + (s[hi] - s[lo]) * frac + + +def run_fixture( + fx: Fixture, + *, + iterations: int = 5, + use_api: bool = True, + skip_cold_start: bool = False, +) -> dict: + """Run one fixture: warm + N timed + cold-start + token count. + + Returns a per-fixture result dict suitable for the history JSONL. + """ + # Import here to avoid circular import at module load. + # When bench runs as pipeline.bench (package context), `import pipeline` + # imports the package (empty __init__). Fall back to the submodule. + import pipeline as _pl_pkg + + if hasattr(_pl_pkg, "run_pipeline"): + pl = _pl_pkg + else: + import pipeline.pipeline as pl # type: ignore[no-redef] + + cfg_path = str(fx.config_path) + + # Bundled fixtures are trusted by construction; short-circuit the trust + # gate so the bench doesn't require `bully trust` on every fixture + # config. Save-and-restore so this doesn't leak to callers that import + # the bench as a library. + prior_trust = os.environ.get("BULLY_TRUST_ALL") + os.environ["BULLY_TRUST_ALL"] = "1" + try: + # Warm run (discarded). + pl.run_pipeline(cfg_path, fx.file_path, fx.diff) + + # Timed runs. + wall_samples_ns: list[int] = [] + phase_samples_ns: dict[str, list[int]] = {} + for _ in range(iterations): + pt = PhaseTimer() + t0 = time.perf_counter_ns() + pl.run_pipeline(cfg_path, fx.file_path, fx.diff, phase_timer=pt) + wall_samples_ns.append(time.perf_counter_ns() - t0) + for name, samples in pt.results_ns().items(): + # Sum of this phase for this iteration (phases may re-enter). + phase_samples_ns.setdefault(name, []).append(sum(samples)) + + wall_ms = [ns / 1_000_000 for ns in wall_samples_ns] + phases_ms = { + name: statistics.median([ns / 1_000_000 for ns in samples]) + for name, samples in phase_samples_ns.items() + } + + # Cold-start: one subprocess invocation, wall-clock only. Use the + # default CLI path (not --hook-mode) so the subprocess doesn't block + # waiting on a Claude Code tool-hook payload. + cold_start_ms: float | None = None + if not skip_cold_start: + pipeline_py = Path(pl.__file__) + t0 = time.perf_counter_ns() + subprocess.run( + [ + sys.executable, + str(pipeline_py), + "--config", + cfg_path, + "--file", + fx.file_path, + "--diff", + fx.diff, + ], + capture_output=True, + text=True, + timeout=30, + ) + cold_start_ms = (time.perf_counter_ns() - t0) / 1_000_000 + + # Tokens: build the real semantic payload and count. Short-circuit + # when no semantic rules match -- a real run wouldn't dispatch at all. + rules = pl.parse_config(cfg_path) + matching = pl.filter_rules(rules, fx.file_path) + passed = [r.id for r in matching if r.engine in ("script", "ast")] + semantic = [r for r in matching if r.engine == "semantic"] + if semantic: + system = load_evaluator_system_prompt() + payload = pl.build_semantic_payload(fx.file_path, fx.diff, passed, semantic) + tokens, method = count_tokens( + payload["_evaluator_input"], system=system, use_api=use_api + ) + else: + tokens, method = 0, "n/a-no-semantic-rules" + + return { + "name": fx.name, + "description": fx.description, + "wall_ms_p50": statistics.median(wall_ms), + "wall_ms_p95": _percentile(wall_ms, 95), + "phases_ms": phases_ms, + "cold_start_ms": cold_start_ms, + "tokens": {"input": tokens, "method": method}, + } + finally: + if prior_trust is None: + os.environ.pop("BULLY_TRUST_ALL", None) + else: + os.environ["BULLY_TRUST_ALL"] = prior_trust + + +def _git_sha() -> str | None: + """Best-effort current git SHA; None if git unavailable or not a repo.""" + try: + r = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + timeout=2, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + if r.returncode != 0: + return None + return r.stdout.strip() or None + + +def _git_dirty() -> bool: + """True iff there are uncommitted changes.""" + try: + r = subprocess.run( + ["git", "status", "--porcelain"], + capture_output=True, + text=True, + timeout=2, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + return bool(r.stdout.strip()) + + +def _anthropic_sdk_version() -> str | None: + mod = _import_anthropic() + return getattr(mod, "__version__", None) if mod else None + + +def run_mode_a( + *, + fixtures_dir: Path, + history_path: Path, + use_api: bool = True, + iterations: int = 5, + skip_cold_start: bool = False, + emit_json: bool = False, +) -> int: + """Run every fixture in `fixtures_dir`, append a record to `history_path`.""" + fixtures = discover_fixtures(fixtures_dir) + if not fixtures: + sys.stderr.write(f"bench: no fixtures found in {fixtures_dir}\n") + return 1 + + results = [] + for fx in fixtures: + results.append( + run_fixture( + fx, + iterations=iterations, + use_api=use_api, + skip_cold_start=skip_cold_start, + ) + ) + + total_wall = sum(r["wall_ms_p50"] for r in results) + cold_vals = [r["cold_start_ms"] for r in results if r.get("cold_start_ms") is not None] + total_cold = sum(cold_vals) if cold_vals else None + total_tokens = sum(r["tokens"]["input"] for r in results) + methods = {r["tokens"]["method"] for r in results if r["tokens"]["input"]} + record = { + "ts": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"), + "git_sha": _git_sha(), + "git_dirty": _git_dirty(), + "python_version": platform.python_version(), + "anthropic_sdk_version": _anthropic_sdk_version(), + "machine": f"{platform.system().lower()}-{platform.release()}", + "fixtures": results, + "aggregates": { + "total_wall_ms_p50": total_wall, + "total_cold_start_ms": total_cold, + "total_input_tokens": total_tokens, + "tokens_method": next(iter(methods)) if len(methods) == 1 else "mixed", + }, + } + history_path.parent.mkdir(parents=True, exist_ok=True) + with history_path.open("a", encoding="utf-8") as f: + f.write(json.dumps(record) + "\n") + + if emit_json: + print(json.dumps(record, indent=2)) + else: + _print_mode_a_summary(record) + return 0 + + +def _print_mode_a_summary(record: dict) -> None: + """Render a human-readable summary of a Mode A record to stdout.""" + print(f"bench run @ {record['ts']} (sha={record['git_sha'] or '?'})") + print(f" python={record['python_version']} machine={record['machine']}") + print() + print(f" {'fixture':<32} {'wall_p50_ms':>12} {'cold_ms':>10} {'tokens':>9} method") + for r in record["fixtures"]: + cold = r.get("cold_start_ms") + cold_str = f"{cold:.1f}" if isinstance(cold, (int, float)) else "-" + print( + f" {r['name']:<32} {r['wall_ms_p50']:>12.2f} {cold_str:>10} " + f"{r['tokens']['input']:>9} {r['tokens']['method']}" + ) + agg = record["aggregates"] + print() + print( + f" totals: wall_p50={agg['total_wall_ms_p50']:.1f}ms " + f"cold={agg['total_cold_start_ms'] or 0:.1f}ms " + f"tokens={agg['total_input_tokens']} ({agg['tokens_method']})" + ) + + +def run_compare(*, history_path: Path) -> int: + """Print deltas between the last two runs in history_path.""" + if not history_path.is_file(): + sys.stderr.write(f"bench: history file not found: {history_path}\n") + return 1 + lines = [ + json.loads(line) + for line in history_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if len(lines) < 2: + sys.stderr.write("bench: --compare needs at least two runs in history\n") + return 1 + + older, newer = lines[-2], lines[-1] + print(f"comparing {older.get('git_sha') or '?'} -> {newer.get('git_sha') or '?'}") + print(f" {older['ts']} -> {newer['ts']}") + print() + + fx_by_name = {f["name"]: f for f in newer.get("fixtures", [])} + old_by_name = {f["name"]: f for f in older.get("fixtures", [])} + all_names = sorted(set(fx_by_name) | set(old_by_name)) + + print(f" {'fixture':<32} {'wall_ms delta':>14} {'tokens delta':>14}") + for name in all_names: + new_fx = fx_by_name.get(name, {}) + old_fx = old_by_name.get(name, {}) + dw = new_fx.get("wall_ms_p50", 0) - old_fx.get("wall_ms_p50", 0) + dt = new_fx.get("tokens", {}).get("input", 0) - old_fx.get("tokens", {}).get("input", 0) + sign_w = "+" if dw >= 0 else "" + sign_t = "+" if dt >= 0 else "" + print(f" {name:<32} {sign_w}{dw:>13.2f} {sign_t}{dt:>13}") + + agg_new = newer.get("aggregates", {}) + agg_old = older.get("aggregates", {}) + dw_tot = agg_new.get("total_wall_ms_p50", 0) - agg_old.get("total_wall_ms_p50", 0) + dt_tot = agg_new.get("total_input_tokens", 0) - agg_old.get("total_input_tokens", 0) + print() + print(f" totals: wall_delta={dw_tot:+.2f}ms tokens_delta={dt_tot:+}") + return 0 + + +def _synth_diff(added_lines: int, file_path: str = "src/synth.py") -> str: + """Build a unified diff that adds `added_lines` new lines to a file.""" + body = "".join(f"+line_{i}\n" for i in range(added_lines)) + return f"--- a/{file_path}\n+++ b/{file_path}\n@@ -0,0 +1,{added_lines} @@\n" + body + + +def run_mode_b( + *, + config_path: Path, + use_api: bool = True, + emit_json: bool = False, +) -> dict | None: + """Analyze a .bully.yml's input-token cost. + + Returns the report dict on success, None on failure (error already + printed to stderr). + """ + import pipeline as _pl_pkg + + if hasattr(_pl_pkg, "run_pipeline"): + pl = _pl_pkg + else: + import pipeline.pipeline as pl # type: ignore[no-redef] + + if not config_path.is_file(): + sys.stderr.write(f"bench: config not found: {config_path}\n") + return None + + try: + rules = pl.parse_config(str(config_path)) + except pl.ConfigError as e: + sys.stderr.write(f"bench: config error: {e}\n") + return None + + semantic_rules = [r for r in rules if r.engine == "semantic"] + deterministic = [r for r in rules if r.engine in ("script", "ast")] + system = load_evaluator_system_prompt() + + example_file = "src/example.py" + floor_payload = pl.build_semantic_payload(example_file, "", [], [])["_evaluator_input"] + if not semantic_rules: + floor_tokens, method = 0, "n/a-no-semantic-rules" + else: + floor_tokens, method = count_tokens(floor_payload, system=system, use_api=use_api) + + per_rule: list[dict] = [] + for r in semantic_rules: + payload = pl.build_semantic_payload(example_file, "", [], [r])["_evaluator_input"] + tokens, _ = count_tokens(payload, system=system, use_api=use_api) + per_rule.append({"id": r.id, "description": r.description, "tokens": tokens - floor_tokens}) + per_rule.sort(key=lambda x: x["tokens"], reverse=True) + + diff_scaling: list[dict] = [] + for size in (1, 10, 100, 1000): + payload = pl.build_semantic_payload( + example_file, _synth_diff(size, example_file), [], semantic_rules + )["_evaluator_input"] + tokens, _ = count_tokens(payload, system=system, use_api=use_api) + diff_scaling.append({"added_lines": size, "total_tokens": tokens}) + + scopes: dict[str, int] = {} + for r in semantic_rules: + payload = pl.build_semantic_payload(example_file, "", [], [r])["_evaluator_input"] + tokens, _ = count_tokens(payload, system=system, use_api=use_api) + for glob in r.scope: + scopes[glob] = scopes.get(glob, 0) + (tokens - floor_tokens) + scope_rows = sorted( + ({"scope": g, "tokens": t} for g, t in scopes.items()), + key=lambda x: x["tokens"], + reverse=True, + ) + + report = { + "config": str(config_path.resolve()), + "method": method, + "floor_tokens": floor_tokens, + "per_rule": per_rule, + "diff_scaling": diff_scaling, + "scope_groups": scope_rows, + "deterministic_rules": [{"id": r.id, "engine": r.engine} for r in deterministic], + } + if emit_json: + print(json.dumps(report, indent=2)) + else: + _print_mode_b_report(report) + return report + + +def _print_mode_b_report(report: dict) -> None: + print(f"config: {report['config']}") + print(f"method: {report['method']}") + print() + print(f"floor tokens (per dispatch): {report['floor_tokens']}") + print() + if report["per_rule"]: + print(f" {'rule':<30} {'tokens':>8}") + for row in report["per_rule"]: + print(f" {row['id']:<30} {row['tokens']:>8}") + else: + print(" no semantic rules") + print() + print("diff scaling (all semantic rules loaded):") + print(f" {'added_lines':<14} {'total_tokens':>12}") + for row in report["diff_scaling"]: + print(f" {row['added_lines']:<14} {row['total_tokens']:>12}") + print() + if report["deterministic_rules"]: + print("deterministic rules (0 model tokens):") + for row in report["deterministic_rules"]: + print(f" {row['id']} ({row['engine']})") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="bully bench", + description="Measure bully's speed and input-token cost.", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "--config", + help="Path to a .bully.yml; enables Mode B (config cost analysis).", + ) + mode.add_argument( + "--compare", + action="store_true", + help="Mode A only: diff the last two runs in bench/history.jsonl.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Emit machine-readable JSON instead of a formatted table.", + ) + parser.add_argument( + "--no-tokens", + action="store_true", + help="Skip Anthropic API call; use char-count proxy for token counts.", + ) + parser.add_argument( + "--fixtures-dir", + default="bench/fixtures", + help="Directory of fixture subdirectories (default: bench/fixtures).", + ) + parser.add_argument( + "--history", + default="bench/history.jsonl", + help="Path to history JSONL (default: bench/history.jsonl).", + ) + + args = parser.parse_args(argv) + + if args.config: + report = run_mode_b( + config_path=Path(args.config), + use_api=not args.no_tokens, + emit_json=args.json, + ) + return 0 if report is not None else 1 + if args.compare: + return run_compare(history_path=Path(args.history)) + return run_mode_a( + fixtures_dir=Path(args.fixtures_dir), + history_path=Path(args.history), + use_api=not args.no_tokens, + emit_json=args.json, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipeline/pipeline.py b/pipeline/pipeline.py index f52e3c0..e1c8072 100644 --- a/pipeline/pipeline.py +++ b/pipeline/pipeline.py @@ -1431,6 +1431,22 @@ def _append_record(log_path: Path, record: dict) -> None: # --------------------------------------------------------------------------- +class _NoopPhaseTimer: + """Default phase timer: every call is a no-op context manager.""" + + def __call__(self, name: str): + return self + + def __enter__(self): + return self + + def __exit__(self, *a) -> bool: + return False + + +_NOOP_PHASE_TIMER = _NoopPhaseTimer() + + def run_pipeline( config_path: str, file_path: str, @@ -1438,6 +1454,7 @@ def run_pipeline( rule_filter: set[str] | None = None, *, include_skipped: bool = False, + phase_timer=_NOOP_PHASE_TIMER, ) -> dict: """Full two-phase pipeline. @@ -1455,35 +1472,39 @@ def run_pipeline( log_path = _telemetry_path(config_path) # Short-circuit auto-generated files (built-in + user-global + project skip). - extra_skip = effective_skip_patterns(config_path)[len(SKIP_PATTERNS) :] - if _path_matches_skip(file_path, extra_patterns=extra_skip): - elapsed_ms = int((time.perf_counter() - start) * 1000) - result = {"status": "skipped", "file": file_path, "reason": "auto-generated"} - if log_path is not None: - _append_telemetry(log_path, file_path, "skipped", rule_records, elapsed_ms) - return result + with phase_timer("skip_check"): + extra_skip = effective_skip_patterns(config_path)[len(SKIP_PATTERNS) :] + if _path_matches_skip(file_path, extra_patterns=extra_skip): + elapsed_ms = int((time.perf_counter() - start) * 1000) + result = {"status": "skipped", "file": file_path, "reason": "auto-generated"} + if log_path is not None: + _append_telemetry(log_path, file_path, "skipped", rule_records, elapsed_ms) + return result # Trust gate: refuse to execute any rules from an un-reviewed config. - trust_status, trust_detail = _trust_status(config_path) - if trust_status != "trusted": - elapsed_ms = int((time.perf_counter() - start) * 1000) - result = { - "status": "untrusted", - "file": file_path, - "config": str(Path(config_path).resolve()), - "trust_status": trust_status, - "trust_detail": trust_detail, - } - if log_path is not None: - _append_telemetry( - log_path, file_path, f"untrusted:{trust_status}", rule_records, elapsed_ms - ) - return result + with phase_timer("trust_gate"): + trust_status, trust_detail = _trust_status(config_path) + if trust_status != "trusted": + elapsed_ms = int((time.perf_counter() - start) * 1000) + result = { + "status": "untrusted", + "file": file_path, + "config": str(Path(config_path).resolve()), + "trust_status": trust_status, + "trust_detail": trust_detail, + } + if log_path is not None: + _append_telemetry( + log_path, file_path, f"untrusted:{trust_status}", rule_records, elapsed_ms + ) + return result - rules = parse_config(config_path) - matching = filter_rules(rules, file_path) - if rule_filter: - matching = [r for r in matching if r.id in rule_filter] + with phase_timer("parse_config"): + rules = parse_config(config_path) + with phase_timer("filter_rules"): + matching = filter_rules(rules, file_path) + if rule_filter: + matching = [r for r in matching if r.id in rule_filter] def flush(status: str, result: dict) -> dict: if log_path is not None: @@ -1547,67 +1568,70 @@ def _run_deterministic( } ) - for rule in script_rules: - _run_deterministic( - rule, - lambda r=rule: execute_script_rule(r, file_path, diff), - "script", - ) + with phase_timer("script_exec"): + for rule in script_rules: + _run_deterministic( + rule, + lambda r=rule: execute_script_rule(r, file_path, diff), + "script", + ) - if ast_rules: - if ast_grep_available(): - for rule in ast_rules: - _run_deterministic( - rule, - lambda r=rule: execute_ast_rule(r, file_path), - "ast", + with phase_timer("ast_exec"): + if ast_rules: + if ast_grep_available(): + for rule in ast_rules: + _run_deterministic( + rule, + lambda r=rule: execute_ast_rule(r, file_path), + "ast", + ) + else: + sys.stderr.write( + "bully: engine:ast rules matched but ast-grep not on PATH; skipping. " + f"{_AST_GREP_INSTALL_HINT}\n" ) - else: - sys.stderr.write( - "bully: engine:ast rules matched but ast-grep not on PATH; skipping. " - f"{_AST_GREP_INSTALL_HINT}\n" - ) - for rule in ast_rules: + for rule in ast_rules: + rule_records.append( + { + "id": rule.id, + "engine": "ast", + "verdict": "skipped", + "severity": rule.severity, + "reason": "ast-grep-not-installed", + } + ) + + # Can't-match filters for semantic rules. + with phase_timer("semantic_build"): + dispatched_semantic: list[Rule] = [] + semantic_skipped: list[dict] = [] + for rule in semantic_rules: + ok, reason = _can_match_diff(rule, diff) + if ok: + dispatched_semantic.append(rule) rule_records.append( { "id": rule.id, - "engine": "ast", - "verdict": "skipped", + "engine": "semantic", + "verdict": "evaluate_requested", "severity": rule.severity, - "reason": "ast-grep-not-installed", } ) - - # Can't-match filters for semantic rules. - dispatched_semantic: list[Rule] = [] - semantic_skipped: list[dict] = [] - for rule in semantic_rules: - ok, reason = _can_match_diff(rule, diff) - if ok: - dispatched_semantic.append(rule) - rule_records.append( - { - "id": rule.id, - "engine": "semantic", - "verdict": "evaluate_requested", - "severity": rule.severity, - } - ) - else: - semantic_skipped.append({"rule": rule.id, "reason": reason}) - if log_path is not None: - _append_record( - log_path, - { - "ts": datetime.now(timezone.utc) - .isoformat(timespec="seconds") - .replace("+00:00", "Z"), - "type": "semantic_skipped", - "file": file_path, - "rule": rule.id, - "reason": reason, - }, - ) + else: + semantic_skipped.append({"rule": rule.id, "reason": reason}) + if log_path is not None: + _append_record( + log_path, + { + "ts": datetime.now(timezone.utc) + .isoformat(timespec="seconds") + .replace("+00:00", "Z"), + "type": "semantic_skipped", + "file": file_path, + "rule": rule.id, + "reason": reason, + }, + ) blocking = [v for v in all_violations if v.severity == "error"] @@ -2216,6 +2240,13 @@ def _hook_mode() -> int: def main() -> None: + # Short-circuit: `bully bench ...` dispatches to the bench CLI directly, + # bypassing the main parser (which uses a flat flag model). + if len(sys.argv) >= 2 and sys.argv[1] == "bench": + from pipeline.bench import main as bench_main + + sys.exit(bench_main(sys.argv[2:])) + args = _parse_args(sys.argv[1:]) # Subcommands. diff --git a/pipeline/tests/test_bench.py b/pipeline/tests/test_bench.py new file mode 100644 index 0000000..6fee771 --- /dev/null +++ b/pipeline/tests/test_bench.py @@ -0,0 +1,587 @@ +"""Bench harness tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import pipeline as pl + + +def test_phase_timer_default_is_noop_and_pipeline_still_runs(tmp_path, monkeypatch): + """run_pipeline works unchanged when no phase_timer is passed.""" + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " trivial:\n" + " description: trivial\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + target = tmp_path / "x.py" + target.write_text("x = 1\n") + result = pl.run_pipeline(str(cfg), str(target), diff="") + assert result["status"] == "pass" + + +def test_phase_timer_records_each_phase(tmp_path, monkeypatch): + """When a phase_timer is passed, it's called for each phase.""" + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " trivial:\n" + " description: trivial\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + target = tmp_path / "x.py" + target.write_text("x = 1\n") + + seen: list[str] = [] + + class Recorder: + def __call__(self, name): + seen.append(name) + return self + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + pl.run_pipeline(str(cfg), str(target), diff="", phase_timer=Recorder()) + expected = { + "skip_check", + "trust_gate", + "parse_config", + "filter_rules", + "script_exec", + "ast_exec", + "semantic_build", + } + assert set(seen) == expected + + +def test_bench_cli_entrypoint_exists(): + """`bully bench --help` exits cleanly (implies argparse wiring is in place).""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "pipeline.pipeline", "bench", "--help"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "bench" in (result.stdout + result.stderr).lower() + + +def test_bench_module_imports(): + """bench module loads without requiring anthropic.""" + import bench + + assert hasattr(bench, "main") + + +def test_load_fixture_reads_config_and_metadata(tmp_path): + """load_fixture returns a Fixture with config_path + metadata.""" + from bench import load_fixture + + fx_dir = tmp_path / "my-fixture" + fx_dir.mkdir() + (fx_dir / "config.yml").write_text( + "rules:\n" + " r1:\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (fx_dir / "fixture.json").write_text( + '{"name": "my-fixture", "description": "x", ' + '"file_path": "src/a.py", "edit_type": "Edit", "diff": "--- a\\n"}' + ) + fx = load_fixture(fx_dir) + assert fx.name == "my-fixture" + assert fx.file_path == "src/a.py" + assert fx.edit_type == "Edit" + assert fx.diff.startswith("--- a") + assert fx.config_path.name == "config.yml" + assert fx.config_path.exists() + + +def test_load_fixture_rejects_missing_files(tmp_path): + """load_fixture raises when either expected file is missing.""" + import pytest + + from bench import FixtureError, load_fixture + + fx_dir = tmp_path / "bad" + fx_dir.mkdir() + # Missing both files. + with pytest.raises(FixtureError, match="config.yml"): + load_fixture(fx_dir) + + +def test_load_fixture_rejects_malformed_json(tmp_path): + """load_fixture raises a clear error on malformed metadata JSON.""" + import pytest + + from bench import FixtureError, load_fixture + + fx_dir = tmp_path / "bad-json" + fx_dir.mkdir() + (fx_dir / "config.yml").write_text("rules: {}\n") + (fx_dir / "fixture.json").write_text("{not json") + with pytest.raises(FixtureError, match="fixture.json"): + load_fixture(fx_dir) + + +def test_discover_fixtures_lists_all_subdirs(tmp_path): + """discover_fixtures returns a sorted list of fixture directories.""" + from bench import discover_fixtures + + for name in ["zeta", "alpha", "mu"]: + d = tmp_path / name + d.mkdir() + (d / "config.yml").write_text("rules: {}\n") + (d / "fixture.json").write_text( + '{"name": "' + name + '", "description": "", ' + '"file_path": "a.py", "edit_type": "Edit", "diff": ""}' + ) + result = discover_fixtures(tmp_path) + assert [f.name for f in result] == ["alpha", "mu", "zeta"] + + +def test_count_tokens_proxy_when_no_api_key(monkeypatch): + """count_tokens falls back to char-count when no API key is present.""" + from bench import count_tokens + + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + payload = {"file": "x.py", "diff": "hello", "evaluate": []} + count, method = count_tokens(payload, system="sys prompt") + assert method == "proxy" + import json as _json + + assert count == len(_json.dumps(payload)) + len("sys prompt") + + +def test_count_tokens_proxy_when_anthropic_missing(monkeypatch): + """If `anthropic` is not importable, fall back to proxy even with key set.""" + import bench + + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + monkeypatch.setattr(bench, "_import_anthropic", lambda: None) + + payload = {"x": 1} + count, method = bench.count_tokens(payload, system="s") + assert method == "proxy" + + +def test_count_tokens_api_path(monkeypatch): + """With API key + anthropic client, call messages.count_tokens.""" + import bench + + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + + class FakeResp: + input_tokens = 42 + + class FakeMessages: + def count_tokens(self, **kwargs): + assert kwargs["model"] == bench.BENCH_MODEL + assert kwargs["system"] == "s" + assert "content" in kwargs["messages"][0] + return FakeResp() + + class FakeClient: + def __init__(self, api_key=None): + self.messages = FakeMessages() + + class FakeAnthropic: + Anthropic = FakeClient + + monkeypatch.setattr(bench, "_import_anthropic", lambda: FakeAnthropic) + + count, method = bench.count_tokens({"a": 1}, system="s") + assert count == 42 + assert method == "count_tokens" + + +def test_load_evaluator_system_prompt_strips_frontmatter(tmp_path, monkeypatch): + """System prompt is read from agents/bully-evaluator.md without frontmatter.""" + import bench + + agents_dir = tmp_path / "agents" + agents_dir.mkdir() + (agents_dir / "bully-evaluator.md").write_text( + "---\n" + "name: bully-evaluator\n" + "model: sonnet\n" + "---\n" + "\n" + "You are the evaluator. Apply each rule.\n" + ) + monkeypatch.setattr(bench, "_repo_root", lambda: tmp_path) + + text = bench.load_evaluator_system_prompt() + assert text.startswith("You are the evaluator") + assert "---" not in text + + +def test_phasetimer_records_all_phase_durations(): + """PhaseTimer records a list of (name, ns) for each phase invocation.""" + import time + + from bench import PhaseTimer + + pt = PhaseTimer() + with pt("a"): + time.sleep(0.001) + with pt("b"): + time.sleep(0.002) + results = pt.results_ns() + assert "a" in results + assert "b" in results + # Each phase tracked at least one sample. + assert len(results["a"]) == 1 + assert len(results["b"]) == 1 + assert results["a"][0] > 0 + assert results["b"][0] > 0 + + +def test_run_fixture_returns_structured_result(tmp_path, monkeypatch): + """run_fixture returns per-phase median/p95 plus cold-start and tokens.""" + from bench import Fixture, run_fixture + + fx_dir = tmp_path / "fx" + fx_dir.mkdir() + cfg = fx_dir / "config.yml" + cfg.write_text( + "rules:\n" + " r1:\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (fx_dir / "fixture.json").write_text( + '{"name": "fx", "description": "x", "file_path": "a.py", "edit_type": "Edit", "diff": ""}' + ) + target = tmp_path / "a.py" + target.write_text("x = 1\n") + + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + monkeypatch.chdir(tmp_path) + + fx = Fixture( + name="fx", + description="x", + file_path=str(target), + edit_type="Edit", + diff="", + config_path=cfg, + ) + result = run_fixture(fx, iterations=3, use_api=False, skip_cold_start=True) + assert result["name"] == "fx" + assert "wall_ms_p50" in result + assert "wall_ms_p95" in result + assert "phases_ms" in result + assert "skip_check" in result["phases_ms"] + assert "parse_config" in result["phases_ms"] + assert result["tokens"]["method"] == "n/a-no-semantic-rules" + assert result["tokens"]["input"] == 0 + + +def test_run_mode_a_writes_history_line(tmp_path, monkeypatch): + """Mode A writes one JSONL line per run with fixture results + aggregates.""" + import json as _json + + from bench import run_mode_a + + # Build two fixtures. + fixtures_root = tmp_path / "fixtures" + for name in ("a", "b"): + d = fixtures_root / name + d.mkdir(parents=True) + (d / "config.yml").write_text( + "rules:\n" + " r:\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (d / "fixture.json").write_text( + '{"name": "' + name + '", "description": "x",' + ' "file_path": "x.py", "edit_type": "Edit", "diff": ""}' + ) + (tmp_path / "x.py").write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + + history_path = tmp_path / "history.jsonl" + rc = run_mode_a( + fixtures_dir=fixtures_root, + history_path=history_path, + use_api=False, + iterations=2, + skip_cold_start=True, + emit_json=True, + ) + assert rc == 0 + assert history_path.is_file() + lines = history_path.read_text().strip().splitlines() + assert len(lines) == 1 + record = _json.loads(lines[0]) + assert "ts" in record + assert "fixtures" in record + assert len(record["fixtures"]) == 2 + assert {f["name"] for f in record["fixtures"]} == {"a", "b"} + assert "aggregates" in record + assert "total_wall_ms_p50" in record["aggregates"] + + +def test_run_mode_a_errors_when_no_fixtures(tmp_path, capsys): + """Mode A returns non-zero and prints an error when no fixtures exist.""" + from bench import run_mode_a + + history_path = tmp_path / "h.jsonl" + rc = run_mode_a( + fixtures_dir=tmp_path / "missing", + history_path=history_path, + use_api=False, + ) + assert rc != 0 + err = capsys.readouterr().err + assert "no fixtures" in err.lower() + + +def test_compare_reports_deltas_between_last_two_runs(tmp_path, capsys): + """--compare prints a delta table for the last two history entries.""" + import json as _json + + from bench import run_compare + + history = tmp_path / "h.jsonl" + older = { + "ts": "2026-04-15T10:00:00Z", + "git_sha": "aaa", + "fixtures": [ + {"name": "a", "wall_ms_p50": 10.0, "tokens": {"input": 100, "method": "count_tokens"}}, + ], + "aggregates": { + "total_wall_ms_p50": 10.0, + "total_input_tokens": 100, + }, + } + newer = { + "ts": "2026-04-17T12:00:00Z", + "git_sha": "bbb", + "fixtures": [ + {"name": "a", "wall_ms_p50": 15.0, "tokens": {"input": 120, "method": "count_tokens"}}, + ], + "aggregates": { + "total_wall_ms_p50": 15.0, + "total_input_tokens": 120, + }, + } + history.write_text(_json.dumps(older) + "\n" + _json.dumps(newer) + "\n") + + rc = run_compare(history_path=history) + assert rc == 0 + out = capsys.readouterr().out + assert "aaa" in out and "bbb" in out + # Wall delta is +5.00; token delta is +20 + assert "+5" in out + assert "+20" in out + + +def test_compare_fails_when_fewer_than_two_runs(tmp_path, capsys): + """--compare needs at least two runs to produce a delta.""" + import json as _json + + from bench import run_compare + + history = tmp_path / "h.jsonl" + history.write_text(_json.dumps({"ts": "t", "fixtures": [], "aggregates": {}}) + "\n") + + rc = run_compare(history_path=history) + assert rc != 0 + err = capsys.readouterr().err + assert "two runs" in err.lower() or "2 runs" in err.lower() + + +def test_mode_b_reports_floor_and_per_rule(tmp_path, monkeypatch): + """Mode B computes floor, per-rule marginal, and diff scaling.""" + from bench import run_mode_b + + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " sem-long:\n" + " description: 'A somewhat long description that should cost more tokens than a short one'\n" + " engine: semantic\n" + " scope: '**/*.py'\n" + " severity: error\n" + " sem-short:\n" + " description: short\n" + " engine: semantic\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script-only:\n" + " description: scripted check\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: error\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + + result = run_mode_b(config_path=cfg, use_api=False, emit_json=True) + assert result is not None + report = result + assert report["floor_tokens"] > 0 + assert len(report["per_rule"]) == 2 # two semantic rules + long_cost = next(r["tokens"] for r in report["per_rule"] if r["id"] == "sem-long") + short_cost = next(r["tokens"] for r in report["per_rule"] if r["id"] == "sem-short") + assert long_cost > short_cost + assert "diff_scaling" in report + sizes = [row["added_lines"] for row in report["diff_scaling"]] + assert sizes == [1, 10, 100, 1000] + # Monotonically non-decreasing as diff grows. + totals = [row["total_tokens"] for row in report["diff_scaling"]] + assert totals == sorted(totals) + # Script rule listed separately as zero-cost model-wise. + assert any(r["id"] == "script-only" for r in report["deterministic_rules"]) + + +def test_mode_b_handles_config_with_no_semantic_rules(tmp_path, monkeypatch): + """Empty-semantic config reports floor=0 and empty per-rule.""" + from bench import run_mode_b + + cfg = tmp_path / ".bully.yml" + cfg.write_text( + "rules:\n" + " scripted:\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + + result = run_mode_b(config_path=cfg, use_api=False, emit_json=True) + assert result is not None + report = result + assert report["floor_tokens"] == 0 + assert report["per_rule"] == [] + assert len(report["deterministic_rules"]) == 1 + + +def test_mode_b_errors_when_config_missing(tmp_path, capsys): + """Missing config path yields a clear error.""" + from bench import run_mode_b + + result = run_mode_b(config_path=tmp_path / "nope.yml", use_api=False) + assert result is None + err = capsys.readouterr().err + assert "not found" in err.lower() + + +def test_run_fixture_restores_trust_env_var(tmp_path, monkeypatch): + """run_fixture must not leak BULLY_TRUST_ALL to the caller's env.""" + from bench import Fixture, run_fixture + + fx_dir = tmp_path / "fx" + fx_dir.mkdir() + cfg = fx_dir / "config.yml" + cfg.write_text( + "rules:\n" + " r:\n" + " description: d\n" + " engine: script\n" + " scope: '**/*.py'\n" + " severity: warning\n" + " script: 'exit 0'\n" + ) + (fx_dir / "fixture.json").write_text( + '{"name": "fx", "description": "x", "file_path": "a.py", "edit_type": "Edit", "diff": ""}' + ) + target = tmp_path / "a.py" + target.write_text("x = 1\n") + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("BULLY_TRUST_ALL", raising=False) + + fx = Fixture( + name="fx", + description="x", + file_path=str(target), + edit_type="Edit", + diff="", + config_path=cfg, + ) + run_fixture(fx, iterations=1, use_api=False, skip_cold_start=True) + # Env var was absent before the call; it must be absent after. + import os as _os + + assert "BULLY_TRUST_ALL" not in _os.environ + + +def test_bench_config_and_compare_are_mutually_exclusive(): + """--config and --compare can't be used together.""" + import subprocess + + result = subprocess.run( + [sys.executable, "-m", "pipeline.pipeline", "bench", "--config", "foo.yml", "--compare"], + capture_output=True, + text=True, + ) + # argparse exits 2 on mutually-exclusive conflict. + assert result.returncode == 2 + assert "not allowed" in result.stderr.lower() or "mutually exclusive" in result.stderr.lower() + + +def test_real_fixtures_complete_successfully(tmp_path, monkeypatch): + """All authored fixtures under bench/fixtures/ run without errors.""" + import json as _json + + from bench import run_mode_a + + repo_root = Path(__file__).resolve().parent.parent.parent + fixtures_dir = repo_root / "bench" / "fixtures" + if not fixtures_dir.is_dir(): + import pytest + + pytest.skip("fixtures directory not present") + + monkeypatch.setenv("BULLY_TRUST_ALL", "1") + monkeypatch.chdir(repo_root) + history = tmp_path / "history.jsonl" + + rc = run_mode_a( + fixtures_dir=fixtures_dir, + history_path=history, + use_api=False, + iterations=2, + skip_cold_start=True, + ) + assert rc == 0 + record = _json.loads(history.read_text().strip().splitlines()[-1]) + names = {f["name"] for f in record["fixtures"]} + assert len(names) >= 8 + # Auto-generated skip fixture should short-circuit (wall time near zero). + skip_fx = next(f for f in record["fixtures"] if "auto-generated-skip" in f["name"]) + assert skip_fx["wall_ms_p50"] < 10.0 diff --git a/pyproject.toml b/pyproject.toml index dca48bb..c6389c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dev = [ "pre-commit>=3.7", "hypothesis>=6.100", ] +bench = ["anthropic>=0.40"] [project.scripts] bully = "pipeline.pipeline:main" @@ -33,7 +34,7 @@ packages = ["pipeline"] [tool.ruff] line-length = 100 target-version = "py310" -extend-exclude = ["pipeline/.pytest_cache"] +extend-exclude = ["pipeline/.pytest_cache", "bench/fixtures"] [tool.ruff.lint] select = [