Add test bench for speed + input-token cost

.bully.yml

@@ -94,3 +94,13 @@ rules:
     scope: ["pipeline/pipeline.py", "pipeline/analyzer.py"]
     severity: error
     pattern: "subprocess.run($$$, shell=True, $$$)"
+
+  ruff-clean:
+    description: >
+      Python files must pass `ruff check`. --force-exclude makes ruff
+      respect pyproject's extend-exclude even when a single file is
+      passed, so test fixtures under bench/fixtures/ are skipped.
+    engine: script
+    scope: "*.py"
+    severity: error
+    script: "ruff check --force-exclude {file}"

README.md

@@ -243,3 +243,32 @@ MIT. See [LICENSE](LICENSE).
 ## Contributing
 
 Issues and PRs welcome. Please read [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) before participating, [SECURITY.md](SECURITY.md) for how to report vulnerabilities, and [CHANGELOG.md](CHANGELOG.md) for release notes.
+
+## Test Bench
+
+Bully ships with a local bench for watching its own speed and input-token cost over time. Two modes:
+
+### Mode A — fixture suite (regression trend)
+
+```bash
+bully bench                    # run all bench/fixtures/, append to bench/history.jsonl
+bully bench --compare          # diff the last two runs
+bully bench --no-tokens        # skip Anthropic API call, use char-count proxy
+bully bench --json             # emit the raw run record on stdout
+```
+
+Results are written to `bench/history.jsonl`, one line per run. Commit a fresh run alongside changes that touch `pipeline/pipeline.py` to make speed/token impact visible in PRs.
+
+### Mode B — config cost analysis
+
+```bash
+bully bench --config path/to/.bully.yml
+```
+
+Reports the input-token cost of the given config per invocation: floor tokens, per-rule marginal cost (sorted), diff scaling at 1/10/100/1000 added lines, and per-scope grouping. Useful for deciding whether a rule or rule pack earns its keep.
+
+### Real token counts
+
+Both modes use Anthropic's `messages/count_tokens` endpoint when `ANTHROPIC_API_KEY` is set and the optional `anthropic` SDK is installed (`pip install -e ".[bench]"`). Without either, both modes fall back to a `len(json.dumps(payload))` proxy and tag the output `method: proxy`.
+
+The bench does not make real model calls — only `count_tokens`, which is free and does not spend credits.

bench/fixtures/01-script-only-small-diff/config.yml

@@ -0,0 +1,7 @@
+rules:
+  no-print:
+    description: Disallow bare print() in production Python code
+    engine: script
+    scope: "**/*.py"
+    severity: warning
+    script: "grep -n 'print(' {file} && exit 1 || exit 0"

bench/fixtures/01-script-only-small-diff/fixture.json

@@ -0,0 +1,7 @@
+{
+  "name": "01-script-only-small-diff",
+  "description": "One script rule, small Python edit",
+  "file_path": "bench/fixtures/01-script-only-small-diff/target.py",
+  "edit_type": "Edit",
+  "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,3 +1,4 @@\n def main():\n+    x = 1\n     return 0\n"
+}

bench/fixtures/01-script-only-small-diff/target.py

@@ -0,0 +1,3 @@
+def main():
+    x = 1
+    return 0

bench/fixtures/02-ast-only-small-diff/config.yml

@@ -0,0 +1,8 @@
+rules:
+  no-var:
+    description: Prefer let/const over var in TypeScript
+    engine: ast
+    scope: "**/*.ts"
+    severity: warning
+    language: ts
+    pattern: "var $X = $Y"

bench/fixtures/02-ast-only-small-diff/fixture.json

@@ -0,0 +1,7 @@
+{
+  "name": "02-ast-only-small-diff",
+  "description": "One ast rule, small TypeScript edit",
+  "file_path": "bench/fixtures/02-ast-only-small-diff/target.ts",
+  "edit_type": "Edit",
+  "diff": "--- a/target.ts\n+++ b/target.ts\n@@ -1,2 +1,3 @@\n function main() {\n+  const x = 1;\n }\n"
+}

bench/fixtures/02-ast-only-small-diff/target.ts

@@ -0,0 +1,3 @@
+function main() {
+  const x = 1;
+}

bench/fixtures/03-semantic-only-small-diff/config.yml

@@ -0,0 +1,9 @@
+rules:
+  no-hardcoded-secrets:
+    description: >
+      Do not hardcode API keys, tokens, passwords, or other secret credentials
+      in source code. Values that look like credentials should be loaded from
+      environment variables or a secret manager.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error

bench/fixtures/03-semantic-only-small-diff/fixture.json

@@ -0,0 +1,7 @@
+{
+  "name": "03-semantic-only-small-diff",
+  "description": "One semantic rule, small Python edit (no violation)",
+  "file_path": "bench/fixtures/03-semantic-only-small-diff/target.py",
+  "edit_type": "Edit",
+  "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,2 +1,3 @@\n def main():\n+    api_url = 'https://example.com'\n     pass\n"
+}

bench/fixtures/03-semantic-only-small-diff/target.py

@@ -0,0 +1,3 @@
+def main():
+    api_url = 'https://example.com'
+    pass

bench/fixtures/04-mixed-engines/config.yml

@@ -0,0 +1,19 @@
+rules:
+  no-print:
+    description: Disallow bare print() in production Python code
+    engine: script
+    scope: "**/*.py"
+    severity: warning
+    script: "grep -n 'print(' {file} && exit 1 || exit 0"
+  no-eval:
+    description: Disallow eval() usage
+    engine: ast
+    scope: "**/*.py"
+    severity: error
+    language: python
+    pattern: "eval($X)"
+  no-hardcoded-secrets:
+    description: Do not hardcode API keys, tokens, or passwords.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error

bench/fixtures/04-mixed-engines/fixture.json

@@ -0,0 +1,7 @@
+{
+  "name": "04-mixed-engines",
+  "description": "One rule per engine, moderate Python edit",
+  "file_path": "bench/fixtures/04-mixed-engines/target.py",
+  "edit_type": "Edit",
+  "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,4 +1,8 @@\n import os\n \n def main():\n+    x = os.getenv('X')\n+    y = {'a': 1, 'b': 2}\n+    for k, v in y.items():\n+        pass\n     return 0\n"
+}

bench/fixtures/04-mixed-engines/target.py

@@ -0,0 +1,8 @@
+import os
+
+def main():
+    x = os.getenv('X')
+    y = {'a': 1, 'b': 2}
+    for k, v in y.items():
+        pass
+    return 0

bench/fixtures/05-big-extends-chain/config.yml

@@ -0,0 +1,8 @@
+extends: ["./parent.yml"]
+rules:
+  local-rule:
+    description: A rule defined at the leaf
+    engine: script
+    scope: "**/*.py"
+    severity: warning
+    script: "exit 0"

bench/fixtures/05-big-extends-chain/fixture.json

@@ -0,0 +1,7 @@
+{
+  "name": "05-big-extends-chain",
+  "description": "Three-level extends chain; stresses parser + skip walker",
+  "file_path": "bench/fixtures/05-big-extends-chain/target.py",
+  "edit_type": "Edit",
+  "diff": "--- a/target.py\n+++ b/target.py\n@@ -1 +1,2 @@\n+x = 1\n y = 2\n"
+}

bench/fixtures/05-big-extends-chain/grandparent.yml

@@ -0,0 +1,7 @@
+rules:
+  grandparent-rule:
+    description: A rule defined three levels up the extends chain
+    engine: script
+    scope: "**/*.py"
+    severity: warning
+    script: "exit 0"

bench/fixtures/05-big-extends-chain/parent.yml

@@ -0,0 +1,8 @@
+extends: ["./grandparent.yml"]
+rules:
+  parent-rule:
+    description: A rule defined two levels up
+    engine: script
+    scope: "**/*.py"
+    severity: warning
+    script: "exit 0"

bench/fixtures/05-big-extends-chain/target.py

@@ -0,0 +1,2 @@
+x = 1
+y = 2

bench/fixtures/06-many-semantic-rules/config.yml

@@ -0,0 +1,101 @@
+rules:
+  sem-01:
+    description: Do not hardcode API keys, tokens, passwords, or other secret credentials in source code.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error
+  sem-02:
+    description: Avoid broad exception handlers that swallow errors silently.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-03:
+    description: Avoid mutable default arguments in function signatures.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-04:
+    description: Use context managers for file and resource handling.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-05:
+    description: Do not use assert for input validation in production code paths.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error
+  sem-06:
+    description: Avoid globally-mutable state; prefer explicit dependency injection.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-07:
+    description: Do not catch Exception or BaseException without re-raising.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-08:
+    description: Prefer f-strings over .format() and %-formatting.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-09:
+    description: Do not log sensitive values (passwords, tokens, PII).
+    engine: semantic
+    scope: "**/*.py"
+    severity: error
+  sem-10:
+    description: Avoid running subprocess with shell=True unless the command is shlex-quoted.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error
+  sem-11:
+    description: Prefer pathlib.Path over os.path for new code.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-12:
+    description: Do not use print() for operational logging; use the logging module.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-13:
+    description: Avoid time.sleep() in request handlers.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-14:
+    description: Prefer list/dict/set comprehensions over map+filter when it improves clarity.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-15:
+    description: Do not silently drop exceptions in background tasks.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error
+  sem-16:
+    description: Check for file existence with Path.exists() rather than try/except FileNotFoundError when not needed.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-17:
+    description: Do not commit debugging breakpoint()/pdb.set_trace() calls.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error
+  sem-18:
+    description: Prefer dataclasses over manually-defined __init__/__eq__/__repr__ for data-holding classes.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-19:
+    description: Use type hints on public function signatures.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning
+  sem-20:
+    description: Do not hard-code environment-specific paths; use configuration.
+    engine: semantic
+    scope: "**/*.py"
+    severity: warning

bench/fixtures/06-many-semantic-rules/fixture.json

@@ -0,0 +1,7 @@
+{
+  "name": "06-many-semantic-rules",
+  "description": "20 semantic rules, small Python edit; stresses payload size",
+  "file_path": "bench/fixtures/06-many-semantic-rules/target.py",
+  "edit_type": "Edit",
+  "diff": "--- a/target.py\n+++ b/target.py\n@@ -1,2 +1,3 @@\n import os\n+path = os.path.join('/tmp', 'f')\n x = 1\n"
+}

bench/fixtures/06-many-semantic-rules/target.py

@@ -0,0 +1,3 @@
+import os
+path = os.path.join('/tmp', 'f')
+x = 1

bench/fixtures/07-large-diff/config.yml

@@ -0,0 +1,6 @@
+rules:
+  no-hardcoded-secrets:
+    description: Do not hardcode API keys, tokens, or passwords.
+    engine: semantic
+    scope: "**/*.py"
+    severity: error