From 73baf3d5d0e430e712821d07ca06afc5e969e1a4 Mon Sep 17 00:00:00 2001 From: "Abheer Kolhatkar (Tessl)" Date: Mon, 29 Jun 2026 15:04:46 +0100 Subject: [PATCH 1/2] fix(review-plugin-creator): make weight-sum eval criteria reliable to judge The weight-sum checklist items are pure arithmetic but graded by an LLM judge. Worded as "equals exactly 1.0", they were flaky: LLMs are unreliable at adding floats, and "exactly" invited the judge to fail a correct config over floating-point representation. Scores swung across reruns and the same check failed in both eval arms despite correct output. Reword each item so the judge enumerates every weight then sums them, passing only if the total is within 0.001 of 1.0. The tolerance accepts legitimate answers like 0.333 x 3 = 0.999 while still rejecting the planted off-by-0.1 errors, and keeping the pass on the arithmetic avoids a prose-based pass path. Applied across scenario-0, scenario-1, scenario-2, and scenario-5. Co-Authored-By: Claude Opus 4.8 (1M context) --- review-plugin-creator/evals/scenario-0/criteria.json | 4 ++-- review-plugin-creator/evals/scenario-1/criteria.json | 4 ++-- review-plugin-creator/evals/scenario-2/criteria.json | 4 ++-- review-plugin-creator/evals/scenario-5/criteria.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/review-plugin-creator/evals/scenario-0/criteria.json b/review-plugin-creator/evals/scenario-0/criteria.json index 228036b..176405f 100644 --- a/review-plugin-creator/evals/scenario-0/criteria.json +++ b/review-plugin-creator/evals/scenario-0/criteria.json @@ -49,7 +49,7 @@ }, { "name": "Dimension weights sum", - "description": "The 'weight' values of all dimensions in security.json sum to exactly 1.0", + "description": "List each dimension's 'weight' value in security.json, then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 10 }, { @@ -64,7 +64,7 @@ }, { "name": "config.json weight invariant", - "description": "In config.json, validation_weight + security judge weight equals exactly 1.0", + "description": "List validation_weight and the security judge weight from config.json, then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 10 }, { diff --git a/review-plugin-creator/evals/scenario-1/criteria.json b/review-plugin-creator/evals/scenario-1/criteria.json index 9c16f60..401e8ed 100644 --- a/review-plugin-creator/evals/scenario-1/criteria.json +++ b/review-plugin-creator/evals/scenario-1/criteria.json @@ -24,7 +24,7 @@ }, { "name": "Plugin-level weight invariant", - "description": "validation_weight + sum of all judge weights in config.json equals exactly 1.0", + "description": "List validation_weight and every judge weight from config.json, then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 12 }, { @@ -34,7 +34,7 @@ }, { "name": "Rubric dimension weights", - "description": "The dimension weights within the new code_examples.json rubric sum to exactly 1.0", + "description": "List the weight of every dimension in the new code_examples.json rubric, then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 10 }, { diff --git a/review-plugin-creator/evals/scenario-2/criteria.json b/review-plugin-creator/evals/scenario-2/criteria.json index 6280e7a..4384222 100644 --- a/review-plugin-creator/evals/scenario-2/criteria.json +++ b/review-plugin-creator/evals/scenario-2/criteria.json @@ -4,7 +4,7 @@ "checklist": [ { "name": "Config weights sum to 1.0", - "description": "In the corrected config.json, validation_weight + sum of all judge weights equals exactly 1.0", + "description": "In the corrected config.json, list validation_weight and every judge weight, then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 20 }, { @@ -14,7 +14,7 @@ }, { "name": "Rubric dimension weights sum to 1.0", - "description": "In the corrected content.json rubric, the sum of all dimensions[].weight values equals exactly 1.0", + "description": "In the corrected content.json rubric, list the weight of every entry in dimensions[], then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 20 }, { diff --git a/review-plugin-creator/evals/scenario-5/criteria.json b/review-plugin-creator/evals/scenario-5/criteria.json index 29d1692..fe767f4 100644 --- a/review-plugin-creator/evals/scenario-5/criteria.json +++ b/review-plugin-creator/evals/scenario-5/criteria.json @@ -14,7 +14,7 @@ }, { "name": "Weight invariant", - "description": "validation_weight + sum of all judge weights in config.json = 1.0 exactly", + "description": "List validation_weight and every judge weight from config.json, then add them together. Pass only if the sum is within 0.001 of 1.0.", "max_score": 12 }, { @@ -59,7 +59,7 @@ }, { "name": "Rubric dimension weights sum", - "description": "Dimension weights within each rubric sum to 1.0", + "description": "For each rubric file, list the weight of every dimension and add them together. Pass only if every rubric's sum is within 0.001 of 1.0.", "max_score": 10 }, { From a13aed62c316b222a9def7a4afedbdf76c5d3583 Mon Sep 17 00:00:00 2001 From: "Abheer Kolhatkar (Tessl)" Date: Mon, 29 Jun 2026 15:05:45 +0100 Subject: [PATCH 2/2] chore(review-plugin-creator): bump version to 0.3.1 Required by the changed-plugin version-bump gate. Co-Authored-By: Claude Opus 4.8 (1M context) --- review-plugin-creator/.tessl-plugin/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/review-plugin-creator/.tessl-plugin/plugin.json b/review-plugin-creator/.tessl-plugin/plugin.json index 43b26ce..8618903 100644 --- a/review-plugin-creator/.tessl-plugin/plugin.json +++ b/review-plugin-creator/.tessl-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "tessl/review-plugin-creator", - "version": "0.3.0", + "version": "0.3.1", "description": "Create custom Tessl reviewer plugins – fork the default rubric, build one from scratch, or derive its rubrics from evidence (existing skills, PR review feedback, agent logs). Scaffolds the plugin directory structure, authors rubrics and config.json, and validates the result with tessl review run.", "private": false, "skills": [