diff --git a/README.md b/README.md index 9869d4d..d278091 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ These are R&D-maintained plugins intended for production use. | --- | --- | --- | | [`skill-optimizer`](skill-optimizer/) | Optimize your skills and plugins: review SKILL.md quality, generate eval scenarios, run evals, compare across models, diagnose gaps, and re-run until scores improve | [![](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftesslio%2Fproduct-plugins%2Fmain%2Fskill-optimizer%2F.tessl-plugin%2Fplugin.json&query=%24.version&label=tessl&color=blue&prefix=v)](https://tessl.io/registry/tessl/skill-optimizer) [![](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.tessl.io%2Fv1%2Fbadges%2Ftessl%2Fskill-optimizer)](https://tessl.io/registry/tessl/skill-optimizer/evals) | | [`review-plugin-creator`](review-plugin-creator/) | Create a custom `tessl review` rubric – fork the default rubric, build one from scratch, or derive it from evidence (existing skills, PR feedback, agent logs) | [![review-plugin-creator version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftesslio%2Fproduct-plugins%2Fmain%2Freview-plugin-creator%2F.tessl-plugin%2Fplugin.json&query=%24.version&label=tessl&color=blue&prefix=v)](https://tessl.io/registry/tessl/review-plugin-creator) [![review-plugin-creator evals](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.tessl.io%2Fv1%2Fbadges%2Ftessl%2Freview-plugin-creator)](https://tessl.io/registry/tessl/review-plugin-creator/evals) | +| [`default-skill-review`](default-skill-review/) | The default `tessl review` rubric – an agentic reviewer that scores a skill's description and content, the same reviewer `tessl review` uses out of the box. Score skills against it, or fork it into a custom rubric | [![default-skill-review version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftesslio%2Fproduct-plugins%2Fmain%2Fdefault-skill-review%2F.tessl-plugin%2Fplugin.json&query=%24.version&label=tessl&color=blue&prefix=v)](https://tessl.io/registry/tessl/default-skill-review) [![default-skill-review evals](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.tessl.io%2Fv1%2Fbadges%2Ftessl%2Fdefault-skill-review)](https://tessl.io/registry/tessl/default-skill-review/evals) | | [`api`](api/) | Find and call Tessl API endpoints token-efficiently, without loading the whole `openapi.json` spec into context | [![api version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftesslio%2Fproduct-plugins%2Fmain%2Fapi%2F.tessl-plugin%2Fplugin.json&query=%24.version&label=tessl&color=blue&prefix=v)](https://tessl.io/registry/tessl/api) [![api evals](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.tessl.io%2Fv1%2Fbadges%2Ftessl%2Fapi)](https://tessl.io/registry/tessl/api/evals) | ## Install diff --git a/default-skill-review/.tessl-plugin/plugin.json b/default-skill-review/.tessl-plugin/plugin.json new file mode 100644 index 0000000..c0fdc6d --- /dev/null +++ b/default-skill-review/.tessl-plugin/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "tessl/default-skill-review", + "version": "0.1.0", + "description": "The default Tessl review rubric. An agentic reviewer that reads a SKILL.md and scores it against rubrics for its description and its content, the same reviewer tessl review uses out of the box. Use it as the reference reviewer to score skills, or as the base to fork a custom rubric from.", + "private": false, + "skills": [ + "skills/skill-reviewer" + ] +} diff --git a/default-skill-review/README.md b/default-skill-review/README.md new file mode 100644 index 0000000..8ee90be --- /dev/null +++ b/default-skill-review/README.md @@ -0,0 +1,31 @@ +# default-skill-review + +The default `tessl review` rubric, packaged as a plugin. It is the exact reviewer `tessl review` uses out of the box. + +## Install + +```bash +tessl install tessl/default-skill-review +``` + +## What it does + +`tessl review` scores a skill against a **reviewer plugin**. This is that reviewer: an agent that reads a `SKILL.md` and scores it against rubrics grounded in Anthropic best practices. + +It ships two rubrics (their weights live in `config.json`): + +| Rubric | What it measures | +|--------|------------------| +| `description` | How well the skill's description drives activation | +| `content` | Quality of the SKILL.md body | + +Use it two ways: + +- **As the reference reviewer** — the reviewer `tessl review` resolves when no other reviewer is selected. +- **As a base to fork** — start from this rubric and adjust weights, anchors, or judges with [`tessl/review-plugin-creator`](../review-plugin-creator/). + +## Skills + +| Skill | Description | +|-------|-------------| +| `skill-reviewer` | An agent that reads a SKILL.md and scores it against the description and content rubrics | diff --git a/default-skill-review/skills/skill-reviewer/SKILL.md b/default-skill-review/skills/skill-reviewer/SKILL.md new file mode 100644 index 0000000..defbff3 --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/SKILL.md @@ -0,0 +1,102 @@ +--- +name: skill-reviewer +description: Evaluate a SKILL.md for quality. An agent that dynamically discovers the rubric files and scores the skill against each, producing one result per rubric. Use when running a skill review workflow or testing skill quality against configurable rubrics. +--- + +You are an expert skill quality evaluator. Your task is to assess the quality of a `SKILL.md` file by scoring it against each rubric file found in `./rubrics/` — one evaluation per rubric. + +## Step 1 — Read the skill + +Read `./SKILL.md`. Parse it into two parts: +- **YAML frontmatter**: everything between the opening and closing `---` delimiters. Extract `description` (and `name` if present). +- **Content body**: all markdown after the closing `---`. + +Then list any bundle files present in `./references/`, `./scripts/`, and `./assets/`. Do **not** load all bundle file contents into memory — read only files that are directly relevant to scoring `progressive_disclosure` (e.g., to verify references in the body are real files). + +## Step 2 — Discover rubrics and scoring config + +List all `.json` files in `./rubrics/`. Each file is one judge. Read each rubric file. Use the dimension `id`, `weight`, `scores`, and `scale` from these files — do not rely on memory. + +The file stem (filename without `.json`) is the judge name (e.g. `description.json` → judge name `description`). + +Read `./config.json`. This file contains: +- `judges`: a map of rubric stem → `{ weight }` expressing each judge's contribution to the final score + +Construct the `scoring.components` list (judge components only — one entry per rubric file in discovery order): +- `{ id: "", weight: config.judges[stem].weight, normalized: }` + +## Step 3 — Run judges + +For each rubric file discovered in Step 2, run one judge against the appropriate part of the skill: +- Rubrics whose `evaluation_target` is `"description"` → evaluate the frontmatter `description` field. +- Rubrics whose `evaluation_target` is `"content"` → evaluate the markdown body. +- For rubrics with any other `evaluation_target`, use your judgment about what part of the skill to evaluate. + +For each judge, follow this process: +1. Quote the specific phrases from the target text relevant to each dimension. +2. Compare against the rubric anchors — which example is the closest match? +3. Confirm why it is not the level above or below. +4. Assign your score (must be within `scale.min`–`scale.max`). + +Produce one evaluation object per judge: +```json +{ + "scores": { + "": { "score": , "reasoning": "<1-2 sentences>" } + }, + "overall_assessment": "<2-3 sentence summary>", + "suggestions": [] +} +``` + +Every dimension `id` from the rubric must appear in `scores`. For strong results leave `suggestions` as `[]`. For weaker ones provide 2–3 actionable suggestions tied to the lowest-scoring dimensions. + +## Step 4 — Compute scores + +For **each judge**: + +**Weighted score** (using the rubric's dimension weights): +```text +weightedScore = sum(dimension.score * dimension.weight) +``` +(All weights sum to 1.0.) + +**Normalized score** (maps weighted score to [0, 1] using rubric scale): +```text +normalizedScore = (weightedScore - scale.min) / (scale.max - scale.min) +``` + +## Step 5 — Write results + +Write `./results.json` conforming to `schemas/results.schema.json`. + +```json +{ + "judges": { + "": { + "success": true, + "scale": { "min": , "max": }, + "evaluation": , + "weightedScore": , + "normalizedScore": + } + }, + "scoring": { + "components": [ + { "id": "", "weight": config.judges[stem].weight, "normalized": } + ] + } +} +``` + +The `judges` object key is the rubric file stem (e.g. `description`, `content`). The scoring component `id` matches the same stem. + +If a judge fails (e.g., cannot parse the skill), set `success: false` and populate `errorMessage`. Do not omit the judge key — include it with `success: false`. + +## Important reminders + +- Read bundle files selectively — list them for context, read only what is needed. +- Every dimension `id` from each rubric must appear in the corresponding judge's `scores` object. +- All scores must be within `scale.min`–`scale.max`. +- `results.json` must be valid JSON conforming to `schemas/results.schema.json`. +- Do not include any commentary outside the `results.json` file. diff --git a/default-skill-review/skills/skill-reviewer/references/config.json b/default-skill-review/skills/skill-reviewer/references/config.json new file mode 100644 index 0000000..ff9b9e8 --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/references/config.json @@ -0,0 +1,8 @@ +{ + "$schema": "../schemas/config.schema.json", + "validation_weight": 0.2, + "judges": { + "description": { "weight": 0.4 }, + "content": { "weight": 0.4 } + } +} diff --git a/default-skill-review/skills/skill-reviewer/references/rubrics/content.json b/default-skill-review/skills/skill-reviewer/references/rubrics/content.json new file mode 100644 index 0000000..bd30cc6 --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/references/rubrics/content.json @@ -0,0 +1,136 @@ +{ + "$schema": "../schemas/rubric.schema.json", + "evaluation_target": "content", + "scope": "SKILL.md body (markdown content after YAML frontmatter)", + "scale": { + "min": 1, + "max": 3 + }, + "reference_examples": { + "judging_guidelines": [ + "Evaluate what is written, not what could be inferred or added.", + "Penalize explanations of concepts Claude already knows (e.g., what a PDF is, how libraries work, basic programming concepts).", + "Reward brevity that preserves clarity; penalize verbosity even if accurate.", + "Code examples should be executable and complete, not pseudocode, unless flexibility is explicitly justified.", + "Time-sensitive information (specific dates, version numbers) should penalize conciseness unless placed in an 'old patterns' or 'deprecated' section.", + "For simple skills (single task, no multi-step process), workflow clarity can score 3 if the single action is unambiguous.", + "For skills under 50 lines with no need for external references, progressive disclosure can score 3 with just well-organized sections.", + "Missing validation/verification steps in workflows involving destructive or batch operations should cap workflow clarity at 2.", + "When bundle files are provided, score `progressive_disclosure` against the actual bundle structure (file listing + any referenced paths) rather than on references alone." + ], + "good_overall_examples": [ + "## Quick start\n\nExtract text with pdfplumber:\n```python\nimport pdfplumber\nwith pdfplumber.open(\"file.pdf\") as pdf:\n text = pdf.pages[0].extract_text()\n```\n\n## Advanced features\n\n**Form filling**: See [FORMS.md](FORMS.md) for complete guide\n**API reference**: See [REFERENCE.md](REFERENCE.md) for all methods", + "## Commit message format\n\nGenerate commit messages following these examples:\n\n**Example 1:**\nInput: Added user authentication with JWT tokens\nOutput:\n```\nfeat(auth): implement JWT-based authentication\n\nAdd login endpoint and token validation middleware\n```\n\nFollow this style: type(scope): brief description, then detailed explanation.", + "## Document editing process\n\n1. Make your edits to `word/document.xml`\n2. **Validate immediately**: `python ooxml/scripts/validate.py unpacked_dir/`\n3. If validation fails:\n - Review the error message carefully\n - Fix the issues in the XML\n - Run validation again\n4. **Only proceed when validation passes**\n5. Rebuild: `python ooxml/scripts/pack.py unpacked_dir/ output.docx`" + ], + "bad_overall_examples": [ + "PDF (Portable Document Format) files are a common file format that contains text, images, and other content. To extract text from a PDF, you'll need to use a library. There are many libraries available for PDF processing, but we recommend pdfplumber because it's easy to use and handles most cases well. First, you'll need to install it using pip...", + "You can use pypdf, or pdfplumber, or PyMuPDF, or pdf2image, or camelot, or tabula-py depending on your needs. Each has different strengths...", + "Process the document.\n\nSee advanced.md for more details.\n\n(In advanced.md: See details.md for the actual instructions.)", + "If you're doing this before August 2025, use the old API. After August 2025, use the new API." + ] + }, + "dimensions": [ + { + "id": "conciseness", + "name": "Conciseness / Token Efficiency", + "weight": 0.3, + "question": "Does it respect token budget and assume Claude's intelligence?", + "rationale": "The context window is a public good. Every unnecessary token competes with conversation history and other context. Skills should add only what Claude doesn't already know.", + "scores": [ + { + "score": 1, + "anchor": "Verbose; explains concepts Claude knows; padded with unnecessary context", + "example": "PDF (Portable Document Format) files are a common file format that contains text, images, and other content. To extract text from a PDF, you'll need to use a library. There are many libraries available..." + }, + { + "score": 2, + "anchor": "Mostly efficient but includes some unnecessary explanation or could be tightened", + "example": "Use pdfplumber for text extraction. It's a good library that handles most PDF formats well.\n\n```python\nimport pdfplumber\nwith pdfplumber.open(\"file.pdf\") as pdf:\n text = pdf.pages[0].extract_text()\n```" + }, + { + "score": 3, + "anchor": "Lean and efficient; assumes Claude's competence; every token earns its place", + "example": "## Extract PDF text\n\n```python\nimport pdfplumber\nwith pdfplumber.open(\"file.pdf\") as pdf:\n text = pdf.pages[0].extract_text()\n```" + } + ] + }, + { + "id": "actionability", + "name": "Actionability", + "weight": 0.3, + "question": "Does it provide concrete, executable guidance vs vague direction?", + "rationale": "Skills succeed when Claude knows exactly what to do. Concrete code, specific commands, and clear examples beat abstract descriptions.", + "scores": [ + { + "score": 1, + "anchor": "Vague or abstract; no concrete code/commands; describes rather than instructs", + "example": "Use a PDF library to extract the text content from the document and process it as needed." + }, + { + "score": 2, + "anchor": "Some concrete guidance but incomplete; pseudocode instead of executable code; missing key details", + "example": "Use pdfplumber to extract text:\n```\n# pseudocode\nopen pdf\nfor each page:\n extract text\n```" + }, + { + "score": 3, + "anchor": "Fully executable code/commands; specific examples; copy-paste ready", + "example": "```python\nimport pdfplumber\nwith pdfplumber.open(\"file.pdf\") as pdf:\n for page in pdf.pages:\n print(page.extract_text())\n```\n\nFor tables: `page.extract_tables()` returns list of lists." + } + ] + }, + { + "id": "workflow_clarity", + "name": "Workflow Clarity", + "weight": 0.25, + "question": "Are multi-step processes clearly sequenced with validation checkpoints?", + "rationale": "Complex operations fail when steps are unclear or validation is missing. Clear workflows with explicit checkpoints catch errors early and guide Claude through fragile operations.", + "scores": [ + { + "score": 1, + "anchor": "Steps unclear or missing; no sequence for multi-step tasks; no validation mentioned for risky operations", + "example": "Edit the XML file, then rebuild the document. Make sure it's valid." + }, + { + "score": 2, + "anchor": "Steps listed but validation gaps; sequence present but checkpoints missing or implicit", + "example": "1. Unpack the document\n2. Edit word/document.xml\n3. Repack the document\n4. Test the output" + }, + { + "score": 3, + "anchor": "Clear sequence with explicit validation steps; feedback loops for error recovery; checklists for complex processes", + "example": "1. Unpack: `python scripts/unpack.py input.docx`\n2. Edit `word/document.xml`\n3. **Validate**: `python scripts/validate.py unpacked/`\n4. If errors: fix and re-validate\n5. **Only when valid**: `python scripts/pack.py unpacked/ output.docx`" + } + ] + }, + { + "id": "progressive_disclosure", + "name": "Progressive Disclosure", + "weight": 0.15, + "question": "Does it appropriately structure content across files with clear navigation?", + "rationale": "SKILL.md should be an overview pointing to detailed materials. References should be one level deep, clearly signaled, and organized for discovery.", + "scores": [ + { + "score": 1, + "anchor": "Monolithic wall of text; deeply nested references (2+ levels); poor organization", + "example": "See advanced.md for details.\n\n(In advanced.md: See details.md for the actual information.)\n\n(In details.md: The real content is here.)" + }, + { + "score": 2, + "anchor": "Some structure but could be better organized; references present but not clearly signaled; content that should be separate is inline", + "example": "## Overview\n[200 lines of API reference that could be in a separate file]\n\n## Advanced\nSee ADVANCED.md" + }, + { + "score": 3, + "anchor": "Clear overview with well-signaled one-level-deep references; content appropriately split; easy navigation", + "example": "## Quick start\n[Concise getting-started content]\n\n## Advanced features\n**Form filling**: See [FORMS.md](FORMS.md)\n**API reference**: See [REFERENCE.md](REFERENCE.md)\n**Examples**: See [EXAMPLES.md](EXAMPLES.md)" + } + ] + } + ], + "scoring_notes": { + "simple_skills": "For simple, single-purpose skills (under 50 lines, single task), workflow_clarity and progressive_disclosure can score 3 if the content is clear and well-organized, even without explicit multi-step workflows or external file references.", + "code_vs_instruction_skills": "Apply the same rubric to both. Code skills should have executable examples; instruction-only skills should have concrete, specific guidance. Absence of code in an instruction-only skill is not penalized if the guidance is actionable.", + "feedback_loops": "Feedback loops (validate -> fix -> retry) are especially important for: batch operations, destructive changes, XML/document manipulation, database operations. Missing feedback loops in these contexts should cap workflow_clarity at 2." + } +} diff --git a/default-skill-review/skills/skill-reviewer/references/rubrics/description.json b/default-skill-review/skills/skill-reviewer/references/rubrics/description.json new file mode 100644 index 0000000..8f6fece --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/references/rubrics/description.json @@ -0,0 +1,124 @@ +{ + "$schema": "../schemas/rubric.schema.json", + "evaluation_target": "description", + "scale": { + "min": 1, + "max": 3 + }, + "reference_examples": { + "judging_guidelines": [ + "Score only what is explicitly stated; do not infer capabilities or intentions.", + "Penalize vague fluff, buzzwords, and over-claims.", + "Do not reward verbosity; concise and clear beats long and padded.", + "Evaluate trigger terms as: 'would a user naturally say this when they need this skill?'", + "A missing 'Use when...' clause or equivalent explicit trigger guidance should cap completeness at 2.", + "Descriptions must use third person voice ('Processes files', 'Analyzes data'). Penalize first person ('I can help') or second person ('You can use this') by reducing specificity score by 1." + ], + "good_overall_examples": [ + "Extract text and tables from PDF files, fill forms, merge documents. Use when working with PDF files or when the user mentions PDFs, forms, or document extraction.", + "Analyze Excel spreadsheets, create pivot tables, generate charts. Use when analyzing Excel files, spreadsheets, tabular data, or .xlsx files.", + "Generate descriptive commit messages by analyzing git diffs. Use when the user asks for help writing commit messages or reviewing staged changes.", + "Presentation creation, editing, and analysis. When Claude needs to work with presentations (.pptx files) for: (1) Creating new presentations, (2) Modifying or editing content, (3) Working with layouts, (4) Adding comments or speaker notes, or any other presentation tasks.", + "Applies Anthropic's official brand colors and typography to any sort of artifact that may benefit from having Anthropic's look-and-feel. Use it when brand colors or style guidelines, visual formatting, or company design standards apply." + ], + "bad_overall_examples": [ + "Helps with documents", + "Processes data", + "Does stuff with files" + ] + }, + "dimensions": [ + { + "id": "specificity", + "name": "Specificity of Capabilities", + "weight": 0.2, + "question": "Does it describe concrete actions vs vague language?", + "scores": [ + { + "score": 1, + "anchor": "Vague or no actions; abstract language", + "example": "Helps with documents" + }, + { + "score": 2, + "anchor": "Names domain and some actions, but not comprehensive", + "example": "Processes PDF files and extracts content" + }, + { + "score": 3, + "anchor": "Lists multiple specific concrete actions", + "example": "Extract text and tables from PDF files, fill forms, merge documents" + } + ] + }, + { + "id": "trigger_term_quality", + "name": "Trigger Term Quality", + "weight": 0.3, + "question": "Does it include natural keywords users would actually say?", + "scores": [ + { + "score": 1, + "anchor": "No natural keywords; technical jargon or overly generic", + "example": "Handles document object model manipulation" + }, + { + "score": 2, + "anchor": "Some relevant keywords but missing common variations", + "example": "Works with PDF files" + }, + { + "score": 3, + "anchor": "Good coverage of natural terms users would say", + "example": "PDF files, PDFs, forms, document extraction, .pdf" + } + ] + }, + { + "id": "completeness", + "name": "Completeness", + "weight": 0.35, + "question": "Does it explicitly answer both 'what does this do' AND 'when should Claude use it'?", + "scores": [ + { + "score": 1, + "anchor": "Missing what OR when, or both very weak", + "example": "Helps with documents" + }, + { + "score": 2, + "anchor": "Has what, but when is missing or only implied", + "example": "Extract text and tables from PDF files, fill forms, merge documents" + }, + { + "score": 3, + "anchor": "Clearly answers both what AND when with explicit triggers", + "example": "Extract text and tables from PDF files, fill forms, merge documents. Use when working with PDF files or when the user mentions PDFs, forms, or document extraction." + } + ] + }, + { + "id": "distinctiveness_conflict_risk", + "name": "Distinctiveness / Conflict Risk", + "weight": 0.15, + "question": "Is it clearly distinguishable from other skills; unlikely to trigger for the wrong skill?", + "scores": [ + { + "score": 1, + "anchor": "Very generic; would conflict with many skills", + "example": "Helps with code and documents" + }, + { + "score": 2, + "anchor": "Somewhat specific but could still overlap with similar skills", + "example": "Works with document files" + }, + { + "score": 3, + "anchor": "Clear niche with distinct triggers; unlikely to conflict", + "example": "Extract text and tables from PDF files, fill forms, merge documents. Use when working with PDF files or when the user mentions PDFs." + } + ] + } + ] +} diff --git a/default-skill-review/skills/skill-reviewer/references/schemas/config.schema.json b/default-skill-review/skills/skill-reviewer/references/schemas/config.schema.json new file mode 100644 index 0000000..4618bb7 --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/references/schemas/config.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "config.schema.json", + "title": "SkillReviewConfig", + "description": "JSON Schema for the reviewer plugin configuration file (config.json).", + "type": "object", + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string" + }, + "validation_weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Weight of the validation component in the overall score (0–1). Default: 0.2." + }, + "judges": { + "type": "object", + "description": "Per-judge weight map keyed by rubric filename stem. Each key must match a rubric file in rubrics/. The judge weights plus validation_weight must sum to 1.0 (validated at publish time, not enforced by this schema).", + "additionalProperties": { + "type": "object", + "required": ["weight"], + "additionalProperties": false, + "properties": { + "weight": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + } + } + } + } +} diff --git a/default-skill-review/skills/skill-reviewer/references/schemas/results.schema.json b/default-skill-review/skills/skill-reviewer/references/schemas/results.schema.json new file mode 100644 index 0000000..7c475a6 --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/references/schemas/results.schema.json @@ -0,0 +1,142 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ReviewRunResults", + "description": "Output schema for the skill-reviewer plugin. Judges are a generic map keyed by rubric file stem (e.g. 'description', 'content') — one entry per rubric. Scoring contains one component per judge; each component's weight comes from that judge's entry in config.json's judges map.", + "type": "object", + "required": [], + "additionalProperties": false, + "properties": { + "judges": { + "type": "object", + "description": "Map of rubric stem → judge result. One entry per rubric file discovered in ./rubrics/. Absent when no judges have run yet.", + "additionalProperties": { "$ref": "#/$defs/SkillReviewJudgeResult" } + }, + "reviewScore": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "description": "Overall review score 0–100. Computed as floor(100 * weighted sum of component scores). Component weights come from config.json's judges map. Absent if the scoring block is incomplete." + }, + "scoring": { + "type": "object", + "description": "Per-component normalized scores — one entry per judge. Each component's weight comes from that judge's entry in config.json's judges map.", + "required": ["components"], + "additionalProperties": false, + "properties": { + "components": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "weight", "normalized"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Component id: the rubric file stem of the judge component." + }, + "weight": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Component weight: the judge's weight from config.json (config.judges[stem].weight)." + }, + "normalized": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Normalized score for this component (0–1): the judge's normalizedScore." + } + } + } + } + } + } + }, + "$defs": { + "SkillReviewDimensionScore": { + "type": "object", + "required": ["score", "reasoning"], + "additionalProperties": false, + "properties": { + "score": { + "type": "number", + "description": "Numeric score within the rubric's scale range." + }, + "reasoning": { + "type": "string", + "description": "1–2 sentence justification." + } + } + }, + "SkillReviewJudgeEvaluation": { + "type": "object", + "required": ["scores", "overall_assessment", "suggestions"], + "additionalProperties": false, + "properties": { + "scores": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "#/$defs/SkillReviewDimensionScore" }, + "description": "Map of dimension id → score. Must include every dimension from the rubric." + }, + "overall_assessment": { + "type": "string" + }, + "suggestions": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "SkillReviewJudgeResult": { + "type": "object", + "required": ["success"], + "additionalProperties": false, + "allOf": [ + { + "if": { "properties": { "success": { "const": true } }, "required": ["success"] }, + "then": { "required": ["evaluation", "weightedScore", "normalizedScore"] } + }, + { + "if": { "properties": { "success": { "const": false } }, "required": ["success"] }, + "then": { "required": ["errorMessage"] } + } + ], + "properties": { + "success": { + "type": "boolean", + "description": "True if the judge ran and produced a valid evaluation. False when the judge branch failed and returns an errorMessage." + }, + "errorMessage": { + "type": "string", + "description": "Error message if success is false." + }, + "evaluation": { + "$ref": "#/$defs/SkillReviewJudgeEvaluation", + "description": "Judge output. Present when success is true." + }, + "weightedScore": { + "type": "number", + "description": "Weighted average of dimension scores. sum(score * weight) / sum(weight)." + }, + "scale": { + "type": "object", + "description": "The rubric's scoring scale, copied from the rubric file's top-level scale field.", + "required": ["min", "max"], + "additionalProperties": false, + "properties": { + "min": { "type": "number" }, + "max": { "type": "number" } + } + }, + "normalizedScore": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "weightedScore normalized to [0, 1]: (weightedScore - scale.min) / (scale.max - scale.min)." + } + } + } + } +} diff --git a/default-skill-review/skills/skill-reviewer/references/schemas/rubric.schema.json b/default-skill-review/skills/skill-reviewer/references/schemas/rubric.schema.json new file mode 100644 index 0000000..4ea1509 --- /dev/null +++ b/default-skill-review/skills/skill-reviewer/references/schemas/rubric.schema.json @@ -0,0 +1,141 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "rubric.schema.json", + "title": "SkillReviewRubric", + "description": "JSON Schema for skill review rubric files (rubrics/description.json and rubrics/content.json).", + "type": "object", + "required": ["evaluation_target", "scale", "reference_examples", "dimensions"], + "additionalProperties": false, + "properties": { + "evaluation_target": { + "type": "string", + "description": "Which part of the skill (or artifact) this rubric evaluates. Built-in values are 'description' and 'content'; custom rubrics may use any string." + }, + "scope": { + "type": "string", + "description": "Human-readable description of what is being evaluated (content rubric only)." + }, + "scale": { + "type": "object", + "required": ["min", "max"], + "additionalProperties": false, + "properties": { + "min": { + "type": "number", + "description": "Minimum valid score (inclusive)." + }, + "max": { + "type": "number", + "description": "Maximum valid score (inclusive)." + } + } + }, + "reference_examples": { + "type": "object", + "required": ["judging_guidelines", "good_overall_examples", "bad_overall_examples"], + "additionalProperties": false, + "properties": { + "judging_guidelines": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "Ordered list of guidelines for the judge to follow." + }, + "good_overall_examples": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "Examples of high-quality skills for this evaluation target." + }, + "bad_overall_examples": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "Examples of low-quality skills for this evaluation target." + } + } + }, + "dimensions": { + "type": "array", + "minItems": 1, + "description": "Ordered list of evaluation dimensions. Weights must sum to 1.0.", + "items": { + "type": "object", + "required": ["id", "name", "weight", "question", "scores"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z][a-z0-9_]*$", + "description": "Unique snake_case identifier for this dimension. Used as key in judge output scores." + }, + "name": { + "type": "string", + "description": "Human-readable name for this dimension." + }, + "weight": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1, + "description": "Fractional weight of this dimension in the overall score. All weights must sum to 1.0." + }, + "question": { + "type": "string", + "description": "The core question the judge should answer for this dimension." + }, + "rationale": { + "type": "string", + "description": "Why this dimension matters (optional, content rubric dimensions)." + }, + "scores": { + "type": "array", + "minItems": 2, + "description": "Anchor definitions for each score level.", + "items": { + "type": "object", + "required": ["score", "anchor", "example"], + "additionalProperties": false, + "properties": { + "score": { + "type": "number", + "description": "The numeric score value." + }, + "anchor": { + "type": "string", + "description": "Brief description of what earns this score." + }, + "example": { + "type": "string", + "description": "Concrete example of content at this score level." + } + } + } + } + } + } + }, + "scoring_notes": { + "type": "object", + "required": ["simple_skills", "code_vs_instruction_skills", "feedback_loops"], + "additionalProperties": false, + "description": "Additional scoring guidance (content rubric only).", + "properties": { + "simple_skills": { + "type": "string" + }, + "code_vs_instruction_skills": { + "type": "string" + }, + "feedback_loops": { + "type": "string" + } + } + } + }, + "if": { + "properties": { "evaluation_target": { "const": "content" } } + }, + "then": { + "required": ["scope", "scoring_notes"] + } +} diff --git a/default-skill-review/tessl.json b/default-skill-review/tessl.json new file mode 100644 index 0000000..46b96a3 --- /dev/null +++ b/default-skill-review/tessl.json @@ -0,0 +1,5 @@ +{ + "name": "tessl/default-skill-review", + "mode": "vendored", + "dependencies": {} +}