diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md index c6e112f..3f7112b 100644 --- a/.gemini/styleguide.md +++ b/.gemini/styleguide.md @@ -53,6 +53,10 @@ The codebase consists of: - Do NOT include Claude-style names (`search_files`, `replace_in_file`, `Read`, `Edit`) — those are not valid in Gemini CLI. - No `model` or `color` field (Gemini CLI schema does not support them). +- The authoritative allowlist lives in `scripts/lib/gemini-tools.js` and is + enforced by `scripts/ci/validate-agents.js` (run on every PR). Update the + lib first if the Gemini CLI tool surface changes; this file is the + human-facing mirror. ## Documentation diff --git a/scripts/ci/validate-agents.js b/scripts/ci/validate-agents.js index fa2bb49..909bfc5 100644 --- a/scripts/ci/validate-agents.js +++ b/scripts/ci/validate-agents.js @@ -1,12 +1,19 @@ #!/usr/bin/env node /** - * Validate agent markdown files have required frontmatter + * Validate agent markdown files against the Gemini CLI agent loader rules. + * + * Catches Issue #34-class regressions: invalid tool names, MCP tools listed + * in frontmatter, Claude-style names, and unsupported keys (color/model). + * + * The validation rules live in scripts/lib/gemini-tools.js so they stay in + * sync with .gemini/styleguide.md. */ +'use strict'; + const path = require('path'); const { extractFrontmatter, validateFiles } = require('../lib/validator'); - -const REQUIRED_FIELDS = ['tools']; +const { validateAgentFrontmatter } = require('../lib/gemini-tools'); validateFiles({ dir: path.join(__dirname, '../../agents'), @@ -15,13 +22,7 @@ validateFiles({ validate(content) { const frontmatter = extractFrontmatter(content); if (!frontmatter) return ['Missing frontmatter']; - - const errors = []; - for (const field of REQUIRED_FIELDS) { - if (!frontmatter[field]) { - errors.push(`Missing required field: ${field}`); - } - } + const errors = validateAgentFrontmatter(frontmatter); return errors.length > 0 ? errors : null; } }); diff --git a/scripts/ci/validate-commands.js b/scripts/ci/validate-commands.js index bf9789f..b6ce069 100644 --- a/scripts/ci/validate-commands.js +++ b/scripts/ci/validate-commands.js @@ -1,17 +1,75 @@ #!/usr/bin/env node /** - * Validate command files are non-empty and readable + * Validate command TOML files. + * + * Beyond the original "non-empty file" check this enforces: + * - filename uses the `egc-` prefix (PR #38 — avoids collisions with + * Gemini CLI built-in commands like `/plan` and `/docs`) + * - a non-empty `description = "..."` field exists + * + * `EGC_PREFIX_EXEMPT` lists filenames that are intentionally allowed to + * skip the prefix (none today; kept as an explicit allowlist so future + * exceptions are reviewed instead of slipping through). */ +'use strict'; + const path = require('path'); const { validateFiles } = require('../lib/validator'); -validateFiles({ - dir: path.join(__dirname, '../../commands'), - label: 'command', - extension: '.toml', - validate(content) { - if (content.trim().length === 0) return ['Empty command file']; - return null; +const EGC_PREFIX_EXEMPT = new Set([]); + +// TOML allows four string forms for `description`: +// description = "basic" description = 'literal' +// description = """basic""" description = '''literal''' +// Try the triple-quoted form first; otherwise the non-greedy single-quote +// regex matches the empty span between the first two quotes of a `"""` +// opener and produces a false "Empty description field" error. +const DESCRIPTION_TRIPLE_RE = /^\s*description\s*=\s*("""|''')([\s\S]*?)\1/m; +const DESCRIPTION_SINGLE_RE = /^\s*description\s*=\s*(['"])([^]*?)\1/m; + +function extractDescriptionValue(content) { + const triple = DESCRIPTION_TRIPLE_RE.exec(content); + if (triple) return triple[2]; + const single = DESCRIPTION_SINGLE_RE.exec(content); + return single ? single[2] : null; +} + +function validateCommandContent(content, filename) { + const errors = []; + + if (content.trim().length === 0) { + errors.push('Empty command file'); + return errors; + } + + if (!EGC_PREFIX_EXEMPT.has(filename) && !filename.startsWith('egc-')) { + errors.push( + `Filename must start with "egc-" prefix (collides with Gemini CLI ` + + `built-ins otherwise — see PR #38).` + ); + } + + const description = extractDescriptionValue(content); + if (description === null) { + errors.push('Missing required field: description'); + } else if (description.trim().length === 0) { + errors.push('Empty description field'); } -}); + + return errors; +} + +if (require.main === module) { + validateFiles({ + dir: path.join(__dirname, '../../commands'), + label: 'command', + extension: '.toml', + validate(content, filename) { + const errors = validateCommandContent(content, filename); + return errors.length > 0 ? errors : null; + } + }); +} + +module.exports = { validateCommandContent }; diff --git a/scripts/lib/gemini-tools.js b/scripts/lib/gemini-tools.js new file mode 100644 index 0000000..467d1f6 --- /dev/null +++ b/scripts/lib/gemini-tools.js @@ -0,0 +1,209 @@ +/** + * Gemini CLI tool allowlist + frontmatter validation helpers. + * + * Single source of truth for what `tools:` and other frontmatter keys are + * accepted by the Gemini CLI agent loader. `.gemini/styleguide.md` mirrors + * this list for human-facing review guidance — keep both in sync. + * + * Issue #34 history: Gemini CLI rejects agents whose frontmatter references + * Claude-style names (Read, Edit), legacy names (search_files), MCP tools + * (mcp__*), or unsupported keys (color, model). The validators below catch + * those before they hit the loader. + */ + +'use strict'; + +const VALID_TOOLS = Object.freeze([ + 'read_file', + 'read_many_files', + 'write_file', + 'replace', + 'glob', + 'search_file_content', + 'list_directory', + 'run_shell_command', + 'save_memory', + 'web_fetch', + 'google_web_search' +]); + +const VALID_TOOL_SET = new Set(VALID_TOOLS); + +// Forbidden top-level frontmatter keys. The Gemini CLI agent schema only +// supports `name`, `description`, `tools`. Anything else (notably `color` +// and `model`) causes the loader to reject the file. +const FORBIDDEN_FRONTMATTER_KEYS = Object.freeze(['color', 'model']); + +const REQUIRED_FRONTMATTER_KEYS = Object.freeze(['name', 'description', 'tools']); + +// Claude Code → Gemini CLI tool name migration table. Used to produce +// actionable error messages when contributors copy frontmatter from a +// Claude-style agent. +const CLAUDE_TO_GEMINI = Object.freeze({ + Read: 'read_file', + Write: 'write_file', + Edit: 'replace', + Bash: 'run_shell_command', + Glob: 'glob', + Grep: 'search_file_content', + LS: 'list_directory', + WebFetch: 'web_fetch', + WebSearch: 'google_web_search', + search_files: 'search_file_content', + replace_in_file: 'replace' +}); + +/** + * Parse the value of a `tools:` frontmatter line into a list of tool names. + * + * Accepts both quoted and unquoted YAML inline arrays: + * tools: ["read_file", "write_file"] + * tools: [read_file, write_file] + * + * Returns null if the value is malformed (missing brackets, etc.) so callers + * can surface a frontmatter parse error instead of silently passing. + * + * @param {string} rawValue + * @returns {string[]|null} + */ +function parseToolsField(rawValue) { + if (typeof rawValue !== 'string') return null; + const trimmed = rawValue.trim(); + if (!trimmed.startsWith('[') || !trimmed.endsWith(']')) return null; + + const inner = trimmed.slice(1, -1).trim(); + if (inner.length === 0) return []; + + return inner + .split(',') + .map(part => part.trim().replace(/^["']|["']$/g, '')) + .filter(part => part.length > 0); +} + +/** + * Compute Levenshtein distance for "did you mean" suggestions. + * Bounded loop, no allocations beyond the working row. + */ +function levenshtein(a, b) { + if (a === b) return 0; + if (a.length === 0) return b.length; + if (b.length === 0) return a.length; + + let prev = new Array(b.length + 1); + let curr = new Array(b.length + 1); + for (let j = 0; j <= b.length; j++) prev[j] = j; + + for (let i = 1; i <= a.length; i++) { + curr[0] = i; + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + curr[j] = Math.min(curr[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost); + } + [prev, curr] = [curr, prev]; + } + return prev[b.length]; +} + +/** + * Suggest the nearest valid tool name for a typo. Returns null if no + * candidate is close enough (distance > 4) to be useful. + */ +function suggestTool(badName) { + if (CLAUDE_TO_GEMINI[badName]) return CLAUDE_TO_GEMINI[badName]; + + let best = null; + let bestDist = Infinity; + for (const candidate of VALID_TOOLS) { + const d = levenshtein(badName.toLowerCase(), candidate); + if (d < bestDist) { + bestDist = d; + best = candidate; + } + } + return bestDist <= 4 ? best : null; +} + +/** + * Validate a list of tool names against the Gemini CLI allowlist. + * Returns an array of human-readable error strings (empty on success). + * + * @param {string[]} tools + * @returns {string[]} + */ +function validateToolList(tools) { + const errors = []; + for (const tool of tools) { + if (VALID_TOOL_SET.has(tool)) continue; + + if (tool.startsWith('mcp__')) { + errors.push( + `Invalid tool "${tool}": MCP tools must not be listed in agent ` + + `frontmatter — they are auto-discovered from configured MCP servers.` + ); + continue; + } + + const suggestion = suggestTool(tool); + const hint = suggestion ? ` Did you mean "${suggestion}"?` : ''; + errors.push(`Invalid tool "${tool}".${hint}`); + } + return errors; +} + +/** + * Validate the full frontmatter object of an agent file. + * @param {object} frontmatter - parsed key/value pairs + * @returns {string[]} list of error messages (empty on success) + */ +function validateAgentFrontmatter(frontmatter) { + const errors = []; + const has = key => Object.prototype.hasOwnProperty.call(frontmatter, key); + const isEmpty = value => value === null || value === undefined || value === ''; + + for (const key of REQUIRED_FRONTMATTER_KEYS) { + if (!has(key)) { + errors.push(`Missing required field: ${key}`); + } else if (isEmpty(frontmatter[key])) { + errors.push(`Empty required field: ${key}`); + } + } + + for (const key of FORBIDDEN_FRONTMATTER_KEYS) { + if (has(key)) { + errors.push( + `Forbidden frontmatter key: "${key}" — Gemini CLI agent schema only ` + + `accepts ${REQUIRED_FRONTMATTER_KEYS.join(', ')}.` + ); + } + } + + // Run the tool-list parse only if the field has a non-empty value. The + // empty / missing cases are already covered above; running parseToolsField + // on `''` would just produce a redundant "Malformed tools field" message. + if (has('tools') && !isEmpty(frontmatter.tools)) { + const tools = parseToolsField(frontmatter.tools); + if (tools === null) { + errors.push( + `Malformed tools field: expected an inline YAML array like ` + + `[read_file, write_file].` + ); + } else if (tools.length === 0) { + errors.push('Empty tools array — agents must declare at least one tool.'); + } else { + errors.push(...validateToolList(tools)); + } + } + + return errors; +} + +module.exports = { + VALID_TOOLS, + FORBIDDEN_FRONTMATTER_KEYS, + REQUIRED_FRONTMATTER_KEYS, + CLAUDE_TO_GEMINI, + parseToolsField, + suggestTool, + validateToolList, + validateAgentFrontmatter +}; diff --git a/tests/lib/gemini-tools.test.js b/tests/lib/gemini-tools.test.js new file mode 100644 index 0000000..d4570c4 --- /dev/null +++ b/tests/lib/gemini-tools.test.js @@ -0,0 +1,224 @@ +/** + * Tests for scripts/lib/gemini-tools.js + * + * Run with: node tests/lib/gemini-tools.test.js + */ + +'use strict'; + +const assert = require('assert'); +const { + VALID_TOOLS, + FORBIDDEN_FRONTMATTER_KEYS, + CLAUDE_TO_GEMINI, + parseToolsField, + suggestTool, + validateToolList, + validateAgentFrontmatter +} = require('../../scripts/lib/gemini-tools'); + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + return true; + } catch (err) { + console.log(` ✗ ${name}`); + console.log(` Error: ${err.message}`); + return false; + } +} + +function runTests() { + console.log('\n=== Testing gemini-tools.js ===\n'); + + let passed = 0; + let failed = 0; + + console.log('parseToolsField:'); + + if (test('parses quoted inline array', () => { + const result = parseToolsField('["read_file", "write_file"]'); + assert.deepStrictEqual(result, ['read_file', 'write_file']); + })) passed++; else failed++; + + if (test('parses unquoted inline array', () => { + const result = parseToolsField('[read_file, write_file]'); + assert.deepStrictEqual(result, ['read_file', 'write_file']); + })) passed++; else failed++; + + if (test('parses single-quoted entries', () => { + const result = parseToolsField("['read_file', 'replace']"); + assert.deepStrictEqual(result, ['read_file', 'replace']); + })) passed++; else failed++; + + if (test('handles empty array', () => { + assert.deepStrictEqual(parseToolsField('[]'), []); + })) passed++; else failed++; + + if (test('returns null on missing brackets', () => { + assert.strictEqual(parseToolsField('read_file, write_file'), null); + })) passed++; else failed++; + + if (test('returns null on non-string input', () => { + assert.strictEqual(parseToolsField(undefined), null); + assert.strictEqual(parseToolsField(42), null); + })) passed++; else failed++; + + console.log('\nsuggestTool:'); + + if (test('maps Claude-style names to Gemini equivalents', () => { + assert.strictEqual(suggestTool('Read'), 'read_file'); + assert.strictEqual(suggestTool('Edit'), 'replace'); + assert.strictEqual(suggestTool('Bash'), 'run_shell_command'); + })) passed++; else failed++; + + if (test('maps legacy Gemini names from Issue #34', () => { + assert.strictEqual(suggestTool('search_files'), 'search_file_content'); + assert.strictEqual(suggestTool('replace_in_file'), 'replace'); + })) passed++; else failed++; + + if (test('finds nearest match for typos', () => { + assert.strictEqual(suggestTool('read_files'), 'read_file'); + assert.strictEqual(suggestTool('list_directorys'), 'list_directory'); + })) passed++; else failed++; + + if (test('returns null for far-off names', () => { + assert.strictEqual(suggestTool('completely_unrelated_xyz'), null); + })) passed++; else failed++; + + console.log('\nvalidateToolList:'); + + if (test('passes a fully valid list', () => { + assert.deepStrictEqual(validateToolList(['read_file', 'write_file']), []); + })) passed++; else failed++; + + if (test('flags MCP tools with dedicated message', () => { + const errors = validateToolList(['mcp__slack__send_message']); + assert.strictEqual(errors.length, 1); + assert.ok(errors[0].includes('MCP tools must not be listed')); + })) passed++; else failed++; + + if (test('flags Claude-style names with did-you-mean hint', () => { + const errors = validateToolList(['Read']); + assert.strictEqual(errors.length, 1); + assert.ok(errors[0].includes('Did you mean "read_file"?')); + })) passed++; else failed++; + + if (test('flags legacy names with hint', () => { + const errors = validateToolList(['search_files']); + assert.strictEqual(errors.length, 1); + assert.ok(errors[0].includes('Did you mean "search_file_content"?')); + })) passed++; else failed++; + + if (test('reports each invalid tool independently', () => { + const errors = validateToolList(['read_file', 'Edit', 'mcp__foo']); + assert.strictEqual(errors.length, 2); + })) passed++; else failed++; + + console.log('\nvalidateAgentFrontmatter:'); + + if (test('accepts a well-formed frontmatter', () => { + const errors = validateAgentFrontmatter({ + name: 'sample', + description: 'A sample agent.', + tools: '[read_file, write_file]' + }); + assert.deepStrictEqual(errors, []); + })) passed++; else failed++; + + if (test('reports missing required keys', () => { + const errors = validateAgentFrontmatter({}); + const missing = errors.filter(e => e.startsWith('Missing required field')); + assert.strictEqual(missing.length, 3); + })) passed++; else failed++; + + if (test('distinguishes empty value from missing key', () => { + const errors = validateAgentFrontmatter({ + name: '', + description: '', + tools: '' + }); + const empty = errors.filter(e => e.startsWith('Empty required field')); + const missing = errors.filter(e => e.startsWith('Missing required field')); + assert.strictEqual(empty.length, 3); + assert.strictEqual(missing.length, 0); + })) passed++; else failed++; + + if (test('does not produce a redundant Malformed tools error when tools is empty', () => { + const errors = validateAgentFrontmatter({ + name: 'x', + description: 'x', + tools: '' + }); + assert.ok(errors.some(e => e === 'Empty required field: tools')); + assert.ok(!errors.some(e => e.includes('Malformed tools field'))); + })) passed++; else failed++; + + if (test('rejects forbidden keys (color, model)', () => { + const errors = validateAgentFrontmatter({ + name: 'x', + description: 'x', + tools: '[read_file]', + color: 'blue', + model: 'sonnet' + }); + assert.ok(errors.some(e => e.includes('"color"'))); + assert.ok(errors.some(e => e.includes('"model"'))); + })) passed++; else failed++; + + if (test('rejects malformed tools field', () => { + const errors = validateAgentFrontmatter({ + name: 'x', + description: 'x', + tools: 'read_file, write_file' + }); + assert.ok(errors.some(e => e.includes('Malformed tools field'))); + })) passed++; else failed++; + + if (test('rejects empty tools array', () => { + const errors = validateAgentFrontmatter({ + name: 'x', + description: 'x', + tools: '[]' + }); + assert.ok(errors.some(e => e.includes('Empty tools array'))); + })) passed++; else failed++; + + if (test('rejects MCP tool in frontmatter', () => { + const errors = validateAgentFrontmatter({ + name: 'x', + description: 'x', + tools: '[read_file, mcp__slack__post]' + }); + assert.ok(errors.some(e => e.includes('MCP tools must not be listed'))); + })) passed++; else failed++; + + console.log('\nAllowlist sanity:'); + + if (test('VALID_TOOLS contains exactly the documented 11 tools', () => { + assert.strictEqual(VALID_TOOLS.length, 11); + assert.ok(VALID_TOOLS.includes('read_file')); + assert.ok(VALID_TOOLS.includes('google_web_search')); + })) passed++; else failed++; + + if (test('FORBIDDEN_FRONTMATTER_KEYS includes color and model', () => { + assert.ok(FORBIDDEN_FRONTMATTER_KEYS.includes('color')); + assert.ok(FORBIDDEN_FRONTMATTER_KEYS.includes('model')); + })) passed++; else failed++; + + if (test('CLAUDE_TO_GEMINI maps the common Claude tool names', () => { + assert.strictEqual(CLAUDE_TO_GEMINI.Read, 'read_file'); + assert.strictEqual(CLAUDE_TO_GEMINI.Bash, 'run_shell_command'); + assert.strictEqual(CLAUDE_TO_GEMINI.Grep, 'search_file_content'); + })) passed++; else failed++; + + console.log('\n=== Test Results ==='); + console.log(`Passed: ${passed}`); + console.log(`Failed: ${failed}`); + console.log(`Total: ${passed + failed}\n`); + + process.exit(failed > 0 ? 1 : 0); +} + +runTests(); diff --git a/tests/lint/fixtures/agents/claude-style-tools.md b/tests/lint/fixtures/agents/claude-style-tools.md new file mode 100644 index 0000000..f64a49a --- /dev/null +++ b/tests/lint/fixtures/agents/claude-style-tools.md @@ -0,0 +1,7 @@ +--- +name: claude-style +description: Fixture — uses Claude-style tool names that Gemini CLI rejects. +tools: ["Read", "Edit", "Bash"] +--- + +Body. diff --git a/tests/lint/fixtures/agents/forbidden-keys.md b/tests/lint/fixtures/agents/forbidden-keys.md new file mode 100644 index 0000000..9d4d9f8 --- /dev/null +++ b/tests/lint/fixtures/agents/forbidden-keys.md @@ -0,0 +1,9 @@ +--- +name: forbidden-keys +description: Fixture — uses color and model keys that are not in the Gemini schema. +tools: ["read_file"] +color: blue +model: sonnet +--- + +Body. diff --git a/tests/lint/fixtures/agents/good.md b/tests/lint/fixtures/agents/good.md new file mode 100644 index 0000000..bdbb34e --- /dev/null +++ b/tests/lint/fixtures/agents/good.md @@ -0,0 +1,7 @@ +--- +name: good-fixture +description: A well-formed agent fixture used by validator integration tests. +tools: ["read_file", "search_file_content", "write_file"] +--- + +# Body intentionally short — fixture only. diff --git a/tests/lint/fixtures/agents/legacy-names.md b/tests/lint/fixtures/agents/legacy-names.md new file mode 100644 index 0000000..1a78de9 --- /dev/null +++ b/tests/lint/fixtures/agents/legacy-names.md @@ -0,0 +1,7 @@ +--- +name: legacy-names +description: Fixture — pre-rename tool names from Issue #34. +tools: ["search_files", "replace_in_file"] +--- + +Body. diff --git a/tests/lint/fixtures/agents/mcp-in-tools.md b/tests/lint/fixtures/agents/mcp-in-tools.md new file mode 100644 index 0000000..166aab2 --- /dev/null +++ b/tests/lint/fixtures/agents/mcp-in-tools.md @@ -0,0 +1,7 @@ +--- +name: mcp-in-tools +description: Fixture — lists MCP tools in frontmatter, which the loader rejects. +tools: ["read_file", "mcp__slack__send_message"] +--- + +Body. diff --git a/tests/lint/fixtures/agents/missing-frontmatter.md b/tests/lint/fixtures/agents/missing-frontmatter.md new file mode 100644 index 0000000..dae21e8 --- /dev/null +++ b/tests/lint/fixtures/agents/missing-frontmatter.md @@ -0,0 +1,3 @@ +# Missing frontmatter + +This fixture intentionally has no YAML frontmatter block. diff --git a/tests/lint/fixtures/commands/egc-empty-description.toml b/tests/lint/fixtures/commands/egc-empty-description.toml new file mode 100644 index 0000000..a198e8f --- /dev/null +++ b/tests/lint/fixtures/commands/egc-empty-description.toml @@ -0,0 +1,4 @@ +description = " " +prompt = ''' +Fixture — whitespace-only description. +''' diff --git a/tests/lint/fixtures/commands/egc-empty-file.toml b/tests/lint/fixtures/commands/egc-empty-file.toml new file mode 100644 index 0000000..e69de29 diff --git a/tests/lint/fixtures/commands/egc-good.toml b/tests/lint/fixtures/commands/egc-good.toml new file mode 100644 index 0000000..ad202a3 --- /dev/null +++ b/tests/lint/fixtures/commands/egc-good.toml @@ -0,0 +1,4 @@ +description = "Well-formed command fixture used by validator integration tests." +prompt = ''' +Body. +''' diff --git a/tests/lint/fixtures/commands/egc-no-description.toml b/tests/lint/fixtures/commands/egc-no-description.toml new file mode 100644 index 0000000..28bfe2d --- /dev/null +++ b/tests/lint/fixtures/commands/egc-no-description.toml @@ -0,0 +1,3 @@ +prompt = ''' +Fixture — no description field. +''' diff --git a/tests/lint/fixtures/commands/egc-triple-description.toml b/tests/lint/fixtures/commands/egc-triple-description.toml new file mode 100644 index 0000000..1528b0c --- /dev/null +++ b/tests/lint/fixtures/commands/egc-triple-description.toml @@ -0,0 +1,4 @@ +description = """Triple-quoted basic description — must not be flagged as empty.""" +prompt = ''' +Body. +''' diff --git a/tests/lint/fixtures/commands/egc-triple-literal.toml b/tests/lint/fixtures/commands/egc-triple-literal.toml new file mode 100644 index 0000000..2115b3c --- /dev/null +++ b/tests/lint/fixtures/commands/egc-triple-literal.toml @@ -0,0 +1,4 @@ +description = '''Triple-quoted literal description — must not be flagged as empty.''' +prompt = ''' +Body. +''' diff --git a/tests/lint/fixtures/commands/missing-prefix.toml b/tests/lint/fixtures/commands/missing-prefix.toml new file mode 100644 index 0000000..8b9f1e7 --- /dev/null +++ b/tests/lint/fixtures/commands/missing-prefix.toml @@ -0,0 +1,4 @@ +description = "Fixture — filename lacks the egc- prefix mandated after PR #38." +prompt = ''' +Body. +''' diff --git a/tests/lint/validators.test.js b/tests/lint/validators.test.js new file mode 100644 index 0000000..6c1a3ed --- /dev/null +++ b/tests/lint/validators.test.js @@ -0,0 +1,191 @@ +/** + * Integration tests for the agent and command validators. + * + * These tests pair `extractFrontmatter` (the actual frontmatter parser used + * by the validator scripts) with `validateAgentFrontmatter` against real + * fixture files in tests/lint/fixtures/. They guard against the regressions + * tracked in Issue #34 and PR #38. + * + * Run with: node tests/lint/validators.test.js + */ + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const { extractFrontmatter } = require('../../scripts/lib/validator'); +const { validateAgentFrontmatter } = require('../../scripts/lib/gemini-tools'); +const { validateCommandContent } = require('../../scripts/ci/validate-commands'); + +const AGENT_FIXTURES = path.join(__dirname, 'fixtures/agents'); +const COMMAND_FIXTURES = path.join(__dirname, 'fixtures/commands'); + +function readFixture(dir, name) { + return fs.readFileSync(path.join(dir, name), 'utf-8'); +} + +function validateAgentFile(filename) { + const content = readFixture(AGENT_FIXTURES, filename); + const frontmatter = extractFrontmatter(content); + if (!frontmatter) return ['Missing frontmatter']; + return validateAgentFrontmatter(frontmatter); +} + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + return true; + } catch (err) { + console.log(` ✗ ${name}`); + console.log(` Error: ${err.message}`); + return false; + } +} + +function runTests() { + console.log('\n=== Testing validators (integration) ===\n'); + + let passed = 0; + let failed = 0; + + console.log('Agent validator:'); + + if (test('good fixture passes with no errors', () => { + assert.deepStrictEqual(validateAgentFile('good.md'), []); + })) passed++; else failed++; + + if (test('claude-style-tools fixture flags Read/Edit/Bash', () => { + const errors = validateAgentFile('claude-style-tools.md'); + assert.strictEqual(errors.length, 3); + assert.ok(errors.some(e => e.includes('"Read"') && e.includes('read_file'))); + assert.ok(errors.some(e => e.includes('"Edit"') && e.includes('replace'))); + assert.ok(errors.some(e => e.includes('"Bash"') && e.includes('run_shell_command'))); + })) passed++; else failed++; + + if (test('mcp-in-tools fixture flags MCP entry only', () => { + const errors = validateAgentFile('mcp-in-tools.md'); + assert.strictEqual(errors.length, 1); + assert.ok(errors[0].includes('MCP tools must not be listed')); + })) passed++; else failed++; + + if (test('forbidden-keys fixture flags both color and model', () => { + const errors = validateAgentFile('forbidden-keys.md'); + assert.ok(errors.some(e => e.includes('"color"'))); + assert.ok(errors.some(e => e.includes('"model"'))); + })) passed++; else failed++; + + if (test('legacy-names fixture flags both Issue #34 names', () => { + const errors = validateAgentFile('legacy-names.md'); + assert.strictEqual(errors.length, 2); + assert.ok(errors.some(e => e.includes('search_files') && e.includes('search_file_content'))); + assert.ok(errors.some(e => e.includes('replace_in_file') && e.includes('replace'))); + })) passed++; else failed++; + + if (test('missing-frontmatter fixture is rejected', () => { + const errors = validateAgentFile('missing-frontmatter.md'); + assert.deepStrictEqual(errors, ['Missing frontmatter']); + })) passed++; else failed++; + + console.log('\nReal agents/ directory:'); + + if (test('all real agent files pass validation', () => { + const realDir = path.join(__dirname, '../../agents'); + const files = fs.readdirSync(realDir).filter(f => f.endsWith('.md')); + assert.ok(files.length > 0, 'Expected at least one agent file'); + + const failures = []; + for (const file of files) { + const content = fs.readFileSync(path.join(realDir, file), 'utf-8'); + const frontmatter = extractFrontmatter(content); + const errors = frontmatter + ? validateAgentFrontmatter(frontmatter) + : ['Missing frontmatter']; + if (errors.length > 0) failures.push({ file, errors }); + } + + if (failures.length > 0) { + const summary = failures + .map(f => `${f.file}: ${f.errors.join('; ')}`) + .join('\n '); + throw new Error(`Real agents failed validation:\n ${summary}`); + } + })) passed++; else failed++; + + console.log('\nCommand validator:'); + + if (test('good fixture passes with no errors', () => { + const content = readFixture(COMMAND_FIXTURES, 'egc-good.toml'); + assert.deepStrictEqual(validateCommandContent(content, 'egc-good.toml'), []); + })) passed++; else failed++; + + if (test('missing-prefix fixture is rejected', () => { + const content = readFixture(COMMAND_FIXTURES, 'missing-prefix.toml'); + const errors = validateCommandContent(content, 'missing-prefix.toml'); + assert.strictEqual(errors.length, 1); + assert.ok(errors[0].includes('egc-')); + })) passed++; else failed++; + + if (test('no-description fixture is rejected', () => { + const content = readFixture(COMMAND_FIXTURES, 'egc-no-description.toml'); + const errors = validateCommandContent(content, 'egc-no-description.toml'); + assert.ok(errors.some(e => e.includes('description'))); + })) passed++; else failed++; + + if (test('empty-description fixture is rejected', () => { + const content = readFixture(COMMAND_FIXTURES, 'egc-empty-description.toml'); + const errors = validateCommandContent(content, 'egc-empty-description.toml'); + assert.ok(errors.some(e => e.includes('Empty description'))); + })) passed++; else failed++; + + if (test('empty file fixture is rejected', () => { + const content = readFixture(COMMAND_FIXTURES, 'egc-empty-file.toml'); + const errors = validateCommandContent(content, 'egc-empty-file.toml'); + assert.deepStrictEqual(errors, ['Empty command file']); + })) passed++; else failed++; + + if (test('triple-quoted basic description is accepted', () => { + const content = readFixture(COMMAND_FIXTURES, 'egc-triple-description.toml'); + const errors = validateCommandContent(content, 'egc-triple-description.toml'); + assert.deepStrictEqual(errors, []); + })) passed++; else failed++; + + if (test('triple-quoted literal description is accepted', () => { + const content = readFixture(COMMAND_FIXTURES, 'egc-triple-literal.toml'); + const errors = validateCommandContent(content, 'egc-triple-literal.toml'); + assert.deepStrictEqual(errors, []); + })) passed++; else failed++; + + console.log('\nReal commands/ directory:'); + + if (test('all real command files pass validation', () => { + const realDir = path.join(__dirname, '../../commands'); + const files = fs.readdirSync(realDir).filter(f => f.endsWith('.toml')); + assert.ok(files.length > 0, 'Expected at least one command file'); + + const failures = []; + for (const file of files) { + const content = fs.readFileSync(path.join(realDir, file), 'utf-8'); + const errors = validateCommandContent(content, file); + if (errors.length > 0) failures.push({ file, errors }); + } + + if (failures.length > 0) { + const summary = failures + .map(f => `${f.file}: ${f.errors.join('; ')}`) + .join('\n '); + throw new Error(`Real commands failed validation:\n ${summary}`); + } + })) passed++; else failed++; + + console.log('\n=== Test Results ==='); + console.log(`Passed: ${passed}`); + console.log(`Failed: ${failed}`); + console.log(`Total: ${passed + failed}\n`); + + process.exit(failed > 0 ? 1 : 0); +} + +runTests(); diff --git a/tests/run-all.js b/tests/run-all.js index f1f6c92..934e482 100644 --- a/tests/run-all.js +++ b/tests/run-all.js @@ -15,7 +15,9 @@ const testFiles = [ 'lib/package-manager.test.js', 'lib/session-manager.test.js', 'lib/session-aliases.test.js', - 'hooks/hooks.test.js' + 'lib/gemini-tools.test.js', + 'hooks/hooks.test.js', + 'lint/validators.test.js' ]; console.log('╔══════════════════════════════════════════════════════════╗');