Skip to content

⚡ Optimize vibe-check-runner.js execution flow #310

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
beginwebdev2002 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

⚡ Optimize vibe-check-runner.js execution flow #310

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 21 additions & 17 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
"@google-cloud/aiplatform": "^6.5.0",
"@google-cloud/storage": "^7.19.0",
"@google-cloud/vertexai": "^1.1.0",
"@google/genai": "^1.50.0",
"axios": "^1.13.6",
"@google/genai": "^1.52.0",
"axios": "^1.16.0",
"ts-morph": "^27.0.2"
}
}
162 changes: 102 additions & 60 deletions vibe-check-runner.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ import fsPromises from 'node:fs/promises';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { GoogleGenAI } from '@google/genai';
import { pathToFileURL } from 'node:url';

const ai = new GoogleGenAI({ apiKey: process.env.GOOGLE_AI_API_KEY });

// Constants for scoring

async function fileOrDirExists(filePath) {
export async function fileOrDirExists(filePath) {
try {
await fsPromises.stat(filePath);
return true;
Expand All @@ -25,7 +26,7 @@ const SCORES = {
EFFICIENCY: 10,
};

function getModifiedFiles() {
export function getModifiedFiles() {
try {
// In CI (daily run), check files modified in the last 24 hours.
// We filter for non-empty lines that end in .md and are in frontend/ or backend/
Expand All @@ -43,7 +44,7 @@ function getModifiedFiles() {
}
}

async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) {
export async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) {
try {
const prompt = `Based on the following documentation:\n\n${mdContent}\n\n1. Generate a "Golden Prompt" (a comprehensive instruction for generating a typical module using this technology) in JSON format: {"golden_prompt": "...", "tech": "${tech}"}\n2. Generate a JSON Schema for TS-Morph AST validation rules enforcing DDD/FSD layers and strict typing for this technology. The generated JSON schema must explicitly follow a nested structure compatible with \`analyzeAST\`. Format: {"$schema": "...", "type": "object", "properties": {"forbidden_types": {"contains": {"enum": ["any"]}}}}.\n\nRespond strictly with ONLY a JSON array containing these two objects in order. No markdown wrappers.`;
const response = await ai.models.generateContent({
Expand Down Expand Up @@ -85,7 +86,7 @@ async function syncBenchmarks(tech, mdContent, retries = 5, delay = 10000) {
}
}

async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, delay = 10000) {
export async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5, delay = 10000) {
try {
const prompt = `${goldenPrompt}\n\nConstraints and instructions from the following documentation:\n\n${mdContent}\n\nGenerate ONLY raw code. No markdown formatting, no explanations.`;
const response = await ai.models.generateContent({
Expand All @@ -110,7 +111,7 @@ async function simulateAIGeneration(goldenPrompt, tech, mdContent, retries = 5,
}
}

function analyzeAST(sourceFile, tech) {
export function analyzeAST(sourceFile, tech) {
let score = {
arch: SCORES.ARCH,
type: SCORES.TYPE,
Expand Down Expand Up @@ -222,7 +223,7 @@ function analyzeAST(sourceFile, tech) {
return { total, breakdown: score };
}

async function runVibeCheck() {
export async function runVibeCheck() {
console.log('Running Vibe-Check Runner...');

const modifiedFiles = getModifiedFiles();
Expand All @@ -241,11 +242,10 @@ async function runVibeCheck() {
console.warn('Failed to configure git user. If running locally, this is expected.');
}

// Deduplicate by technology and read contents for syncBenchmarks
const techMap = new Map();
for (const file of modifiedFiles) {
console.log(`Processing ${file}...`);

if (!fs.existsSync(file)) {
console.log(`File ${file} does not exist. Skipping.`);
if (!await fileOrDirExists(file)) {
continue;
}

Expand All @@ -256,7 +256,6 @@ async function runVibeCheck() {
else if (file.includes('/express/')) tech = 'express';
else if (file.includes('/nodejs/')) tech = 'nodejs';
else {
// Fallback
const parts = file.split('/');
if (parts.length > 1) {
tech = parts[1];
Expand All @@ -266,76 +265,119 @@ async function runVibeCheck() {
}

const mdContent = await fsPromises.readFile(file, 'utf-8');
if (!techMap.has(tech)) {
techMap.set(tech, { content: '' });
}
const entry = techMap.get(tech);
entry.content += `\n\n--- Content from ${file} ---\n\n${mdContent}`;
}

await syncBenchmarks(tech, mdContent);
// Process syncBenchmarks sequentially by technology
for (const [tech, data] of techMap.entries()) {
await syncBenchmarks(tech, data.content);
}

const suitePath = path.join('benchmarks', 'suites', `${tech}.json`);
if (!fs.existsSync(suitePath)) {
console.log(`No benchmark suite found for ${tech}. Skipping.`);
continue;
}
// Concurrent generation and AST analysis per file
const generationTasks = [];
for (const file of modifiedFiles) {
if (!await fileOrDirExists(file)) {
continue;
}

const suiteConfig = JSON.parse(await fsPromises.readFile(suitePath, 'utf-8'));
let tech = '';
if (file.includes('/angular/')) tech = 'angular';
else if (file.includes('/nestjs/')) tech = 'nestjs';
else if (file.includes('/typescript/')) tech = 'typescript';
else if (file.includes('/express/')) tech = 'express';
else if (file.includes('/nodejs/')) tech = 'nodejs';
else {
const parts = file.split('/');
if (parts.length > 1) {
tech = parts[1];
} else {
continue;
}
}

const generatedCode = await simulateAIGeneration(suiteConfig.golden_prompt, tech, mdContent);
generationTasks.push((async () => {
const mdContent = await fsPromises.readFile(file, 'utf-8');
const suitePath = path.join('benchmarks', 'suites', `${tech}.json`);
if (!await fileOrDirExists(suitePath)) {
console.log(`No benchmark suite found for ${tech}. Skipping.`);
return { file, tech, mdContent, generatedCode: null, score: null, breakdown: null };
}

if (!generatedCode) {
console.error(`Failed to generate code for ${tech}.`);
continue;
}
const suiteConfig = JSON.parse(await fsPromises.readFile(suitePath, 'utf-8'));
const generatedCode = await simulateAIGeneration(suiteConfig.golden_prompt, tech, mdContent);

const sourceFile = project.createSourceFile(`temp_${tech}.ts`, generatedCode, { overwrite: true });
const { total: score, breakdown } = analyzeAST(sourceFile, tech);
if (!generatedCode) {
console.error(`Failed to generate code for ${tech}.`);
return { file, tech, mdContent, generatedCode: null, score: null, breakdown: null };
}

const sourceFile = project.createSourceFile(`temp_${tech}_${Buffer.from(file).toString('base64').substring(0,8)}.ts`, generatedCode, { overwrite: true });
const { total: score, breakdown } = analyzeAST(sourceFile, tech);
return { file, tech, mdContent, generatedCode, score, breakdown };
})());
}

const results = await Promise.all(generationTasks);

// Sequential stateful side effects
for (const result of results) {
if (!result.generatedCode) continue;

const { file, tech, generatedCode, score, breakdown } = result;

console.log(`Fidelity Score for ${file}: ${score}%`);
console.log(`Breakdown:`, breakdown);

if (score >= 95) {
console.log(`✅ Validation passed for ${file}. Updating badge and auto-committing.`);

let content = await fsPromises.readFile(file, 'utf-8');
if (!content.includes('[![Vibe-Coding Verified]')) {
content = content.replace(/^# /, '[![Vibe-Coding Verified](https://img.shields.io/badge/Vibe--Coding-Verified-brightgreen?style=for-the-badge)](#)\n\n# ');
await fsPromises.writeFile(file, content);
}
console.log(`✅ Validation passed for ${file}. Updating badge and auto-committing.`);

try {
execFileSync('git', ['add', file]);
try { execFileSync('sh', ['-c', 'git add benchmarks/suites/*.json benchmarks/criteria/*.json 2>/dev/null || true']); } catch (e) {}
// Only commit if there are changes (badge might already be there)
const status = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' });
if (status.includes(file) || status.includes('benchmarks/')) {
execFileSync('git', ['commit', '-m', '[chore: benchmark-sync]']);
execFileSync('git', ['push', 'origin', 'HEAD:main']);
} else {
console.log(`Badge already present in ${file}, skipping commit.`);
let content = await fsPromises.readFile(file, 'utf-8');
if (!content.includes('[![Vibe-Coding Verified]')) {
content = content.replace(/^# /, '[![Vibe-Coding Verified](https://img.shields.io/badge/Vibe--Coding-Verified-brightgreen?style=for-the-badge)](#)\n\n# ');
await fsPromises.writeFile(file, content);
}
} catch (err) {
console.error('Failed to commit or push:', err.message);
}

try {
execFileSync('git', ['add', file]);
try { execFileSync('sh', ['-c', 'git add benchmarks/suites/*.json benchmarks/criteria/*.json 2>/dev/null || true']); } catch (e) {}
const status = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' });
if (status.includes(file) || status.includes('benchmarks/')) {
execFileSync('git', ['commit', '-m', '[chore: fidelity-pass]']);
execFileSync('git', ['push', 'origin', 'HEAD:main']);
} else {
console.log(`Badge already present in ${file}, skipping commit.`);
}
} catch (err) {
console.error('Failed to commit or push:', err.message);
}
} else {
console.error(`❌ Validation failed for ${file}. Score below 95%.`);
console.error(`❌ Validation failed for ${file}. Score below 95%.`);

const reportDir = path.join('benchmarks', 'logs');
if (!await fileOrDirExists(reportDir)) await fsPromises.mkdir(reportDir, { recursive: true });
const reportDir = path.join('benchmarks', 'logs');
if (!await fileOrDirExists(reportDir)) await fsPromises.mkdir(reportDir, { recursive: true });

const reportPath = path.join(reportDir, `violation-report.md`);
const reportContent = `# Critical Violation Report\n\n> [!CAUTION]\n> Fidelity Score dropped below 95%.\n\n**File:** \`${file}\`\n**Fidelity Score:** ${score}%\n**Threshold:** 95%\n\n## Breakdown\n| Metric | Score |\n|---|---|\n| Arch Integrity | ${breakdown.arch} |\n| Type Safety | ${breakdown.type} |\n| Security | ${breakdown.security} |\n| Efficiency | ${breakdown.efficiency} |\n\n## Generated Code\n\`\`\`typescript\n${generatedCode}\n\`\`\`\n\nReview the AST rules.`;
const reportPath = path.join(reportDir, `violation-report.md`);
const reportContent = `# Critical Violation Report\n\n> [!CAUTION]\n> Fidelity Score dropped below 95%.\n\n**File:** \`${file}\`\n**Fidelity Score:** ${score}%\n**Threshold:** 95%\n\n## Breakdown\n| Metric | Score |\n|---|---|\n| Arch Integrity | ${breakdown.arch} |\n| Type Safety | ${breakdown.type} |\n| Security | ${breakdown.security} |\n| Efficiency | ${breakdown.efficiency} |\n\n## Generated Code\n\`\`\`typescript\n${generatedCode}\n\`\`\`\n\nReview the AST rules.`;

await fsPromises.writeFile(reportPath, reportContent);
console.log(`Generated violation report: ${reportPath}`);
await fsPromises.writeFile(reportPath, reportContent);
console.log(`Generated violation report: ${reportPath}`);

try {
execFileSync('gh', ['issue', 'create', '--title', `Critical Issue: Fidelity Gap for ${file}`, '--label', 'critical,bug', '--body-file', reportPath]);
console.log(`Created GitHub Issue for ${file}`);
} catch (err) {
console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message);
}
try {
execFileSync('gh', ['issue', 'create', '--title', `Critical Issue: Fidelity Gap for ${file}`, '--label', 'critical,bug', '--body-file', reportPath]);
console.log(`Created GitHub Issue for ${file}`);
} catch (err) {
console.error('Failed to create GitHub Issue (gh cli might not be installed or authenticated):', err.message);
}

process.exitCode = 1;
process.exitCode = 1;
}
}
}

runVibeCheck().catch(console.error);
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
runVibeCheck().catch(console.error);
}
Loading