diff --git a/.gitignore b/.gitignore index efbd9f5..8a9ef28 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # For our testing scripts /cache /test/fixtures/actual-results.json +/test/fixtures-multithread # For DS Store **/.DS_Store diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 35bd20b..312bb3a 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,5 +8,7 @@ // "esbenp.prettier-vscode", // Prettier for consistent formatting / not maintained and causes crashes https://github.com/prettier/prettier-vscode/issues/3743 "dbaeumer.vscode-eslint", "arcanis.vscode-zipfs" // Required to use Prettier with Yarn in VS Code + , + "esbenp.prettier-vscode" ] } diff --git a/package.json b/package.json index d920d17..ac01d73 100644 --- a/package.json +++ b/package.json @@ -2,14 +2,14 @@ "license": "UNLICENSED", "devDependencies": { "@shopify/prettier-plugin-liquid": "^1.9.3", - "better-sqlite3": "^11.10.0", - "cheerio": "^1.1.0", + "better-sqlite3": "^12.2.0", + "cheerio": "^1.1.2", "cli-progress": "^3.12.0", "css": "^3.0.0", - "glob": "^11.0.2", - "html-validate": "^9.5.5", + "glob": "^11.0.3", + "html-validate": "^10.0.0", "markdownlint-cli2": "^0.18.1", - "prettier": "^3.5.3", + "prettier": "^3.6.2", "shell-quote": "^1.8.3" }, "scripts": { diff --git a/test/build-html-validate.mjs b/test/build-html-validate.mjs index d2ce5d5..cbf9826 100644 --- a/test/build-html-validate.mjs +++ b/test/build-html-validate.mjs @@ -1,67 +1,127 @@ -import { HtmlValidate, FileSystemConfigLoader, formatterFactory, esmResolver } from "html-validate"; import { glob } from "glob"; import cliProgress from "cli-progress"; +import { Worker } from "worker_threads"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); // In the future, the CLI may improve and this script may be unnecessary. // SEE: https://gitlab.com/html-validate/html-validate/-/issues/273 +// Configuration +const MAX_WORKERS = parseInt(process.env.HTML_VALIDATE_WORKERS) || 4; +const WORKER_SCRIPT_PATH = path.join(__dirname, "html-validate-worker.mjs"); + // Find and sort all HTML files in the 'build' directory const targets = glob.sync("build/**/*.html").sort(); -// Initialize HtmlValidate instance -const resolver = esmResolver(); -const loader = new FileSystemConfigLoader([resolver]); -const htmlValidate = new HtmlValidate(loader); -const formatter = formatterFactory("stylish"); -let allTestsPassed = true; - -// Initialize progress bar -const bar = new cliProgress.SingleBar({ - format: "๐Ÿงช [{bar}] {percentage}% | {value}/{total} | ETA: {eta}s | {file}", - forceRedraw: true, -}); -// Monkey-patch in logging support https://github.com/npkgz/cli-progress/issues/159#issuecomment-2959578474 -bar.loggingBuffer = []; -bar.log = function (message) { - bar.loggingBuffer.push(message); -}; -bar.on("redraw-pre", (data) => { - if (bar.loggingBuffer.length > 0) { - bar.terminal.clearLine(); - bar.terminal.cursorTo(0); - while (bar.loggingBuffer.length > 0) { - bar.terminal.write(bar.loggingBuffer.shift()); - bar.terminal.write("\n"); +if (targets.length === 0) { + console.log("โš ๏ธ No HTML files found in build directory"); + console.log(" Make sure to build the site first"); + process.exit(0); +} + +console.log(`๐Ÿงช Validating ${targets.length} files with ${MAX_WORKERS} parallel workers...`); + +await validateParallel(); + +async function validateParallel() { + const multibar = new cliProgress.MultiBar({ + format: "[{bar}] {percentage}% | {value}/{total} | {status}", + hideCursor: true, + clearOnComplete: false, + stopOnComplete: true, + forceRedraw: false, + }, cliProgress.Presets.shades_classic); + + const overallBar = multibar.create(targets.length, 0, { + status: "Starting..." + }); + + let completedTasks = 0; + let allTestsPassed = true; + const results = []; + const workers = []; + const taskQueue = [...targets]; + + let isDone = false; + function completeParallelProcessing() { + if (isDone) return; + isDone = true; + + multibar.stop(); + workers.forEach(worker => worker.terminate()); + + const failedResults = results.filter(r => !r.isValid); + const passedCount = results.length - failedResults.length; + + console.log("\n๐Ÿ“Š Results summary:"); + console.log(`โœ… ${passedCount} files passed validation`); + + if (failedResults.length > 0) { + console.log(`โŒ ${failedResults.length} files failed validation`); + process.exit(1); + } else { + console.log("โœจ All tests passed!\n"); } } -}); -console.log("๐Ÿงช Validating files..."); -bar.start(targets.length, 0, { file: "Starting..." }); + function createWorker(workerId) { + const worker = new Worker(WORKER_SCRIPT_PATH); -for (const target of targets) { - try { - bar.increment(0, { file: target }); - const report = await htmlValidate.validateFile(target); - if (!report.valid) { - bar.log(formatter(report.results)); + worker.on("message", (result) => { + completedTasks++; + results.push(result); + + if (!result.isValid) { + allTestsPassed = false; + multibar.log(`โŒ ${path.relative(process.cwd(), result.filePath)}\n${result.message.trim()}\n\n`); + } + + overallBar.increment(1, { + status: path.basename(result.filePath), + }); + + if (taskQueue.length > 0) { + const nextTask = taskQueue.shift(); + worker.postMessage({ filePath: nextTask, workerId }); + } + + if (completedTasks === targets.length) { + completeParallelProcessing(); + } + }); + + worker.on("error", (error) => { + console.error(`Worker ${workerId} error:`, error); allTestsPassed = false; - } else { - bar.log(`โœ… ${target}`); - } - } catch (error) { - bar.log(`โŒ Error validating ${target}: ${error.message}`); - allTestsPassed = false; + completeParallelProcessing(); + }); + + return worker; } - bar.increment(1); -} -bar.stop(); + function startParallelProcessing() { + for (let i = 0; i < MAX_WORKERS; i++) { + const worker = createWorker(i); + workers.push(worker); + } + + const initialTasks = Math.min(MAX_WORKERS, taskQueue.length); + for (let i = 0; i < initialTasks; i++) { + const task = taskQueue.shift(); + workers[i].postMessage({ filePath: task, workerId: i }); + } + } -// Display final result -if (allTestsPassed) { - console.log("โœจ All tests passed!\n"); -} else { - console.log("โŒ Some tests failed.\n"); - process.exit(1); + return new Promise((resolve) => { + const originalComplete = completeParallelProcessing; + completeParallelProcessing = () => { + originalComplete(); + resolve(); + }; + startParallelProcessing(); + }); } diff --git a/test/html-validate-worker.mjs b/test/html-validate-worker.mjs new file mode 100644 index 0000000..ba5f97e --- /dev/null +++ b/test/html-validate-worker.mjs @@ -0,0 +1,37 @@ +import { parentPort, workerData } from "worker_threads"; +import { HtmlValidate, FileSystemConfigLoader, formatterFactory, esmResolver } from "html-validate"; + +// Initialize HtmlValidate instance (same as main script) +const resolver = esmResolver(); +const loader = new FileSystemConfigLoader([resolver]); +const htmlValidate = new HtmlValidate(loader); +const formatter = formatterFactory("text"); + +// Listen for messages from parent thread +parentPort.on("message", async (data) => { + const { filePath, workerId } = data; + + try { + const report = await htmlValidate.validateFile(filePath); + + const result = { + workerId, + filePath, + success: report.valid, + message: report.valid ? `โœ… ${filePath}` : formatter(report.results), + isValid: report.valid, + report: report + }; + + parentPort.postMessage(result); + } catch (error) { const result = { + workerId, + filePath, + success: false, + message: `โŒ Error validating`, + isValid: false + }; + + parentPort.postMessage(result); + } +}); \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 694f09b..f7389e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14,6 +14,22 @@ __metadata: languageName: node linkType: hard +"@isaacs/balanced-match@npm:^4.0.1": + version: 4.0.1 + resolution: "@isaacs/balanced-match@npm:4.0.1" + checksum: 10c0/7da011805b259ec5c955f01cee903da72ad97c5e6f01ca96197267d3f33103d5b2f8a1af192140f3aa64526c593c8d098ae366c2b11f7f17645d12387c2fd420 + languageName: node + linkType: hard + +"@isaacs/brace-expansion@npm:^5.0.0": + version: 5.0.0 + resolution: "@isaacs/brace-expansion@npm:5.0.0" + dependencies: + "@isaacs/balanced-match": "npm:^4.0.1" + checksum: 10c0/b4d4812f4be53afc2c5b6c545001ff7a4659af68d4484804e9d514e183d20269bb81def8682c01a22b17c4d6aed14292c8494f7d2ac664e547101c1a905aa977 + languageName: node + linkType: hard + "@isaacs/cliui@npm:^8.0.2": version: 8.0.2 resolution: "@isaacs/cliui@npm:8.0.2" @@ -261,14 +277,14 @@ __metadata: languageName: node linkType: hard -"better-sqlite3@npm:^11.10.0": - version: 11.10.0 - resolution: "better-sqlite3@npm:11.10.0" +"better-sqlite3@npm:^12.2.0": + version: 12.2.0 + resolution: "better-sqlite3@npm:12.2.0" dependencies: bindings: "npm:^1.5.0" node-gyp: "npm:latest" prebuild-install: "npm:^7.1.1" - checksum: 10c0/1fffbf9e5fc9d24847a3ecf09491bceab1c294b46ba41df1c449dc20b6f5c5d9d94ff24becd0b1632ee282bd21278b7fea53a5a6215bb99209ded0ae05eda3b0 + checksum: 10c0/842247e9bbb775f366ac91f604117112c312497e643bac21648d8b69f479763de0ac049b14b609d6d5ecaee50debcc09a854f682d3dc099a1d933fea92ce68d0 languageName: node linkType: hard @@ -382,22 +398,22 @@ __metadata: languageName: node linkType: hard -"cheerio@npm:^1.1.0": - version: 1.1.0 - resolution: "cheerio@npm:1.1.0" +"cheerio@npm:^1.1.2": + version: 1.1.2 + resolution: "cheerio@npm:1.1.2" dependencies: cheerio-select: "npm:^2.1.0" dom-serializer: "npm:^2.0.0" domhandler: "npm:^5.0.3" domutils: "npm:^3.2.2" - encoding-sniffer: "npm:^0.2.0" + encoding-sniffer: "npm:^0.2.1" htmlparser2: "npm:^10.0.0" parse5: "npm:^7.3.0" parse5-htmlparser2-tree-adapter: "npm:^7.1.0" parse5-parser-stream: "npm:^7.1.2" - undici: "npm:^7.10.0" + undici: "npm:^7.12.0" whatwg-mimetype: "npm:^4.0.0" - checksum: 10c0/f7b940a89e1fe77bf6b4fe3b993f17b02a358942cc0b9d3b55ea235a0bc322829dbc47151763ef9986fd237494c00380909af759e46582c72470a10643b85abd + checksum: 10c0/2c6d2274666fe122f54fdca457ee76453e1a993b19563acaa23eb565bf7776f0f01e4c3800092f00e84aa13c83a161f0cf000ac0a8332d1d7f2b2387d6ecc5fc languageName: node linkType: hard @@ -454,7 +470,7 @@ __metadata: languageName: node linkType: hard -"cross-spawn@npm:^7.0.0": +"cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.6": version: 7.0.6 resolution: "cross-spawn@npm:7.0.6" dependencies: @@ -634,13 +650,13 @@ __metadata: languageName: node linkType: hard -"encoding-sniffer@npm:^0.2.0": - version: 0.2.0 - resolution: "encoding-sniffer@npm:0.2.0" +"encoding-sniffer@npm:^0.2.1": + version: 0.2.1 + resolution: "encoding-sniffer@npm:0.2.1" dependencies: iconv-lite: "npm:^0.6.3" whatwg-encoding: "npm:^3.1.1" - checksum: 10c0/b312e0d67f339bec44e021e5210ee8ee90d7b8f9975eb2c79a36fd467eb07709e88dcf62ee20f62ee0d74a13874307d99557852a2de9b448f1e3fb991fc68257 + checksum: 10c0/d6b591880788f3baf8dd1744636dd189d24a1ec93e6f9817267c60ac3458a5191ca70ab1a186fb67731beff1c3489c6527dfdc4718158ed8460ab2f400dd5e7d languageName: node linkType: hard @@ -776,6 +792,16 @@ __metadata: languageName: node linkType: hard +"foreground-child@npm:^3.3.1": + version: 3.3.1 + resolution: "foreground-child@npm:3.3.1" + dependencies: + cross-spawn: "npm:^7.0.6" + signal-exit: "npm:^4.0.1" + checksum: 10c0/8986e4af2430896e65bc2788d6679067294d6aee9545daefc84923a0a4b399ad9c7a3ea7bd8c0b2b80fdf4a92de4c69df3f628233ff3224260e9c1541a9e9ed3 + languageName: node + linkType: hard + "front-matter@npm:^4.0.2": version: 4.0.2 resolution: "front-matter@npm:4.0.2" @@ -826,7 +852,7 @@ __metadata: languageName: node linkType: hard -"glob@npm:^10.0.0, glob@npm:^10.2.2, glob@npm:^10.3.10": +"glob@npm:^10.2.2, glob@npm:^10.3.10": version: 10.4.5 resolution: "glob@npm:10.4.5" dependencies: @@ -842,19 +868,19 @@ __metadata: languageName: node linkType: hard -"glob@npm:^11.0.2": - version: 11.0.2 - resolution: "glob@npm:11.0.2" +"glob@npm:^11.0.0, glob@npm:^11.0.3": + version: 11.0.3 + resolution: "glob@npm:11.0.3" dependencies: - foreground-child: "npm:^3.1.0" - jackspeak: "npm:^4.0.1" - minimatch: "npm:^10.0.0" + foreground-child: "npm:^3.3.1" + jackspeak: "npm:^4.1.1" + minimatch: "npm:^10.0.3" minipass: "npm:^7.1.2" package-json-from-dist: "npm:^1.0.0" path-scurry: "npm:^2.0.0" bin: glob: dist/esm/bin.mjs - checksum: 10c0/49f91c64ca882d5e3a72397bd45a146ca91fd3ca53dafb5254daf6c0e83fc510d39ea66f136f9ac7ca075cdd11fbe9aaa235b28f743bd477622e472f4fdc0240 + checksum: 10c0/7d24457549ec2903920dfa3d8e76850e7c02aa709122f0164b240c712f5455c0b457e6f2a1eee39344c6148e39895be8094ae8cfef7ccc3296ed30bce250c661 languageName: node linkType: hard @@ -886,23 +912,23 @@ __metadata: languageName: node linkType: hard -"html-validate@npm:^9.5.5": - version: 9.5.5 - resolution: "html-validate@npm:9.5.5" +"html-validate@npm:^10.0.0": + version: 10.0.0 + resolution: "html-validate@npm:10.0.0" dependencies: "@html-validate/stylish": "npm:^4.1.0" "@sidvind/better-ajv-errors": "npm:4.0.0" ajv: "npm:^8.0.0" - glob: "npm:^10.0.0" + glob: "npm:^11.0.0" kleur: "npm:^4.1.0" minimist: "npm:^1.2.0" prompts: "npm:^2.0.0" semver: "npm:^7.0.0" peerDependencies: - jest: ^27.1 || ^28.1.3 || ^29.0.3 - jest-diff: ^27.1 || ^28.1.3 || ^29.0.3 - jest-snapshot: ^27.1 || ^28.1.3 || ^29.0.3 - vitest: ^0.34.0 || ^1.0.0 || ^2.0.0 || ^3.0.0 + jest: ^28.1.3 || ^29.0.3 || ^30.0.0 + jest-diff: ^28.1.3 || ^29.0.3 || ^30.0.0 + jest-snapshot: ^28.1.3 || ^29.0.3 || ^30.0.0 + vitest: ^1.0.0 || ^2.0.0 || ^3.0.0 peerDependenciesMeta: jest: optional: true @@ -914,7 +940,7 @@ __metadata: optional: true bin: html-validate: bin/html-validate.mjs - checksum: 10c0/ee1becce55d39aef19884509da887bdf7aea2952ece69c9ab1919c24690b673b22e4c76e2b5b65535a58da501d56e2441c639559bd88be472694d0381630a3b0 + checksum: 10c0/39af9649a1990b0a70993951ed29ac5c7c769a0434c2e4248f0b66b2d6e04a27ee15f850f3f8e4e2eb62494bf2606f45af9f5193c11196056594fdbf29b8bc0e languageName: node linkType: hard @@ -1129,12 +1155,12 @@ __metadata: languageName: node linkType: hard -"jackspeak@npm:^4.0.1": - version: 4.0.2 - resolution: "jackspeak@npm:4.0.2" +"jackspeak@npm:^4.1.1": + version: 4.1.1 + resolution: "jackspeak@npm:4.1.1" dependencies: "@isaacs/cliui": "npm:^8.0.2" - checksum: 10c0/b26039d11c0163a95b1e58851b9ac453cce64ad6d1eb98a00b303ad5eeb761b29d33c9419d1e16c016d3f7151c8edf7df223e6cf93a1907655fd95d6ce85c0de + checksum: 10c0/84ec4f8e21d6514db24737d9caf65361511f75e5e424980eebca4199f400874f45e562ac20fa8aeb1dd20ca2f3f81f0788b6e9c3e64d216a5794fd6f30e0e042 languageName: node linkType: hard @@ -1675,12 +1701,12 @@ __metadata: languageName: node linkType: hard -"minimatch@npm:^10.0.0": - version: 10.0.1 - resolution: "minimatch@npm:10.0.1" +"minimatch@npm:^10.0.3": + version: 10.0.3 + resolution: "minimatch@npm:10.0.3" dependencies: - brace-expansion: "npm:^2.0.1" - checksum: 10c0/e6c29a81fe83e1877ad51348306be2e8aeca18c88fdee7a99df44322314279e15799e41d7cb274e4e8bb0b451a3bc622d6182e157dfa1717d6cda75e9cd8cd5d + "@isaacs/brace-expansion": "npm:^5.0.0" + checksum: 10c0/e43e4a905c5d70ac4cec8530ceaeccb9c544b1ba8ac45238e2a78121a01c17ff0c373346472d221872563204eabe929ad02669bb575cb1f0cc30facab369f70f languageName: node linkType: hard @@ -2008,12 +2034,12 @@ __metadata: languageName: node linkType: hard -"prettier@npm:^3.5.3": - version: 3.5.3 - resolution: "prettier@npm:3.5.3" +"prettier@npm:^3.6.2": + version: 3.6.2 + resolution: "prettier@npm:3.6.2" bin: prettier: bin/prettier.cjs - checksum: 10c0/3880cb90b9dc0635819ab52ff571518c35bd7f15a6e80a2054c05dbc8a3aa6e74f135519e91197de63705bcb38388ded7e7230e2178432a1468005406238b877 + checksum: 10c0/488cb2f2b99ec13da1e50074912870217c11edaddedeadc649b1244c749d15ba94e846423d062e2c4c9ae683e2d65f754de28889ba06e697ac4f988d44f45812 languageName: node linkType: hard @@ -2119,15 +2145,15 @@ __metadata: resolution: "root-workspace-0b6124@workspace:." dependencies: "@shopify/prettier-plugin-liquid": "npm:^1.9.3" - better-sqlite3: "npm:^11.10.0" - cheerio: "npm:^1.1.0" + better-sqlite3: "npm:^12.2.0" + cheerio: "npm:^1.1.2" cli-progress: "npm:^3.12.0" css: "npm:^3.0.0" front-matter: "npm:^4.0.2" - glob: "npm:^11.0.2" - html-validate: "npm:^9.5.5" + glob: "npm:^11.0.3" + html-validate: "npm:^10.0.0" markdownlint-cli2: "npm:^0.18.1" - prettier: "npm:^3.5.3" + prettier: "npm:^3.6.2" shell-quote: "npm:^1.8.3" xml2js: "npm:^0.6.2" dependenciesMeta: @@ -2425,10 +2451,10 @@ __metadata: languageName: node linkType: hard -"undici@npm:^7.10.0": - version: 7.10.0 - resolution: "undici@npm:7.10.0" - checksum: 10c0/756ac876a8df845bc89eb8348c35d33a0ff63c17eb45b664075c961a7fbd4a398f94f9dce438262f55fe66e4bbb0a46aa63a3fd58ce51361c616aff11a270450 +"undici@npm:^7.12.0": + version: 7.13.0 + resolution: "undici@npm:7.13.0" + checksum: 10c0/8865d40b141f073215a6763aad5d1b2f4bd4e252600e93e68055d6c5d23a8a0e5782669236b2ecfa4d415d1d969d9c4623ff1c0386d32fa60088a19ffa58c611 languageName: node linkType: hard