diff --git a/.gitignore b/.gitignore index 750baeb..045a6c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +dist result result-* diff --git a/flake.lock b/flake.lock index 3ddb1b7..fa64f82 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,24 @@ "type": "github" } }, + "globset": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_2" + }, + "locked": { + "lastModified": 1756146523, + "narHash": "sha256-dn8WNgwVQe0xBJ/d4CiRRcEVSgNclG6WSbqFEf6V024=", + "owner": "pdtpartners", + "repo": "globset", + "rev": "62da8904981f01dc7c97f56c3ea4f131a8393411", + "type": "github" + }, + "original": { + "owner": "pdtpartners", + "repo": "globset", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1779622335, @@ -49,11 +67,55 @@ "type": "github" } }, + "nixpkgs-lib_2": { + "locked": { + "lastModified": 1754788789, + "narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "a73b9c743612e4244d865a2fdee11865283c04e6", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "package-lock2nix": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "globset": "globset", + "nixpkgs": [ + "nixpkgs" + ], + "systems": "systems", + "treefmt-nix": [ + "treefmt-nix" + ] + }, + "locked": { + "lastModified": 1781065150, + "narHash": "sha256-HrqxSjZ1/66T9U/Kwh3Ooq89EK0rH/jn6lnEa0yib0s=", + "owner": "anteriorcore", + "repo": "package-lock2nix", + "rev": "5a5b27fa19ecbca004fb0dc429dd853b37e2e318", + "type": "github" + }, + "original": { + "owner": "anteriorcore", + "repo": "package-lock2nix", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", "nixpkgs": "nixpkgs", - "systems": "systems", + "package-lock2nix": "package-lock2nix", + "systems": "systems_2", "treefmt-nix": "treefmt-nix" } }, @@ -71,6 +133,20 @@ "type": "indirect" } }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 35cd9a5..0b81914 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,12 @@ # keep-sorted start block=true flake-parts.url = "github:hercules-ci/flake-parts"; nixpkgs.url = "github:nixos/nixpkgs/nixos-26.05"; + package-lock2nix = { + url = "github:anteriorcore/package-lock2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + inputs.flake-parts.follows = "flake-parts"; + inputs.treefmt-nix.follows = "treefmt-nix"; + }; systems.url = "systems"; treefmt-nix = { url = "github:numtide/treefmt-nix"; @@ -25,8 +31,12 @@ { packages = let + package-lock2nix = pkgs.callPackage inputs.package-lock2nix.lib.package-lock2nix { + inherit (pkgs) nodejs; + }; all = lib.packagesFromDirectoryRecursive { - inherit (pkgs) callPackage newScope; + newScope = self: pkgs.newScope (self // { inherit package-lock2nix; }); + inherit (pkgs) callPackage; directory = ./packages; }; in @@ -34,6 +44,7 @@ inherit (all) # keep-sorted start conventional-commit + docsync nix-flake-check-changed nix-grep-to-build npm-list diff --git a/packages/docsync/package-lock.json b/packages/docsync/package-lock.json new file mode 100644 index 0000000..b855cf4 --- /dev/null +++ b/packages/docsync/package-lock.json @@ -0,0 +1,146 @@ +{ + "name": "docsync", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "docsync", + "version": "0.0.1", + "license": "GPL-3.0-only", + "dependencies": { + "tree-sitter": "^0.22.4", + "tree-sitter-python": "^0.23.6", + "tree-sitter-typescript": "^0.23.2" + }, + "bin": { + "docsync-check": "src/docsync-check.ts", + "docsync-get": "src/docsync-get.ts" + }, + "devDependencies": { + "@types/node": "^24.1.0", + "typescript": "^6.0.3" + } + }, + "node_modules/@types/node": { + "version": "24.1.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.1.0.tgz", + "integrity": "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.8.0" + } + }, + "node_modules/node-addon-api": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.5.0.tgz", + "integrity": "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, + "node_modules/tree-sitter": { + "version": "0.22.4", + "resolved": "https://registry.npmjs.org/tree-sitter/-/tree-sitter-0.22.4.tgz", + "integrity": "sha512-usbHZP9/oxNsUY65MQUsduGRqDHQOou1cagUSwjhoSYAmSahjQDAVsh9s+SlZkn8X8+O1FULRGwHu7AFP3kjzg==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.3.0", + "node-gyp-build": "^4.8.4" + } + }, + "node_modules/tree-sitter-javascript": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz", + "integrity": "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-python": { + "version": "0.23.6", + "resolved": "https://registry.npmjs.org/tree-sitter-python/-/tree-sitter-python-0.23.6.tgz", + "integrity": "sha512-yIM9z0oxKIxT7bAtPOhgoVl6gTXlmlIhue7liFT4oBPF/lha7Ha4dQBS82Av6hMMRZoVnFJI8M6mL+SwWoLD3A==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.3.0", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.22.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-typescript": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/tree-sitter-typescript/-/tree-sitter-typescript-0.23.2.tgz", + "integrity": "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2", + "tree-sitter-javascript": "^0.23.1" + }, + "peerDependencies": { + "tree-sitter": "^0.21.0" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz", + "integrity": "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/packages/docsync/package.json b/packages/docsync/package.json new file mode 100644 index 0000000..c477a53 --- /dev/null +++ b/packages/docsync/package.json @@ -0,0 +1,32 @@ +{ + "name": "docsync", + "type": "module", + "version": "0.0.1", + "description": "", + "main": "index.js", + "scripts": { + "start": "node src/index.ts", + "test": "tsc" + }, + "bin": { + "docsync-check": "src/docsync-check.ts", + "docsync-get": "src/docsync-get.ts" + }, + "keywords": [], + "author": "", + "license": "AGPL-3.0-only", + "dependencies": { + "tree-sitter": "^0.22.4", + "tree-sitter-python": "^0.23.6", + "tree-sitter-typescript": "^0.23.2" + }, + "devDependencies": { + "@types/node": "^24.1.0", + "typescript": "^6.0.3" + }, + "overrides": { + "tree-sitter-typescript": { + "tree-sitter": "^0.22.4" + } + } +} diff --git a/packages/docsync/package.nix b/packages/docsync/package.nix new file mode 100644 index 0000000..6794850 --- /dev/null +++ b/packages/docsync/package.nix @@ -0,0 +1,39 @@ +{ + diffutils, + gnused, + jq, + lib, + package-lock2nix, + runCommand, +}: + +package-lock2nix.mkNpmModule { + src = ./.; + doInstallCheck = true; + nativeBuildInputs = [ + diffutils + jq + ]; + postBuild = '' + npm explore tree-sitter -- npm run install + ''; + meta.license = lib.licenses.agpl3Only; + installCheckPhase = + let + fixtures = { + CmdCheck = "docsync-check checks if two directories' doc tags are in sync."; + CmdGet = "docsync-get extracts all docsync nodes under a path."; + }; + fixture = builtins.toFile "fixture" (builtins.toJSON fixtures); + in + '' + $out/bin/docsync-get ${./src} > full-out + diff -u <(jq --sort-keys . ${fixture}) <(jq --sort-keys . full-out) + + $out/bin/docsync-get ${./src} CmdGet > single-out + diff -u ${builtins.toFile "test" (fixtures.CmdGet + "\n")} single-out + + ! $out/bin/docsync-get ${./src} KeyWhichDoesntExistForTestingSake + ! $out/bin/docsync-get ${./src} CmdGet extra-arg + ''; +} diff --git a/packages/docsync/src/cmds.ts b/packages/docsync/src/cmds.ts new file mode 100755 index 0000000..754e4b4 --- /dev/null +++ b/packages/docsync/src/cmds.ts @@ -0,0 +1,74 @@ +import { deepStrictEqual } from "node:assert"; +import { join } from "node:path"; +import process from "node:process"; + +import { PathParser } from "./path-parser.ts"; + +/** + * docsync-check checks if two directories' doc tags are in sync. + * + * CmdCheck + */ +export async function docsyncCheck(): Promise { + const [dirA, dirB] = process.argv.slice(2); + if (!dirA || !dirB) { + console.error(` +Usage: docsync-check + +E.g.: + + $ docsync-check ./typescript/src ./python/src + +`); + process.exit(1); + } + + console.log("Comparing", dirA, "and", dirB); + + const parser = new PathParser(); + const [a, b] = await Promise.all([ + parser.getPath(dirA), + parser.getPath(dirB), + ]); + + deepStrictEqual(a, b); +} + +/** + * docsync-get extracts all docsync nodes under a path. + * + * CmdGet + */ +export async function docsyncGet(): Promise { + const [path, slug, ...extra] = process.argv.slice(2); + if (!path || extra.length > 0) { + console.error(` +Usage: docsync-get [SLUG] + +Example: + + $ docsync-get ./some/example.ts | jq + { + "Foo": "Foo class with its foo docstring", + "Bar": "The Bar class also has an important docstring" + } + + $ docsync-get ./some/example.ts Foo + Foo class with its docstring +`); + process.exit(1); + } + + const parser = new PathParser(); + const m = await parser.getPath(path); + if (slug !== undefined) { + if (!m.has(slug)) { + console.error(`No such key ${slug} in docsync tags for ${path}.`); + process.exit(1); + } + console.log(m.get(slug)); + } else { + const o = Object.fromEntries(m.entries()); + console.log(JSON.stringify(o)); + } +} diff --git a/packages/docsync/src/docsync-check.ts b/packages/docsync/src/docsync-check.ts new file mode 100644 index 0000000..305e4fa --- /dev/null +++ b/packages/docsync/src/docsync-check.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env node --enable-source-maps + +import { docsyncCheck } from "./cmds.ts"; + +await docsyncCheck(); diff --git a/packages/docsync/src/docsync-get.ts b/packages/docsync/src/docsync-get.ts new file mode 100644 index 0000000..281d973 --- /dev/null +++ b/packages/docsync/src/docsync-get.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env node --enable-source-maps + +import { docsyncGet } from "./cmds.ts"; + +await docsyncGet(); diff --git a/packages/docsync/src/path-parser.ts b/packages/docsync/src/path-parser.ts new file mode 100644 index 0000000..38ea762 --- /dev/null +++ b/packages/docsync/src/path-parser.ts @@ -0,0 +1,51 @@ +import { glob, stat } from "node:fs/promises"; +import { extname } from "node:path"; + +import { PythonParser } from "./python.ts"; +import { SentinelParser } from "./sentinel-parser.ts"; +import { TsParser } from "./typescript.ts"; + +import { mergeMaps } from "./utils.ts"; + +export class PathParser { + private readonly parsers: Record; + + constructor() { + this.parsers = { + ".py": new PythonParser(), + ".ts": new TsParser(), + }; + } + + private getParser(path: string): SentinelParser | null { + return this.parsers[extname(path)] || null; + } + + async getFile(path: string): Promise> { + const parser = this.getParser(path); + if (!parser) { + throw new Error(`No parser for ${path}`); + } + return parser.parseFile(path); + } + + async getGlob(globstr: string): Promise> { + const files = await Array.fromAsync(glob(globstr)); + const docstringMap = new Map(); + return mergeMaps(await Promise.all(files.map((x) => this.getFile(x)))); + } + + async getDir(path: string): Promise> { + const exts = Object.keys(this.parsers).join(","); + return this.getGlob(`${path}/**/*{${exts}}`); + } + + async getPath(path: string): Promise> { + const s = await stat(path); + if (s.isDirectory()) { + return this.getDir(path); + } else { + return this.getFile(path); + } + } +} diff --git a/packages/docsync/src/python.ts b/packages/docsync/src/python.ts new file mode 100644 index 0000000..ea629b1 --- /dev/null +++ b/packages/docsync/src/python.ts @@ -0,0 +1,41 @@ +import Parser, { type SyntaxNode } from "tree-sitter"; +import Python from "tree-sitter-python"; + +import { type SentinelDocstring, SentinelParser } from "./sentinel-parser.ts"; + +export class PythonParser extends SentinelParser { + constructor() { + const parser = new Parser(); + parser.setLanguage(Python as any); + super(parser); + } + + protected override extractDocString(node: SyntaxNode): string | null { + if ( + !["module", "class_definition", "function_definition"].includes(node.type) + ) { + return null; + } + const body = node.namedChildren.find( + ({ type }) => type === "block" || type === "suite", + ); + const stmt = body?.namedChildren?.at(0); + if ( + stmt?.type === "expression_statement" && + stmt.firstNamedChild?.type === "string" + ) { + return stmt.firstNamedChild.text; + } + return null; + } + + protected override cleanDocstring(docstring: string): string { + return docstring + .replace(/.*?<\/docsync>/g, "") // remove tags + .replace(/^[ \t]*"""/, "") // remove leading triple quotes + .replace(/"""[ \t]*$/, "") // remove trailing triple quotes + .replace(/\s*\n\s*/g, " ") // replace newlines with spaces + .replace(/\s+/g, " ") // normalize whitespace + .trim(); + } +} diff --git a/packages/docsync/src/sentinel-parser.ts b/packages/docsync/src/sentinel-parser.ts new file mode 100644 index 0000000..0038b21 --- /dev/null +++ b/packages/docsync/src/sentinel-parser.ts @@ -0,0 +1,56 @@ +import { readFile } from "node:fs/promises"; + +import Parser, { type SyntaxNode } from "tree-sitter"; + +import { parseSentinel, treeFold } from "./utils.ts"; + +export interface SentinelDocstring { + sentinel: string; + docstring: string; +} + +export abstract class SentinelParser { + protected abstract cleanDocstring(docstring: string): string; + protected abstract extractDocString(node: SyntaxNode): null | string; + private readonly parser: Parser; + + constructor(parser: Parser) { + this.parser = parser; + } + + protected fetchDocStrings(root: SyntaxNode): string[] { + return treeFold( + root, + (acc: string[], node: SyntaxNode) => { + const comment = this.extractDocString(node); + if (comment) { + acc.push(comment); + } + return acc; + }, + [], + ); + } + + private parseSentinel(docstring: string): null | SentinelDocstring { + const sentinel = parseSentinel(docstring); + if (!sentinel) { + return null; + } + return { sentinel, docstring: this.cleanDocstring(docstring) }; + } + + async parseFile(path: string): Promise> { + const content = await readFile(path, "utf-8"); + const tree = this.parser.parse(content); + const docstrings = this.fetchDocStrings(tree.rootNode); + return new Map( + docstrings + .map((x) => this.parseSentinel(x)) + .flatMap((x) => (x === null ? [] : [[x.sentinel, x.docstring]])) as [ + string, + string, + ][], + ); + } +} diff --git a/packages/docsync/src/typescript.ts b/packages/docsync/src/typescript.ts new file mode 100644 index 0000000..53cf323 --- /dev/null +++ b/packages/docsync/src/typescript.ts @@ -0,0 +1,28 @@ +import Parser, { type SyntaxNode } from "tree-sitter"; +import TS from "tree-sitter-typescript"; +import { glob, readFile } from "node:fs/promises"; +import { type SentinelDocstring, SentinelParser } from "./sentinel-parser.ts"; + +export class TsParser extends SentinelParser { + constructor() { + const parser = new Parser(); + parser.setLanguage(TS.typescript as any); + super(parser); + } + + protected override extractDocString(node: SyntaxNode): string | null { + const prev = node.previousSibling; + return prev && prev.type === "comment" ? prev.text : null; + } + + protected override cleanDocstring(docstring: string): string { + return docstring + .replace(/.*?<\/docsync>/g, "") // remove tags + .replace(/^\/\*\*?/, "") // remove leading "/**" or "/*" + .replace(/\*\/$/, "") // remove trailing "*/" + .replace(/^\s*\*\s?/gm, "") // remove leading "*" + .replace(/\/\/\s?/g, "") // remove line comment prefix + .replace(/\s+/g, " ") + .trim(); + } +} diff --git a/packages/docsync/src/utils.ts b/packages/docsync/src/utils.ts new file mode 100644 index 0000000..cfa468a --- /dev/null +++ b/packages/docsync/src/utils.ts @@ -0,0 +1,23 @@ +import { type SyntaxNode } from "tree-sitter"; + +const sentinelRegex = /(.*?)<\/docsync>/; + +export function mergeMaps(maps: Map[]): Map { + return new Map(maps.flatMap((x) => [...x])); +} + +export function parseSentinel(docstring: string) { + return docstring.match(sentinelRegex)?.at(1); +} + +export function treeFold( + node: SyntaxNode, + callback: (acc: T, node: SyntaxNode) => T, + init: T, +): T { + let acc = callback(init, node); + for (const child of node.namedChildren) { + acc = treeFold(child, callback, acc); + } + return acc; +} diff --git a/packages/docsync/tsconfig.json b/packages/docsync/tsconfig.json new file mode 100644 index 0000000..3f3ae60 --- /dev/null +++ b/packages/docsync/tsconfig.json @@ -0,0 +1,37 @@ +{ + "compilerOptions": { + /* Base Options */ + "esModuleInterop": true, + "skipLibCheck": true, + "target": "esnext", + "moduleDetection": "force", + "isolatedModules": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + + /* Strictness */ + "strict": true, + "noUncheckedIndexedAccess": true, + "noUncheckedSideEffectImports": true, + "noImplicitOverride": true, + "exactOptionalPropertyTypes": true, + "useUnknownInCatchVariables": true, + + /* Module Resolution */ + "module": "nodenext", + "moduleResolution": "nodenext", + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "lib": ["esnext", "dom"], + + /* Build */ + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "rootDir": "src", + "outDir": "dist", + "incremental": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +}