diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4d3014e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,94 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +This repository already has a detailed **AGENTS.md** at the repo root — read it first for +full architecture diagrams, event-handler tables, commit/PR conventions, and common-pattern +recipes (adding a language, a marker type, a CLI command, a config option). This file only +covers what's needed to get moving quickly. + +## What this project is + +sphinx-codelinks is a Sphinx extension providing fast source-code traceability for +Sphinx-Needs: it scans source files (C++, Python, C#, Rust, TypeScript, Go, YAML, JSON) for +marker comments via tree-sitter, and generates Sphinx-Needs items / RST that link +documentation back to exact source locations. + +## Commands + +All commands run through `tox` (uses `tox-uv`). + +```bash +# Run default test env (py312-sphinx8-needs5) +tox + +# List all test env combinations (py{312,313,314}-sphinx{7,8,9}-needs{5,6,7,8}) +tox -a + +# Run a specific env / file / test +tox -e py312-sphinx8-needs5 +tox -e py312-sphinx8-needs5 -- tests/test_analyse.py +tox -e py312-sphinx8-needs5 -- tests/test_analyse.py::test_function_name + +# Update syrupy snapshots +tox -e py312-sphinx8-needs5 -- --snapshot-update + +# Type check / lint / format +tox -e mypy +tox -e ruff-check +tox -e ruff-fmt +pre-commit run --all-files + +# Docs +tox -e docs-clean +tox -e docs-update +BUILDER=linkcheck tox -e docs-clean +tox -e docs-live + +# End-to-end demo (analyse -> write RST -> build docs) +tox -e demo +``` + +The CLI itself is installed as `codelinks` (`codelinks analyse `, +`codelinks write rst --outpath `). + +## Architecture + +Pipeline: **Source Files → Discovery → Parsing → Analysis → Results (JSON) → RST Generation** + +- `source_discover/` — finds source files by include/exclude patterns, respects `.gitignore`. +- `analyse/oneline_parser.py` — tree-sitter based parser extracting comment marker nodes. +- `analyse/projects.py` — per-language analyzers, registered in a `LANGUAGE_ANALYZERS` dict. +- `analyse/analyse.py` — orchestrates discovery + parsing + analysis into `analyse/models.py` + Pydantic result models. +- `needextend_write.py` — turns analysis JSON into RST with Sphinx-Needs `needextend` + directives. +- `config.py` — Pydantic v2 config models (`AnalyseConfig` etc.), loadable from TOML. +- `sphinx_extension/source_tracing.py` — the Sphinx extension `setup()`; wires into Sphinx + build events (`config-inited`, `builder-inited`, `env-before-read-docs`, + `html-collect-pages`, `html-page-context`, `build-finished`) to register sphinx-needs extra + options/types, generate standalone traced-source HTML pages, and inject CSS + (`sphinx_extension/ub_sct.css`). See AGENTS.md for the full event table and mermaid diagram. + +Adding a new language analyzer, marker type, CLI command, or config option each follow a +short recipe documented in AGENTS.md under "Common Patterns" — follow those rather than +inventing a new approach. + +## Code style + +- Ruff for lint/format (strict rule set incl. `S`, `PL`, `PTH`, `SIM`, `SLF`; see + `pyproject.toml` for per-file ignores). +- Mypy strict mode (`disallow_any_*`, `disallow_untyped_*`); relaxed for `tests/*` and + `sphinx_codelinks.*` via overrides in `pyproject.toml`. +- Full type annotations everywhere; Pydantic models (frozen where possible) for config/data. +- Sphinx-style docstrings (`:param:`, `:return:`, `:raises:`), no types in docstrings. +- Prefer pure functions and immutable data structures. + +## Testing + +- `pytest` with fixtures in `tests/conftest.py`; test data in `tests/data/`; Sphinx + integration tests use real minimal Sphinx projects in `tests/doc_test/`. +- `syrupy` for snapshot testing of complex outputs (JSON, doctrees) — use + `snapshot.assert_match()` and re-run with `--snapshot-update` when output intentionally + changes. +- Use `@pytest.mark.parametrize` for multi-language / multi-scenario tests. diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 0f411ed..4a5dcec 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -47,7 +47,7 @@ Limitations **Current Limitations:** -- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported +- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript/JavaScript (``//``, ``/* */``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time Extraction Examples diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 2b2b10a..90a56ee 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -175,7 +175,7 @@ Configures how **Sphinx-CodeLinks** discovers and processes source files within [codelinks.projects.my_project.source_discover] src_dir = "./" - exclude = [] + exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"] include = [] gitignore = true follow_links = false @@ -217,7 +217,7 @@ exclude Defines a list of glob patterns for files and directories to exclude from discovery. This is useful for ignoring build artifacts, temporary files, or specific source files that shouldn't be processed. **Type:** ``list[str]`` -**Default:** ``[]`` +**Default:** ``["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"]`` .. code-block:: toml @@ -236,6 +236,8 @@ Defines a list of glob patterns for files and directories to exclude from discov - ``"**/__pycache__/**"`` - Exclude Python cache directories - ``"node_modules/**"`` - Exclude Node.js dependencies +.. note:: When ``exclude`` is not set, it defaults to a list of common generated-output and dependency directory globs (``node_modules``, ``dist``, ``build``, ``lib``, ``out``, ``coverage``). This matters most for the ``ts`` :ref:`comment_type `, which also discovers ``.js``/``.jsx``/``.mjs``/``.cjs`` files: without this default, checked-in bundler/``tsc`` output would be scanned as source alongside the ``.ts`` it was generated from, producing duplicate need ids for the same marker. Setting ``exclude`` explicitly — including to ``[]`` — replaces this default outright rather than adding to it. + include ^^^^^^^ @@ -271,7 +273,7 @@ Specifies the comment syntax style used in the source code files. This determine **Type:** ``str`` **Default:** ``"cpp"`` -**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"yaml"``, ``"rust"``, ``"go"``, ``"jsonc"``, ``"bash"`` +**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"ts"``, ``"yaml"``, ``"rust"``, ``"go"``, ``"jsonc"``, ``"bash"`` .. code-block:: toml @@ -304,6 +306,12 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` + * - TypeScript / JavaScript + - ``"ts"`` + - ``//`` (single-line), + ``/* */`` (multi-line) + - ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, ``.mjs`` + and ``.cjs`` * - YAML - ``"yaml"`` - ``#`` (single-line) @@ -397,7 +405,7 @@ Configures how **Sphinx-CodeLinks** analyse source files to extract markers from [codelinks.projects.my_project.source_discover] src_dir = "./" - exclude = [] + exclude = ["**/node_modules/**", "**/dist/**", "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"] include = [] gitignore = true follow_links = false @@ -539,6 +547,8 @@ Is equivalent to this RST directive: .. important:: The ``type`` and ``title`` fields must be configured in ``needs_fields`` as they are mandatory for **Sphinx-Needs**. +.. note:: For the TS/JS family (``comment_type = "ts"``), the default ``start_sequence = "@"`` collides with JSDoc tags such as ``@param``, ``@returns``, and ``@deprecated``: a tag description containing a comma is misparsed as a bogus one-line need. Set a more specific ``start_sequence`` (e.g. ``"@need"``) to avoid this. + analyse.need_id_refs ^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 33a2d52..8b78645 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -38,3 +38,13 @@ Usage Examples include = [] exclude = ["tests/**", "setup.py"] comment_type = "python" + +**TypeScript Project:** + +.. code-block:: toml + + [source_discover] + src_dir = "./frontend" + include = ["**/*.ts", "**/*.tsx"] + exclude = ["**/*.test.ts", "**/*.spec.ts"] + comment_type = "ts" diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index f342bc4..6ffe607 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -266,6 +266,44 @@ Features .. fault:: Sphinx-codelinks hallucinates traceability objects in Bash :id: FAULT_BASH_2 +.. feature:: TypeScript Language Support + :id: FE_TS + + Support for defining traceability objects in TypeScript and JavaScript source + files via one-line comment annotations. + + The TypeScript language parser leverages tree-sitter to accurately identify and + extract comments from TypeScript and JavaScript sources, including single-line + (``//``) and multi-line (``/* */``) comment styles. The grammar is chosen per + file from its extension: ``.ts``, ``.mts``, and ``.cts`` — TypeScript's own + module variants — are parsed with the plain TypeScript grammar, since a legacy + angle-bracket type assertion (``x``) is valid there but is JSX syntax + under the TSX grammar. Every other extension (``.tsx``, ``.jsx``, ``.js``, + ``.mjs``, ``.cjs``) is parsed with the TSX grammar, which is safe for plain + JavaScript and additionally handles JSX (including JSX comments such as + ``{/* ... */}``) embedded in ``.tsx`` or ``.js`` sources. + + Key capabilities: + + * Detection of inline and block comments + * Association of comments with function, class, and method declarations + * ``const``/``let``/``var`` declarations count as scopes only when they assign + a function or arrow function + * File extensions ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, + ``.mjs`` and ``.cjs`` auto-discovered when ``comment_type = "ts"`` + + A ``@need-ids:`` reference marker that shares a line with a block comment's + closing ``*/`` — as in a single-line JSDoc comment such as + ``/** @need-ids: ID */`` — has the ``*/`` swallowed into the last need id. + Put the marker on its own line inside the block, or use a ``//`` comment + for reference markers, to avoid this. + + .. fault:: Traceability objects are not detected in TypeScript language + :id: FAULT_TS_1 + + .. fault:: Sphinx-codelinks hallucinates traceability objects in TypeScript + :id: FAULT_TS_2 + .. feature:: Preprocessor-Aware C/C++ Extraction :id: FE_PREPROC diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 420cdd8..78a13ec 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -3,6 +3,48 @@ Changelog ========= +Under development +----------------- + +New and Improved +................ + +- ✨ Added TypeScript comment type support for source discovery and analysis. + + TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``. + The tree-sitter grammar is chosen per file from its extension: ``.ts``, ``.mts``, + and ``.cts`` use the plain TypeScript grammar, and everything else (``.tsx``, + ``.jsx``, ``.js``, ``.mjs``, ``.cjs``) falls back to the TSX grammar. Source + discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, + ``.mjs`` and ``.cjs`` extensions by default. + +Fixes +..... + +- 🐛 Recognized legacy HTML-style comments (````) under ``comment_type = "ts"``. + + The TypeScript and TSX grammars emit these as a separate ``html_comment`` node from + ``comment``. Markers written this way in a ``.js`` source were silently dropped, with + no warning; the extraction query now matches both node kinds. + +- 🐛 Excluded common generated-output and dependency directories from source discovery by default. + + With ``src_dir`` defaulting to ``"./"`` and ``comment_type = "ts"`` also discovering + ``.js``/``.jsx``/``.mjs``/``.cjs`` files, checked-in bundler/``tsc`` output was scanned as + source alongside the ``.ts`` it was generated from, producing duplicate need ids for the + same marker. ``exclude`` now defaults to ``["**/node_modules/**", "**/dist/**", + "**/build/**", "**/lib/**", "**/out/**", "**/coverage/**"]`` when not set explicitly; an + explicit ``exclude`` (including ``[]``) replaces this default outright. + +- 📚 Documented JSDoc caveats for the ``ts`` comment type. + + The default one-line ``start_sequence = "@"`` collides with JSDoc tags (``@param``, + ``@returns``, ``@deprecated``) whose description contains a comma; a more specific + sequence such as ``"@need"`` avoids this. Separately, a ``@need-ids:`` marker sharing a + line with a block comment's closing ``*/`` (as in a single-line JSDoc comment) has the + ``*/`` swallowed into the last need id — keep such markers on their own line, or use + ``//`` comments for reference markers. + .. _`release:1.4.0`: 1.4.0 diff --git a/pyproject.toml b/pyproject.toml index 3e0d54c..84cb24d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ # https://github.com/tree-sitter/py-tree-sitter/issues/386#issuecomment-3101430799 "tree-sitter~=0.25.1", "tree-sitter-c-sharp>=0.23.1", + "tree-sitter-typescript>=0.23.2", "tree-sitter-yaml>=0.7.1", "tree-sitter-rust>=0.23.0", "tree-sitter-go>=0.23.0", diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index c2c4f7a..a26a7bf 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -5,6 +5,7 @@ from typing import Any, TypedDict, cast from tree_sitter import Node as TreeSitterNode +from tree_sitter import Parser, Query from sphinx_codelinks.analyse import utils from sphinx_codelinks.analyse.models import ( @@ -97,9 +98,29 @@ def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: yield src_path, text.encode("utf-8") def create_src_objects(self) -> None: - parser, query = utils.init_tree_sitter(self.analyse_config.comment_type) + comment_type = self.analyse_config.comment_type + # One (parser, query) pair per distinct grammar actually needed, built + # lazily so a parser is never rebuilt per file. Every comment type + # except TypeScript uses a single grammar for the whole run; + # TypeScript alone varies its grammar per file (utils.ts_grammar_key) + # because a legacy TypeScript-only cast parses as JSX under the wrong + # grammar — see the CommentType.ts branch of utils.init_tree_sitter. + parser_cache: dict[str, tuple[Parser, Query]] = {} for src_path, src_string in self.get_src_strings(): + # `comment_type` is normally a CommentType member, but a few call + # sites carry it as a plain (possibly invalid) str instead — see + # SourceAnalyseConfig.comment_type — so key on `str(comment_type)` + # rather than `.value`, which only the enum has. + cache_key = ( + utils.ts_grammar_key(src_path) + if comment_type == CommentType.ts + else str(comment_type) + ) + if cache_key not in parser_cache: + parser_cache[cache_key] = utils.init_tree_sitter(comment_type, src_path) + parser, query = parser_cache[cache_key] + comments: list[TreeSitterNode] | None = utils.extract_comments( src_string, parser, query ) diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 48e726a..8eb4975 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -28,6 +28,14 @@ # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP] CommentType.cpp: {"function_definition", "class_definition"}, CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"}, + # @TypeScript Scope Node Types, IMPL_TS_2, impl, [FE_TS] + CommentType.ts: { + "function_declaration", + "class_declaration", + "method_definition", + "lexical_declaration", + "variable_declaration", + }, # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST]; CommentType.rust: { "function_item", @@ -66,6 +74,15 @@ """ CPP_QUERY = """(comment) @comment""" C_SHARP_QUERY = """(comment) @comment""" +# @TypeScript comment query for tree-sitter, IMPL_TS_3, impl, [FE_TS] +# ``html_comment`` is a separate node kind the TypeScript/TSX grammars emit +# for legacy ```` comments, which are valid in the ``.js`` sources +# this comment type also covers. Without matching it, markers written in that +# style are silently dropped. +TYPE_SCRIPT_QUERY = """ + (comment) @comment + (html_comment) @comment +""" YAML_QUERY = """(comment) @comment""" RUST_QUERY = """ (line_comment) @comment @@ -91,6 +108,25 @@ "null", } +# TypeScript's own module variants. Legacy angle-bracket type assertions +# (``x``) are valid syntax only here, not under the TSX grammar (see the +# CommentType.ts branch of init_tree_sitter for what goes wrong otherwise), so +# these three suffixes get the plain TypeScript grammar and everything else in +# the JS/TS family falls back to TSX. +TS_STRICT_GRAMMAR_SUFFIXES = {".ts", ".mts", ".cts"} + + +def ts_grammar_key(src_path: Path) -> str: + """Return which tree-sitter-typescript grammar ``src_path`` needs. + + ``"typescript"`` for TypeScript's own module variants (``.ts``, ``.mts``, + ``.cts``); ``"tsx"`` for the rest of the JavaScript/TypeScript family + (``.tsx``, ``.jsx``, ``.js``, ``.mjs``, ``.cjs``). Used both to pick the + grammar in ``init_tree_sitter`` and, by callers that parse many files, to + cache one parser per grammar instead of rebuilding one per file. + """ + return "typescript" if src_path.suffix in TS_STRICT_GRAMMAR_SUFFIXES else "tsx" + def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: """Return True if file is likely text, False if binary.""" @@ -107,8 +143,18 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: return False -# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH] -def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: +# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH, FE_TS] +def init_tree_sitter( + comment_type: CommentType, src_path: Path | None = None +) -> tuple[Parser, Query]: + """Build the (parser, query) pair for ``comment_type``. + + ``src_path`` only matters for ``CommentType.ts``, whose grammar varies by + file suffix (see ``ts_grammar_key``); every other comment type ignores it + and uses a single grammar. When ``src_path`` is omitted the TSX grammar is + assumed, which is the safe default for the whole JS/TS family except + TypeScript's own module variants. + """ if comment_type == CommentType.cpp: import tree_sitter_cpp # noqa: PLC0415 @@ -124,6 +170,23 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: parsed_language = Language(tree_sitter_c_sharp.language()) query = Query(parsed_language, C_SHARP_QUERY) + elif comment_type == CommentType.ts: + import tree_sitter_typescript # noqa: PLC0415 + + # Legacy angle-bracket type assertions (``x``) are valid TypeScript + # syntax in .ts/.mts/.cts, but the same text is JSX syntax under the TSX + # grammar: ``x`` parses as a jsx_opening_element and swallows the + # rest of the file into a single jsx_text node, silently dropping every + # marker after it. So the plain TypeScript grammar is required for those + # three suffixes. The TSX grammar remains the fallback for the rest of + # the JS/TS family (.tsx, .jsx, .js, .mjs, .cjs): JavaScript has no such + # cast syntax, so TSX is safe there, and it additionally handles JSX + # embedded in plain .js. + if src_path is not None and ts_grammar_key(src_path) == "typescript": + parsed_language = Language(tree_sitter_typescript.language_typescript()) + else: + parsed_language = Language(tree_sitter_typescript.language_tsx()) + query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 @@ -177,6 +240,38 @@ def extract_comments( return captures.get("comment") +TS_FUNCTION_VALUE_TYPES = {"arrow_function", "function_expression"} + + +def _is_function_like_lexical_declaration(node: TreeSitterNode) -> bool: + """True if a TS lexical/variable declaration's declarator is a function. + + ``const``/``let``/``var`` declarations are only treated as scopes when they + assign a function or arrow function, so a leading comment doesn't bind to an + unrelated ``const`` that merely precedes the function it documents. + """ + for declarator in node.named_children: + if declarator.type != "variable_declarator": + continue + value = declarator.child_by_field_name("value") + if value is not None and value.type in TS_FUNCTION_VALUE_TYPES: + return True + return False + + +def _matches_scope( + node: TreeSitterNode, scope_types: set[str], comment_type: CommentType +) -> bool: + if node.type not in scope_types: + return False + if comment_type == CommentType.ts and node.type in { + "lexical_declaration", + "variable_declaration", + }: + return _is_function_like_lexical_declaration(node) + return True + + def find_enclosing_scope( node: TreeSitterNode, comment_type: CommentType = CommentType.cpp ) -> TreeSitterNode | None: @@ -184,7 +279,7 @@ def find_enclosing_scope( scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp]) current: TreeSitterNode = node while current: - if current.type in scope_types: + if _matches_scope(current, scope_types, comment_type): return current current: TreeSitterNode | None = current.parent # type: ignore[no-redef] # required for node traversal return None @@ -197,12 +292,12 @@ def find_next_scope( scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp]) current: TreeSitterNode = node while current: - if current.type in scope_types: + if _matches_scope(current, scope_types, comment_type): return current current: TreeSitterNode | None = current.next_named_sibling # type: ignore[no-redef] # required for node traversal - if current and current.type == "block": + if current and current.type in {"block", "export_statement"}: for child in current.named_children: - if child.type in scope_types: + if _matches_scope(child, scope_types, comment_type): return child return None diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 0c6d295..97a7668 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -9,6 +9,14 @@ "cpp": ["c", "ci", "cpp", "cc", "cxx", "h", "hpp", "hxx", "hh", "ihl"], "python": ["py"], "cs": ["cs"], + # ".mts"/".cts" are TypeScript's own ESM/CJS module variants. ".js"/".jsx"/ + # ".mjs"/".cjs" are JavaScript. All of these share the "ts" comment type: + # comment syntax is identical across the family, and the analyse stage + # picks the actual tree-sitter grammar per file from the suffix (the plain + # TypeScript grammar for ".ts"/".mts"/".cts", the TSX grammar for + # everything else — see utils.ts_grammar_key), so no separate + # comment_type value is needed here. + "ts": ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"], "yaml": ["yml", "yaml"], "rust": ["rs"], "go": ["go"], @@ -22,10 +30,37 @@ } +# Default ``exclude`` glob patterns applied when a project's configuration does +# not set ``exclude`` explicitly. +# +# ``src_dir`` defaults to ``"./"`` and the ``ts`` comment type claims ``.js``/ +# ``.jsx``/``.mjs``/``.cjs`` in addition to TypeScript's own extensions, so a +# checked-in ``tsc``/bundler output directory (``dist/``, ``build/``, ``lib/``, +# ...) is otherwise scanned as source alongside the ``.ts`` it was generated +# from, producing duplicate need ids for the same marker. These directory +# names are common generated-output or dependency locations across the JS/TS +# ecosystem (and beyond), so excluding them by default avoids that duplication +# for most projects out of the box. +# +# Setting ``exclude`` explicitly in a project's configuration replaces this +# default outright (dataclass fields don't merge) — including setting it to +# ``[]`` to scan everything. +DEFAULT_EXCLUDE = [ + "**/node_modules/**", + "**/dist/**", + "**/build/**", + "**/lib/**", + "**/out/**", + "**/coverage/**", +] + + class CommentType(str, Enum): python = "python" cpp = "cpp" cs = "cs" + # @Support TypeScript style comments, IMPL_TS_1, impl, [FE_TS]; + ts = "ts" yaml = "yaml" # @Support Rust style comments, IMPL_RUST_1, impl, [FE_RUST]; rust = "rust" @@ -71,10 +106,11 @@ def field_names(cls) -> set[str]: """The root of the source directory.""" exclude: list[str] = field( - default_factory=list, + default_factory=lambda: list(DEFAULT_EXCLUDE), metadata={"schema": {"type": "array", "items": {"type": "string"}}}, ) - """The glob pattern to exclude files.""" + """The glob pattern to exclude files. Defaults to ``DEFAULT_EXCLUDE``; set + this explicitly (e.g. to ``[]``) to replace that default outright.""" include: list[str] = field( default_factory=list, diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json new file mode 100644 index 0000000..c53a7fb --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_TS", + "title": "Ts Title", + "type": "impl", + "links": { + "links": [ + "REQ_TS" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json new file mode 100644 index 0000000..5ab935a --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-html_comment_oneliner_js].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_HTML", + "title": "Html Title", + "type": "impl", + "links": { + "links": [ + "REQ_HTML" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json new file mode 100644 index 0000000..b1d1f53 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_TSX", + "title": "Tsx Title", + "type": "impl", + "links": { + "links": [ + "REQ_TSX" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 7ffd5d4..14516dd 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -75,7 +75,9 @@ "config": { "src_dir": "src", "include": [], - "exclude": ["**/build/**"], + "exclude": [ + "**/build/**" + ], "gitignore": false, "comment_type": "cpp" }, @@ -94,7 +96,9 @@ }, "config": { "src_dir": "src", - "include": ["**/*.cpp"], + "include": [ + "**/*.cpp" + ], "exclude": [], "gitignore": false, "comment_type": "cpp" @@ -115,8 +119,12 @@ }, "config": { "src_dir": "src", - "include": ["**/*.cpp"], - "exclude": ["**/test_*.cpp"], + "include": [ + "**/*.cpp" + ], + "exclude": [ + "**/test_*.cpp" + ], "gitignore": false, "comment_type": "cpp" }, @@ -280,7 +288,9 @@ "config": { "src_dir": "src", "include": [], - "exclude": ["**/test_*.cpp"], + "exclude": [ + "**/test_*.cpp" + ], "gitignore": true, "comment_type": "cpp" }, @@ -386,5 +396,39 @@ "expected": [ "src/Program.cs" ] + }, + { + "name": "typescript_comment_type", + "description": "TypeScript comment type discovers the full TypeScript and JavaScript family", + "git_init": false, + "files": { + "src/main.ts": "// main", + "src/component.tsx": "// component", + "src/esm.mts": "// esm module", + "src/cjs.cts": "// cjs module", + "src/script.js": "// script", + "src/widget.jsx": "// widget", + "src/esm.mjs": "// esm js", + "src/legacy.cjs": "// legacy", + "src/main.cpp": "// not ts", + "src/util.py": "# not ts" + }, + "config": { + "src_dir": "src", + "include": [], + "exclude": [], + "gitignore": false, + "comment_type": "ts" + }, + "expected": [ + "src/cjs.cts", + "src/component.tsx", + "src/esm.mjs", + "src/esm.mts", + "src/legacy.cjs", + "src/main.ts", + "src/script.js", + "src/widget.jsx" + ] } ] diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index dd845c7..7df7168 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -12,7 +12,7 @@ Each `*.yaml` file in this directory is a map of `case_name → case`: ```yaml default_oneliner_cpp: - lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash + lang: cpp # cpp | c | cpp_header | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx config: default # "default", or an inline config block (see below) source: | // @My Title, IMPL_1, impl, [REQ_1] diff --git a/tests/data/extraction/oneline.yaml b/tests/data/extraction/oneline.yaml index 9a423ae..ad9beba 100644 --- a/tests/data/extraction/oneline.yaml +++ b/tests/data/extraction/oneline.yaml @@ -57,3 +57,42 @@ shebang_oneliner_bash: #!/bin/bash # @Bash Title, IMPL_BASH_SHEBANG, impl, [REQ_BASH] function greet { echo hi; } + +default_oneliner_typescript: + lang: typescript + config: default + source: | + // @Ts Title, IMPL_TS, impl, [REQ_TS] + function f(): void {} + +# a JSX comment ({/* ... */}) is an ordinary block-comment node in the TSX +# grammar; the marker inside one must extract like any /* */ comment. The +# marker sits on its own line: with the default newline end_sequence, a +# marker sharing a line with the closing */ swallows the */ into its last +# field (pre-existing engine behavior; deliberately not exercised by the +# shared fixtures) +jsx_oneliner_tsx: + lang: tsx + config: default + source: | + const App = () => ( +
+ {/* + @Tsx Title, IMPL_TSX, impl, [REQ_TSX] + */} +
+ ); + +# legacy HTML-style comments (`` sits on +# its own line — sharing the marker's line would swallow it into the last +# field, the same pre-existing engine behavior noted above for jsx_oneliner_tsx +html_comment_oneliner_js: + lang: js + config: default + source: | + + const x = 1; diff --git a/tests/data/typescript/demo.ts b/tests/data/typescript/demo.ts new file mode 100644 index 0000000..66aade0 --- /dev/null +++ b/tests/data/typescript/demo.ts @@ -0,0 +1,17 @@ +// regular comment +function testA() { + // @type,TS_REQ_002,TypeScript one-line test + return 1; +} + +/* regular block comment */ +const testB = () => { + return 2; +}; + +// another comment +class Demo { + methodA() { + return 3; + } +} diff --git a/tests/data/typescript/demo.tsx b/tests/data/typescript/demo.tsx new file mode 100644 index 0000000..a7e7896 --- /dev/null +++ b/tests/data/typescript/demo.tsx @@ -0,0 +1,4 @@ +// @type,TS_REQ_003,TypeScript JSX component test +export function Button() { + return ; +} diff --git a/tests/test_analyse.py b/tests/test_analyse.py index e465a10..cad5bfb 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -56,103 +56,144 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): @pytest.mark.parametrize( - "src_dir, src_paths , oneline_comment_style, result", + "case", [ - ( - TEST_DIR / "data" / "dcdc", - [ + { + "src_dir": TEST_DIR / "data" / "dcdc", + "src_paths": [ TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", TEST_DIR / "data" / "dcdc" / "supercharge.cpp", ], - ONELINE_COMMENT_STYLE, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE, + "result": { "num_src_files": 4, "num_uncached_files": 4, "num_cached_files": 0, "num_comments": 29, "num_oneline_warnings": 0, + "num_oneline_needs": 12, }, - ), - ( - TEST_DIR / "data" / "oneline_comment_basic", - [ + }, + { + "src_dir": TEST_DIR / "data" / "oneline_comment_basic", + "src_paths": [ TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", ], - ONELINE_COMMENT_STYLE, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 14, "num_oneline_warnings": 0, + "num_oneline_needs": 8, "warnings_path_exists": True, }, - ), - ( - TEST_DIR / "data" / "oneline_comment_default", - [ + }, + { + "src_dir": TEST_DIR / "data" / "oneline_comment_default", + "src_paths": [ TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", ], - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 5, "num_oneline_warnings": 1, + "num_oneline_needs": 4, "warnings_path_exists": True, }, - ), - ( - TEST_DIR / "data" / "rust", - [ + }, + { + "src_dir": TEST_DIR / "data" / "rust", + "src_paths": [ TEST_DIR / "data" / "rust" / "demo.rs", ], - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "rust", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 6, "num_oneline_warnings": 0, + "num_oneline_needs": 4, }, - ), - ( - TEST_DIR / "data" / "jsonc", - [ + }, + { + "src_dir": TEST_DIR / "data" / "typescript", + "src_paths": [ + TEST_DIR / "data" / "typescript" / "demo.ts", + ], + "comment_type": "ts", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_comments": 4, + "num_oneline_warnings": 0, + "num_oneline_needs": 1, + }, + }, + { + "src_dir": TEST_DIR / "data" / "typescript", + "src_paths": [ + TEST_DIR / "data" / "typescript" / "demo.tsx", + ], + "comment_type": "ts", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_comments": 1, + "num_oneline_warnings": 0, + "num_oneline_needs": 1, + }, + }, + { + "src_dir": TEST_DIR / "data" / "jsonc", + "src_paths": [ TEST_DIR / "data" / "jsonc" / "demo.jsonc", ], - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": CommentType.jsonc, + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, - "comment_type": CommentType.jsonc, + "num_oneline_needs": 3, }, - ), + }, ], ) -def test_analyse_oneline_needs( - tmp_path, src_dir, src_paths, oneline_comment_style, result -): +def test_analyse_oneline_needs(tmp_path, case): src_analyse_config = SourceAnalyseConfig( - src_files=src_paths, - src_dir=src_dir, + src_files=case["src_paths"], + src_dir=case["src_dir"], get_need_id_refs=False, get_oneline_needs=True, get_rst=False, - oneline_comment_style=oneline_comment_style, - comment_type=result.get("comment_type", CommentType.cpp), + oneline_comment_style=case["oneline_comment_style"], + comment_type=case["comment_type"], ) src_analyse = SourceAnalyse(src_analyse_config) src_analyse.run() + result = case["result"] assert len(src_analyse.src_files) == result["num_src_files"] assert len(src_analyse.oneline_warnings) == result["num_oneline_warnings"] + assert len(src_analyse.oneline_needs) == result["num_oneline_needs"] cnt_comments = 0 for src_file in src_analyse.src_files: diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index bbf348f..1833acb 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -13,6 +13,7 @@ import tree_sitter_json import tree_sitter_python import tree_sitter_rust +import tree_sitter_typescript import tree_sitter_yaml from sphinx_codelinks.analyse import utils @@ -60,6 +61,19 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: return parser, query +@pytest.fixture(scope="session") +def init_typescript_tree_sitter() -> tuple[Parser, Query]: + # The TSX grammar, matching what utils.init_tree_sitter picks for + # CommentType.ts when the file isn't one of TypeScript's own module + # variants (.ts/.mts/.cts) — see utils.ts_grammar_key. Fine for the plain + # TS fixtures below too, since none of them use a legacy angle-bracket + # cast (the one construct where the two grammars disagree). + parsed_language = Language(tree_sitter_typescript.language_tsx()) + query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) + parser = Parser(parsed_language) + return parser, query + + @pytest.fixture(scope="session") def init_go_tree_sitter() -> tuple[Parser, Query]: parsed_language = Language(tree_sitter_go.language()) @@ -481,6 +495,99 @@ def test_find_associated_scope_bash(code, result, init_bash_tree_sitter): assert func_def.startswith(result) +@pytest.mark.parametrize( + ("code", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + class DummyClass { + // @req-id: need_001 + method1() { + } + } + """, + "method1()", + ), + # leading comment on an exported function must descend into + # export_statement, not resolve to no scope + ( + b""" + // @req-id: need_001 + export function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + # leading comment on an exported class + ( + b""" + // @req-id: need_001 + export class DummyClass { + } + """, + "class DummyClass", + ), + # leading comment on an exported const arrow function + ( + b""" + // @req-id: need_001 + export const dummyFunc1 = () => { + }; + """, + "dummyFunc1", + ), + # a plain (non-function) const between the comment and the function it + # documents must not steal the association + ( + b""" + // @req-id: need_001 + const helperFlag = true; + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + # arrow-function const still resolves to itself + ( + b""" + // @req-id: need_001 + const dummyFunc1 = () => { + }; + """, + "dummyFunc1", + ), + # JSX-returning component parses cleanly and resolves scope (.tsx content) + ( + b""" + // @req-id: need_001 + export function Button() { + return ; + } + """, + "function Button()", + ), + ], +) +def test_find_associated_scope_typescript(code, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments = utils.extract_comments(code, parser, query) + node: TreeSitterNode | None = utils.find_associated_scope( + comments[0], CommentType.ts + ) + assert node + assert node.text + ts_def = node.text.decode("utf-8") + assert result in ts_def + + @pytest.mark.parametrize( ("code", "result"), [ @@ -635,6 +742,93 @@ def test_find_next_scope_csharp(code, result, init_csharp_tree_sitter): assert result in func_def +@pytest.mark.parametrize( + ("code", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + // @req-id: need_001 + export function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + // @req-id: need_001 + const helperFlag = true; + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ], +) +def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments = utils.extract_comments(code, parser, query) + node: TreeSitterNode | None = utils.find_next_scope(comments[0], CommentType.ts) + assert node + assert node.text + func_def = node.text.decode("utf-8") + assert result in func_def + + +def test_find_associated_scope_typescript_jsx_no_parse_error( + init_typescript_tree_sitter, +): + """A JSX-returning component must parse cleanly under the TSX grammar.""" + code = b""" + // @req-id: need_001 + export function Button() { + return ; + } + """ + parser, _ = init_typescript_tree_sitter + tree = parser.parse(code) + assert not tree.root_node.has_error + + +def test_typescript_ts_suffix_recovers_markers_around_angle_bracket_cast(): + """A ``.ts`` file must use the plain TypeScript grammar, not TSX. + + ``x`` is a legacy angle-bracket type assertion: valid TypeScript + syntax, but JSX syntax under the TSX grammar. There it parses as a + ``jsx_opening_element`` and swallows the rest of the file into a single + ``jsx_text`` node, silently dropping every marker after it (``has_error`` + is also set). ``init_tree_sitter`` must pick the plain TypeScript grammar + for a ``.ts`` path, via ``ts_grammar_key``, so markers both above and + below the cast all survive. + """ + code = b"""// @Top, IMPL_TOP +const v = x; +// @Bottom, IMPL_BOTTOM +const y = 2; +// @Third, IMPL_THIRD +""" + parser, query = utils.init_tree_sitter(CommentType.ts, Path("dummy.ts")) + tree = parser.parse(code) + assert not tree.root_node.has_error + + comments = utils.extract_comments(code, parser, query) + assert comments is not None + comments.sort(key=lambda node: node.start_point.row) + texts = [node.text.decode("utf-8") for node in comments if node.text] + assert texts == [ + "// @Top, IMPL_TOP", + "// @Bottom, IMPL_BOTTOM", + "// @Third, IMPL_THIRD", + ] + + @pytest.mark.parametrize( ("code", "result"), [ @@ -889,6 +1083,37 @@ def test_csharp_comment(code, num_comments, result, init_csharp_tree_sitter): assert comments[0].text.decode("utf-8") == result +@pytest.mark.parametrize( + ("code", "num_comments", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + 1, + "// @req-id: need_001", + ), + ( + b""" + /* @req-id: need_001 */ + const value = 1; + """, + 1, + "/* @req-id: need_001 */", + ), + ], +) +def test_typescript_comment(code, num_comments, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments: list[TreeSitterNode] = utils.extract_comments(code, parser, query) + comments.sort(key=lambda x: x.start_point.row) + assert len(comments) == num_comments + assert comments[0].text + assert comments[0].text.decode("utf-8") == result + + @pytest.mark.parametrize( ("code", "num_comments", "result"), [ diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 1a0e560..ba10779 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -36,6 +36,21 @@ "go": (CommentType.go, "go"), "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), + "typescript": (CommentType.ts, "ts"), + # `.tsx` matters here: extraction now picks the tree-sitter grammar per + # file from the suffix (utils.ts_grammar_key), and `.tsx` is one of the + # suffixes that gets the TSX grammar rather than the plain TypeScript one + # (the `.ts`/`.mts`/`.cts` suffixes get the latter — see + # utils.init_tree_sitter). What this case pins: a marker on its own line + # inside a multi-line JSX block comment anchors to that line, which + # requires the source (an arrow function returning JSX) to parse cleanly + # under the TSX grammar in the first place. + "tsx": (CommentType.ts, "tsx"), + # `.js` matters here: it is parsed with the TSX grammar (see + # utils.ts_grammar_key), which also emits legacy ```` + # ``html_comment`` nodes as a separate node kind from ``comment`` — this + # case pins that the query captures both. + "js": (CommentType.ts, "js"), } diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 4e8ec8a..ca37bb1 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -7,6 +7,7 @@ from sphinx_codelinks.source_discover.config import ( COMMENT_FILETYPE, + DEFAULT_EXCLUDE, SourceDiscoverConfig, SourceDiscoverConfigType, ) @@ -49,7 +50,7 @@ "comment_type": "java", }, [ - "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'python', 'rust', 'yaml']" + "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'python', 'rust', 'ts', 'yaml']" ], ), ( @@ -99,6 +100,13 @@ def test_schema_negative(config, msgs): "gitignore": True, "comment_type": "python", }, + { + "src_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "comment_type": "ts", + }, { "src_dir": "/path/to/root", "follow_links": True, @@ -182,6 +190,7 @@ def create_source_files(tmp_path: Path) -> Path: [ ("cpp", len(COMMENT_FILETYPE["cpp"])), ("python", len(COMMENT_FILETYPE["python"])), + ("ts", len(COMMENT_FILETYPE["ts"])), ("bash", len(COMMENT_FILETYPE["bash"])), ], ) @@ -209,6 +218,61 @@ def test_jsonc_discover_gate() -> None: assert "plain.json" not in discovered +def _make_generated_output_tree(tmp_path: Path) -> Path: + """Lay out a source file alongside checked-in generated output. + + Mirrors a ``tsc``/bundler output tree: ``src/app.ts`` is the real source, + while ``lib/app.js``, ``dist/app.js`` and ``node_modules/pkg/index.js`` + stand in for generated or vendored output that carries the same marker. + """ + (tmp_path / "src").mkdir() + (tmp_path / "src" / "app.ts").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "lib").mkdir() + (tmp_path / "lib" / "app.js").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "dist").mkdir() + (tmp_path / "dist" / "app.js").write_text( + "// @Feature A, IMPL_1, impl\n", encoding="utf-8" + ) + (tmp_path / "node_modules" / "pkg").mkdir(parents=True) + (tmp_path / "node_modules" / "pkg" / "index.js").write_text( + "// vendored\n", encoding="utf-8" + ) + return tmp_path + + +def test_default_exclude_skips_generated_output(tmp_path: Path) -> None: + """The default ``exclude`` keeps generated/vendored JS out of discovery.""" + src_dir = _make_generated_output_tree(tmp_path) + config = SourceDiscoverConfig(src_dir=src_dir, comment_type="ts", gitignore=False) + assert config.exclude == DEFAULT_EXCLUDE + + discover = SourceDiscover(config) + discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths) + assert discovered == [str(Path("src") / "app.ts")] + + +def test_explicit_exclude_replaces_default(tmp_path: Path) -> None: + """An explicit ``exclude`` (even ``[]``) fully replaces the default list.""" + src_dir = _make_generated_output_tree(tmp_path) + config = SourceDiscoverConfig( + src_dir=src_dir, comment_type="ts", gitignore=False, exclude=[] + ) + assert config.exclude == [] + + discover = SourceDiscover(config) + discovered = sorted(str(p.relative_to(src_dir)) for p in discover.source_paths) + assert discovered == [ + str(Path("dist") / "app.js"), + str(Path("lib") / "app.js"), + str(Path("node_modules") / "pkg" / "index.js"), + str(Path("src") / "app.ts"), + ] + + def test_follow_links(tmp_path: Path) -> None: """Test that follow_links controls whether symbolic links are followed.""" # Create a real directory with a source file diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 7389e81..3fd2689 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -59,7 +59,7 @@ [ "Project 'dcdc' has the following errors:", "Schema validation error in field 'exclude': 123 is not of type 'string'", - "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'python', 'rust', 'yaml']", + "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'python', 'rust', 'ts', 'yaml']", "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", "Schema validation error in field 'include': 345 is not of type 'string'", "Schema validation error in field 'src_dir': ['../dcdc'] is not of type 'string'",