From d569bff8a44fc30cf71eabdefd74ede575cd5510 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:04:00 +0200 Subject: [PATCH 01/11] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20TypeScript=20sup?= =?UTF-8?q?port=20for=20discovery=20and=20analyse=20(#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/components/analyse.rst | 2 +- docs/source/components/configuration.rst | 7 +- docs/source/components/discover.rst | 10 ++ docs/source/development/change_log.rst | 6 +- pyproject.toml | 1 + src/sphinx_codelinks/analyse/utils.py | 13 +++ .../source_discover/config.py | 2 + tests/data/discover_fixtures.json | 44 +++++++-- tests/data/typescript/demo.ts | 17 ++++ tests/test_analyse.py | 24 ++++- tests/test_analyse_utils.py | 98 +++++++++++++++++++ tests/test_source_discover.py | 10 +- tests/test_src_trace.py | 2 +- 13 files changed, 222 insertions(+), 14 deletions(-) create mode 100644 tests/data/typescript/demo.ts diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 2b47c5a..e00f1f1 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -47,7 +47,7 @@ Limitations **Current Limitations:** -- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), Python (``#``), YAML (``#``) and Rust (``//``, ``/* */``, ``///``) comment styles are supported +- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript (``//``, ``/* */``), Python (``#``), YAML (``#``) and Rust (``//``, ``/* */``, ``///``) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time Extraction Examples diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 0d354dd..0f15c9d 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -271,7 +271,7 @@ Specifies the comment syntax style used in the source code files. This determine **Type:** ``str`` **Default:** ``"cpp"`` -**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"yaml"``, ``"rust"`` +**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"ts"``, ``"yaml"``, ``"rust"`` .. code-block:: toml @@ -304,6 +304,11 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` + * - TypeScript + - ``"ts"`` + - ``//`` (single-line), + ``/* */`` (multi-line) + - ``.ts``, ``.tsx`` * - YAML - ``"yaml"`` - ``#`` (single-line) diff --git a/docs/source/components/discover.rst b/docs/source/components/discover.rst index 33a2d52..8b78645 100644 --- a/docs/source/components/discover.rst +++ b/docs/source/components/discover.rst @@ -38,3 +38,13 @@ Usage Examples include = [] exclude = ["tests/**", "setup.py"] comment_type = "python" + +**TypeScript Project:** + +.. code-block:: toml + + [source_discover] + src_dir = "./frontend" + include = ["**/*.ts", "**/*.tsx"] + exclude = ["**/*.test.ts", "**/*.spec.ts"] + comment_type = "ts" diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index e9f20e2..3847ddc 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -7,6 +7,10 @@ Upcoming -------- - ⬆️ Support and test sphinx-needs v5-8 +- ✨ Added TypeScript comment type support for source discovery and analysis. + + TypeScript files can now be processed using ``comment_type = "ts"``. + Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. .. _`release:1.2.0`: @@ -31,7 +35,6 @@ New and Improved Warning messages now include more context to help diagnose parsing issues. - 📚 Added traceability page to the documentation. - - 📚 Added ``features.rst`` page documenting the full feature set with source tracing. Fixes @@ -41,7 +44,6 @@ Fixes Leading and trailing spaces in extracted marker content are now correctly stripped. - .. _`release:1.1.0`: 1.1.0 diff --git a/pyproject.toml b/pyproject.toml index 93cb917..8dc6e87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ # https://github.com/tree-sitter/py-tree-sitter/issues/386#issuecomment-3101430799 "tree-sitter~=0.25.1", "tree-sitter-c-sharp>=0.23.1", + "tree-sitter-typescript>=0.23.2", "tree-sitter-yaml>=0.7.1", "tree-sitter-rust>=0.23.0", ] diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 5a11fdd..b96cd5c 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -19,6 +19,13 @@ # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP] CommentType.cpp: {"function_definition", "class_definition"}, CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"}, + CommentType.ts: { + "function_declaration", + "class_declaration", + "method_definition", + "lexical_declaration", + "variable_declaration", + }, CommentType.yaml: {"block_mapping_pair", "block_sequence_item", "document"}, # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST]; CommentType.rust: { @@ -55,6 +62,7 @@ """ CPP_QUERY = """(comment) @comment""" C_SHARP_QUERY = """(comment) @comment""" +TYPE_SCRIPT_QUERY = """(comment) @comment""" YAML_QUERY = """(comment) @comment""" RUST_QUERY = """ (line_comment) @comment @@ -94,6 +102,11 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: parsed_language = Language(tree_sitter_c_sharp.language()) query = Query(parsed_language, C_SHARP_QUERY) + elif comment_type == CommentType.ts: + import tree_sitter_typescript # noqa: PLC0415 + + parsed_language = Language(tree_sitter_typescript.language_typescript()) + query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 51ae3c0..45a51fd 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -9,6 +9,7 @@ "cpp": ["c", "ci", "cpp", "cc", "cxx", "h", "hpp", "hxx", "hh", "ihl"], "python": ["py"], "cs": ["cs"], + "ts": ["ts", "tsx"], "yaml": ["yml", "yaml"], "rust": ["rs"], } @@ -18,6 +19,7 @@ class CommentType(str, Enum): python = "python" cpp = "cpp" cs = "cs" + ts = "ts" yaml = "yaml" # @Support Rust style comments, IMPL_RUST_1, impl, [FE_RUST]; rust = "rust" diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 7ffd5d4..738c619 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -75,7 +75,9 @@ "config": { "src_dir": "src", "include": [], - "exclude": ["**/build/**"], + "exclude": [ + "**/build/**" + ], "gitignore": false, "comment_type": "cpp" }, @@ -94,7 +96,9 @@ }, "config": { "src_dir": "src", - "include": ["**/*.cpp"], + "include": [ + "**/*.cpp" + ], "exclude": [], "gitignore": false, "comment_type": "cpp" @@ -115,8 +119,12 @@ }, "config": { "src_dir": "src", - "include": ["**/*.cpp"], - "exclude": ["**/test_*.cpp"], + "include": [ + "**/*.cpp" + ], + "exclude": [ + "**/test_*.cpp" + ], "gitignore": false, "comment_type": "cpp" }, @@ -280,7 +288,9 @@ "config": { "src_dir": "src", "include": [], - "exclude": ["**/test_*.cpp"], + "exclude": [ + "**/test_*.cpp" + ], "gitignore": true, "comment_type": "cpp" }, @@ -386,5 +396,27 @@ "expected": [ "src/Program.cs" ] + }, + { + "name": "typescript_comment_type", + "description": "TypeScript comment type discovers .ts and .tsx files", + "git_init": false, + "files": { + "src/main.ts": "// main", + "src/component.tsx": "// component", + "src/main.cpp": "// not ts", + "src/util.py": "# not ts" + }, + "config": { + "src_dir": "src", + "include": [], + "exclude": [], + "gitignore": false, + "comment_type": "ts" + }, + "expected": [ + "src/component.tsx", + "src/main.ts" + ] } -] +] \ No newline at end of file diff --git a/tests/data/typescript/demo.ts b/tests/data/typescript/demo.ts new file mode 100644 index 0000000..66aade0 --- /dev/null +++ b/tests/data/typescript/demo.ts @@ -0,0 +1,17 @@ +// regular comment +function testA() { + // @type,TS_REQ_002,TypeScript one-line test + return 1; +} + +/* regular block comment */ +const testB = () => { + return 2; +}; + +// another comment +class Demo { + methodA() { + return 3; + } +} diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 6e6c2a7..500451c 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -55,7 +55,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): @pytest.mark.parametrize( - "src_dir, src_paths , oneline_comment_style, result", + "src_dir, src_paths, comment_type, oneline_comment_style, result", [ ( TEST_DIR / "data" / "dcdc", @@ -65,6 +65,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", TEST_DIR / "data" / "dcdc" / "supercharge.cpp", ], + "cpp", ONELINE_COMMENT_STYLE, { "num_src_files": 4, @@ -79,6 +80,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): [ TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", ], + "cpp", ONELINE_COMMENT_STYLE, { "num_src_files": 1, @@ -94,6 +96,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): [ TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", ], + "cpp", ONELINE_COMMENT_STYLE_DEFAULT, { "num_src_files": 1, @@ -109,6 +112,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): [ TEST_DIR / "data" / "rust" / "demo.rs", ], + "rust", ONELINE_COMMENT_STYLE_DEFAULT, { "num_src_files": 1, @@ -118,10 +122,25 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_oneline_warnings": 0, }, ), + ( + TEST_DIR / "data" / "typescript", + [ + TEST_DIR / "data" / "typescript" / "demo.ts", + ], + "ts", + ONELINE_COMMENT_STYLE_DEFAULT, + { + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_comments": 4, + "num_oneline_warnings": 0, + }, + ), ], ) def test_analyse_oneline_needs( - tmp_path, src_dir, src_paths, oneline_comment_style, result + tmp_path, src_dir, src_paths, comment_type, oneline_comment_style, result ): src_analyse_config = SourceAnalyseConfig( src_files=src_paths, @@ -130,6 +149,7 @@ def test_analyse_oneline_needs( get_oneline_needs=True, get_rst=False, oneline_comment_style=oneline_comment_style, + comment_type=comment_type, ) src_analyse = SourceAnalyse(src_analyse_config) src_analyse.run() diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index 207b1f9..c1a2dc8 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -10,6 +10,7 @@ import tree_sitter_cpp import tree_sitter_python import tree_sitter_rust +import tree_sitter_typescript import tree_sitter_yaml from sphinx_codelinks.analyse import utils @@ -57,6 +58,14 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: return parser, query +@pytest.fixture(scope="session") +def init_typescript_tree_sitter() -> tuple[Parser, Query]: + parsed_language = Language(tree_sitter_typescript.language_typescript()) + query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) + parser = Parser(parsed_language) + return parser, query + + @pytest.mark.parametrize( ("code", "result"), [ @@ -365,6 +374,41 @@ def test_find_associated_scope_rust(code, result, init_rust_tree_sitter): assert result in rust_def +@pytest.mark.parametrize( + ("code", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + class DummyClass { + // @req-id: need_001 + method1() { + } + } + """, + "method1()", + ), + ], +) +def test_find_associated_scope_typescript(code, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments = utils.extract_comments(code, parser, query) + node: TreeSitterNode | None = utils.find_associated_scope( + comments[0], CommentType.ts + ) + assert node + assert node.text + ts_def = node.text.decode("utf-8") + assert result in ts_def + + @pytest.mark.parametrize( ("code", "result"), [ @@ -519,6 +563,29 @@ def test_find_next_scope_csharp(code, result, init_csharp_tree_sitter): assert result in func_def +@pytest.mark.parametrize( + ("code", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ], +) +def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments = utils.extract_comments(code, parser, query) + node: TreeSitterNode | None = utils.find_next_scope(comments[0], CommentType.ts) + assert node + assert node.text + func_def = node.text.decode("utf-8") + assert result in func_def + + @pytest.mark.parametrize( ("code", "result"), [ @@ -773,6 +840,37 @@ def test_csharp_comment(code, num_comments, result, init_csharp_tree_sitter): assert comments[0].text.decode("utf-8") == result +@pytest.mark.parametrize( + ("code", "num_comments", "result"), + [ + ( + b""" + // @req-id: need_001 + function dummyFunc1() { + } + """, + 1, + "// @req-id: need_001", + ), + ( + b""" + /* @req-id: need_001 */ + const value = 1; + """, + 1, + "/* @req-id: need_001 */", + ), + ], +) +def test_typescript_comment(code, num_comments, result, init_typescript_tree_sitter): + parser, query = init_typescript_tree_sitter + comments: list[TreeSitterNode] = utils.extract_comments(code, parser, query) + comments.sort(key=lambda x: x.start_point.row) + assert len(comments) == num_comments + assert comments[0].text + assert comments[0].text.decode("utf-8") == result + + @pytest.mark.parametrize( ("code", "num_comments", "result"), [ diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 5a6f5c1..30decf4 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -49,7 +49,7 @@ "comment_type": "java", }, [ - "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'yaml']" + "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'ts', 'yaml']" ], ), ( @@ -99,6 +99,13 @@ def test_schema_negative(config, msgs): "gitignore": True, "comment_type": "python", }, + { + "src_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "comment_type": "ts", + }, { "src_dir": "/path/to/root", "follow_links": True, @@ -182,6 +189,7 @@ def create_source_files(tmp_path: Path) -> Path: [ ("cpp", len(COMMENT_FILETYPE["cpp"])), ("python", len(COMMENT_FILETYPE["python"])), + ("ts", len(COMMENT_FILETYPE["ts"])), ], ) def test_comment_filetype( diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 8e87a71..2ecc3fd 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -58,7 +58,7 @@ [ "Project 'dcdc' has the following errors:", "Schema validation error in field 'exclude': 123 is not of type 'string'", - "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'yaml']", + "Schema validation error in field 'comment_type': 'java' is not one of ['cpp', 'cs', 'python', 'rust', 'ts', 'yaml']", "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", "Schema validation error in field 'include': 345 is not of type 'string'", "Schema validation error in field 'src_dir': ['../dcdc'] is not of type 'string'", From 4e9ef51d27534fffd4e67f29487388dd8f44feb8 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:17:25 +0200 Subject: [PATCH 02/11] =?UTF-8?q?=E2=9C=A8=20Update=20changelog:=20Add=20T?= =?UTF-8?q?ypeScript=20comment=20type=20support=20for=20source=20discovery?= =?UTF-8?q?=20and=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/development/change_log.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index fe2e138..4901b69 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -3,6 +3,14 @@ Changelog ========= +Under development +----------------- + +- ✨ Added TypeScript comment type support for source discovery and analysis. + + TypeScript files can now be processed using ``comment_type = "ts"``. + Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. + .. _`release:1.3.0`: 1.3.0 @@ -23,11 +31,6 @@ New and Improved Comments in JSONC files are now parsed for need ID references and one-line need definitions. ``.json`` files are also checked when they begin with a comment (see jsonc.org). -- ✨ Added TypeScript comment type support for source discovery and analysis. - - TypeScript files can now be processed using ``comment_type = "ts"``. - Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. - - 👌 Replaced ``gitignore-parser`` with ``ignore-python`` for source discovery. This adds native nested ``.gitignore`` support, improves performance, and brings behavioral From 4d8509481f5a11e40de5b88f8e6205b423990c49 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:18:13 +0200 Subject: [PATCH 03/11] =?UTF-8?q?=E2=9C=A8=20Update=20changelog:=20Add=20T?= =?UTF-8?q?ypeScript=20support=20details=20for=20source=20discovery=20and?= =?UTF-8?q?=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/development/change_log.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 4901b69..cfb8246 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -6,11 +6,17 @@ Changelog Under development ----------------- +New and Improved +................ + - ✨ Added TypeScript comment type support for source discovery and analysis. TypeScript files can now be processed using ``comment_type = "ts"``. Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. +Fixes +..... + .. _`release:1.3.0`: 1.3.0 From 28e12e1c07e31cc8f58fff3c91beaa77d10e5852 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 22 Jun 2026 16:28:21 +0200 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Resolve=20CI=20pre-?= =?UTF-8?q?commit=20and=20docs=20build=20failures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/components/configuration.rst | 10 +-- tests/data/discover_fixtures.json | 2 +- tests/test_analyse.py | 99 ++++++++++++------------ 3 files changed, 55 insertions(+), 56 deletions(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index cdad9d8..8e6d36d 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -304,11 +304,11 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` - * - TypeScript - - ``"ts"`` - - ``//`` (single-line), - ``/* */`` (multi-line) - - ``.ts``, ``.tsx`` + * - TypeScript + - ``"ts"`` + - ``//`` (single-line), + ``/* */`` (multi-line) + - ``.ts``, ``.tsx`` * - YAML - ``"yaml"`` - ``#`` (single-line) diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 738c619..5959cab 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -419,4 +419,4 @@ "src/main.ts" ] } -] \ No newline at end of file +] diff --git a/tests/test_analyse.py b/tests/test_analyse.py index 2973ed6..fe7098a 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -56,34 +56,34 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): @pytest.mark.parametrize( - "src_dir, src_paths, comment_type, oneline_comment_style, result", + "case", [ - ( - TEST_DIR / "data" / "dcdc", - [ + { + "src_dir": TEST_DIR / "data" / "dcdc", + "src_paths": [ TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", TEST_DIR / "data" / "dcdc" / "supercharge.cpp", ], - "cpp", - ONELINE_COMMENT_STYLE, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE, + "result": { "num_src_files": 4, "num_uncached_files": 4, "num_cached_files": 0, "num_comments": 29, "num_oneline_warnings": 0, }, - ), - ( - TEST_DIR / "data" / "oneline_comment_basic", - [ + }, + { + "src_dir": TEST_DIR / "data" / "oneline_comment_basic", + "src_paths": [ TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", ], - "cpp", - ONELINE_COMMENT_STYLE, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, @@ -91,15 +91,15 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_oneline_warnings": 0, "warnings_path_exists": True, }, - ), - ( - TEST_DIR / "data" / "oneline_comment_default", - [ + }, + { + "src_dir": TEST_DIR / "data" / "oneline_comment_default", + "src_paths": [ TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", ], - "cpp", - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "cpp", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, @@ -107,69 +107,68 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_oneline_warnings": 1, "warnings_path_exists": True, }, - ), - ( - TEST_DIR / "data" / "rust", - [ + }, + { + "src_dir": TEST_DIR / "data" / "rust", + "src_paths": [ TEST_DIR / "data" / "rust" / "demo.rs", ], - "rust", - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "rust", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 6, "num_oneline_warnings": 0, }, - ), - ( - TEST_DIR / "data" / "typescript", - [ + }, + { + "src_dir": TEST_DIR / "data" / "typescript", + "src_paths": [ TEST_DIR / "data" / "typescript" / "demo.ts", ], - "ts", - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": "ts", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, }, - ), - ( - TEST_DIR / "data" / "jsonc", - [ + }, + { + "src_dir": TEST_DIR / "data" / "jsonc", + "src_paths": [ TEST_DIR / "data" / "jsonc" / "demo.jsonc", ], - CommentType.jsonc, - ONELINE_COMMENT_STYLE_DEFAULT, - { + "comment_type": CommentType.jsonc, + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { "num_src_files": 1, "num_uncached_files": 1, "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, }, - ), + }, ], ) -def test_analyse_oneline_needs( - tmp_path, src_dir, src_paths, comment_type, oneline_comment_style, result -): +def test_analyse_oneline_needs(tmp_path, case): src_analyse_config = SourceAnalyseConfig( - src_files=src_paths, - src_dir=src_dir, + src_files=case["src_paths"], + src_dir=case["src_dir"], get_need_id_refs=False, get_oneline_needs=True, get_rst=False, - oneline_comment_style=oneline_comment_style, - comment_type=comment_type, + oneline_comment_style=case["oneline_comment_style"], + comment_type=case["comment_type"], ) src_analyse = SourceAnalyse(src_analyse_config) src_analyse.run() + result = case["result"] assert len(src_analyse.src_files) == result["num_src_files"] assert len(src_analyse.oneline_warnings) == result["num_oneline_warnings"] From 5e7d48beadf14a7f8ec9d3d0ffa5797f209cf4b8 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 13 Jul 2026 20:48:38 +0200 Subject: [PATCH 05/11] =?UTF-8?q?=E2=9C=A8=20NEW:=20Add=20CLAUDE.md=20for?= =?UTF-8?q?=20project=20guidance=20and=20command=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4d3014e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,94 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +This repository already has a detailed **AGENTS.md** at the repo root — read it first for +full architecture diagrams, event-handler tables, commit/PR conventions, and common-pattern +recipes (adding a language, a marker type, a CLI command, a config option). This file only +covers what's needed to get moving quickly. + +## What this project is + +sphinx-codelinks is a Sphinx extension providing fast source-code traceability for +Sphinx-Needs: it scans source files (C++, Python, C#, Rust, TypeScript, Go, YAML, JSON) for +marker comments via tree-sitter, and generates Sphinx-Needs items / RST that link +documentation back to exact source locations. + +## Commands + +All commands run through `tox` (uses `tox-uv`). + +```bash +# Run default test env (py312-sphinx8-needs5) +tox + +# List all test env combinations (py{312,313,314}-sphinx{7,8,9}-needs{5,6,7,8}) +tox -a + +# Run a specific env / file / test +tox -e py312-sphinx8-needs5 +tox -e py312-sphinx8-needs5 -- tests/test_analyse.py +tox -e py312-sphinx8-needs5 -- tests/test_analyse.py::test_function_name + +# Update syrupy snapshots +tox -e py312-sphinx8-needs5 -- --snapshot-update + +# Type check / lint / format +tox -e mypy +tox -e ruff-check +tox -e ruff-fmt +pre-commit run --all-files + +# Docs +tox -e docs-clean +tox -e docs-update +BUILDER=linkcheck tox -e docs-clean +tox -e docs-live + +# End-to-end demo (analyse -> write RST -> build docs) +tox -e demo +``` + +The CLI itself is installed as `codelinks` (`codelinks analyse `, +`codelinks write rst --outpath `). + +## Architecture + +Pipeline: **Source Files → Discovery → Parsing → Analysis → Results (JSON) → RST Generation** + +- `source_discover/` — finds source files by include/exclude patterns, respects `.gitignore`. +- `analyse/oneline_parser.py` — tree-sitter based parser extracting comment marker nodes. +- `analyse/projects.py` — per-language analyzers, registered in a `LANGUAGE_ANALYZERS` dict. +- `analyse/analyse.py` — orchestrates discovery + parsing + analysis into `analyse/models.py` + Pydantic result models. +- `needextend_write.py` — turns analysis JSON into RST with Sphinx-Needs `needextend` + directives. +- `config.py` — Pydantic v2 config models (`AnalyseConfig` etc.), loadable from TOML. +- `sphinx_extension/source_tracing.py` — the Sphinx extension `setup()`; wires into Sphinx + build events (`config-inited`, `builder-inited`, `env-before-read-docs`, + `html-collect-pages`, `html-page-context`, `build-finished`) to register sphinx-needs extra + options/types, generate standalone traced-source HTML pages, and inject CSS + (`sphinx_extension/ub_sct.css`). See AGENTS.md for the full event table and mermaid diagram. + +Adding a new language analyzer, marker type, CLI command, or config option each follow a +short recipe documented in AGENTS.md under "Common Patterns" — follow those rather than +inventing a new approach. + +## Code style + +- Ruff for lint/format (strict rule set incl. `S`, `PL`, `PTH`, `SIM`, `SLF`; see + `pyproject.toml` for per-file ignores). +- Mypy strict mode (`disallow_any_*`, `disallow_untyped_*`); relaxed for `tests/*` and + `sphinx_codelinks.*` via overrides in `pyproject.toml`. +- Full type annotations everywhere; Pydantic models (frozen where possible) for config/data. +- Sphinx-style docstrings (`:param:`, `:return:`, `:raises:`), no types in docstrings. +- Prefer pure functions and immutable data structures. + +## Testing + +- `pytest` with fixtures in `tests/conftest.py`; test data in `tests/data/`; Sphinx + integration tests use real minimal Sphinx projects in `tests/doc_test/`. +- `syrupy` for snapshot testing of complex outputs (JSON, doctrees) — use + `snapshot.assert_match()` and re-run with `--snapshot-update` when output intentionally + changes. +- Use `@pytest.mark.parametrize` for multi-language / multi-scenario tests. From b36c271396a42d26e9cd7bba5e04a023acc773d5 Mon Sep 17 00:00:00 2001 From: Arnaud Riess Date: Mon, 13 Jul 2026 20:57:42 +0200 Subject: [PATCH 06/11] =?UTF-8?q?=E2=9C=A8=20Add=20TypeScript=20support=20?= =?UTF-8?q?for=20TSX=20files=20and=20enhance=20related=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/sphinx_codelinks/analyse/utils.py | 45 +++++++++++-- tests/data/typescript/demo.tsx | 4 ++ tests/test_analyse.py | 23 +++++++ tests/test_analyse_utils.py | 94 ++++++++++++++++++++++++++- 4 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 tests/data/typescript/demo.tsx diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 8fcef2e..050a2c1 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -131,7 +131,10 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: elif comment_type == CommentType.ts: import tree_sitter_typescript # noqa: PLC0415 - parsed_language = Language(tree_sitter_typescript.language_typescript()) + # The TSX grammar is a strict superset of the TypeScript grammar (it also + # parses plain .ts fine), so use it for both to support .tsx files without + # needing a per-file grammar choice. + parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 @@ -181,6 +184,38 @@ def extract_comments( return captures.get("comment") +TS_FUNCTION_VALUE_TYPES = {"arrow_function", "function_expression"} + + +def _is_function_like_lexical_declaration(node: TreeSitterNode) -> bool: + """True if a TS lexical/variable declaration's declarator is a function. + + ``const``/``let``/``var`` declarations are only treated as scopes when they + assign a function or arrow function, so a leading comment doesn't bind to an + unrelated ``const`` that merely precedes the function it documents. + """ + for declarator in node.named_children: + if declarator.type != "variable_declarator": + continue + value = declarator.child_by_field_name("value") + if value is not None and value.type in TS_FUNCTION_VALUE_TYPES: + return True + return False + + +def _matches_scope( + node: TreeSitterNode, scope_types: set[str], comment_type: CommentType +) -> bool: + if node.type not in scope_types: + return False + if comment_type == CommentType.ts and node.type in { + "lexical_declaration", + "variable_declaration", + }: + return _is_function_like_lexical_declaration(node) + return True + + def find_enclosing_scope( node: TreeSitterNode, comment_type: CommentType = CommentType.cpp ) -> TreeSitterNode | None: @@ -188,7 +223,7 @@ def find_enclosing_scope( scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp]) current: TreeSitterNode = node while current: - if current.type in scope_types: + if _matches_scope(current, scope_types, comment_type): return current current: TreeSitterNode | None = current.parent # type: ignore[no-redef] # required for node traversal return None @@ -201,12 +236,12 @@ def find_next_scope( scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp]) current: TreeSitterNode = node while current: - if current.type in scope_types: + if _matches_scope(current, scope_types, comment_type): return current current: TreeSitterNode | None = current.next_named_sibling # type: ignore[no-redef] # required for node traversal - if current and current.type == "block": + if current and current.type in {"block", "export_statement"}: for child in current.named_children: - if child.type in scope_types: + if _matches_scope(child, scope_types, comment_type): return child return None diff --git a/tests/data/typescript/demo.tsx b/tests/data/typescript/demo.tsx new file mode 100644 index 0000000..a7e7896 --- /dev/null +++ b/tests/data/typescript/demo.tsx @@ -0,0 +1,4 @@ +// @type,TS_REQ_003,TypeScript JSX component test +export function Button() { + return ; +} diff --git a/tests/test_analyse.py b/tests/test_analyse.py index fe7098a..cad5bfb 100644 --- a/tests/test_analyse.py +++ b/tests/test_analyse.py @@ -74,6 +74,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 29, "num_oneline_warnings": 0, + "num_oneline_needs": 12, }, }, { @@ -89,6 +90,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 14, "num_oneline_warnings": 0, + "num_oneline_needs": 8, "warnings_path_exists": True, }, }, @@ -105,6 +107,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 5, "num_oneline_warnings": 1, + "num_oneline_needs": 4, "warnings_path_exists": True, }, }, @@ -121,6 +124,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 6, "num_oneline_warnings": 0, + "num_oneline_needs": 4, }, }, { @@ -136,6 +140,23 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, + "num_oneline_needs": 1, + }, + }, + { + "src_dir": TEST_DIR / "data" / "typescript", + "src_paths": [ + TEST_DIR / "data" / "typescript" / "demo.tsx", + ], + "comment_type": "ts", + "oneline_comment_style": ONELINE_COMMENT_STYLE_DEFAULT, + "result": { + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_comments": 1, + "num_oneline_warnings": 0, + "num_oneline_needs": 1, }, }, { @@ -151,6 +172,7 @@ def test_analyse(src_dir, src_paths, tmp_path, snapshot_marks): "num_cached_files": 0, "num_comments": 4, "num_oneline_warnings": 0, + "num_oneline_needs": 3, }, }, ], @@ -171,6 +193,7 @@ def test_analyse_oneline_needs(tmp_path, case): result = case["result"] assert len(src_analyse.src_files) == result["num_src_files"] assert len(src_analyse.oneline_warnings) == result["num_oneline_warnings"] + assert len(src_analyse.oneline_needs) == result["num_oneline_needs"] cnt_comments = 0 for src_file in src_analyse.src_files: diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index 95f5b9b..9841014 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -62,7 +62,9 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: @pytest.fixture(scope="session") def init_typescript_tree_sitter() -> tuple[Parser, Query]: - parsed_language = Language(tree_sitter_typescript.language_typescript()) + # TSX grammar is a superset of the TypeScript grammar (parses plain .ts too), + # matching what utils.init_tree_sitter uses for CommentType.ts. + parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) parser = Parser(parsed_language) return parser, query @@ -455,6 +457,64 @@ class DummyClass { """, "method1()", ), + # leading comment on an exported function must descend into + # export_statement, not resolve to no scope + ( + b""" + // @req-id: need_001 + export function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + # leading comment on an exported class + ( + b""" + // @req-id: need_001 + export class DummyClass { + } + """, + "class DummyClass", + ), + # leading comment on an exported const arrow function + ( + b""" + // @req-id: need_001 + export const dummyFunc1 = () => { + }; + """, + "dummyFunc1", + ), + # a plain (non-function) const between the comment and the function it + # documents must not steal the association + ( + b""" + // @req-id: need_001 + const helperFlag = true; + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + # arrow-function const still resolves to itself + ( + b""" + // @req-id: need_001 + const dummyFunc1 = () => { + }; + """, + "dummyFunc1", + ), + # JSX-returning component parses cleanly and resolves scope (.tsx content) + ( + b""" + // @req-id: need_001 + export function Button() { + return ; + } + """, + "function Button()", + ), ], ) def test_find_associated_scope_typescript(code, result, init_typescript_tree_sitter): @@ -634,6 +694,23 @@ def test_find_next_scope_csharp(code, result, init_csharp_tree_sitter): """, "function dummyFunc1()", ), + ( + b""" + // @req-id: need_001 + export function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), + ( + b""" + // @req-id: need_001 + const helperFlag = true; + function dummyFunc1() { + } + """, + "function dummyFunc1()", + ), ], ) def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): @@ -646,6 +723,21 @@ def test_find_next_scope_typescript(code, result, init_typescript_tree_sitter): assert result in func_def +def test_find_associated_scope_typescript_jsx_no_parse_error( + init_typescript_tree_sitter, +): + """A JSX-returning component must parse cleanly under the TSX grammar.""" + code = b""" + // @req-id: need_001 + export function Button() { + return ; + } + """ + parser, _ = init_typescript_tree_sitter + tree = parser.parse(code) + assert not tree.root_node.has_error + + @pytest.mark.parametrize( ("code", "result"), [ From 852e50c821e416b480a914d322e586300c23fecd Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 7 Aug 2026 16:13:37 +0200 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=A7=AA=20TEST:=20Add=20TypeScript?= =?UTF-8?q?=20declarative=20extraction=20fixture=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...[oneline-default_oneliner_typescript].json | 19 ++++++++++++++ ...ion_fixture[oneline-jsx_oneliner_tsx].json | 19 ++++++++++++++ tests/data/extraction/README.md | 2 +- tests/data/extraction/oneline.yaml | 25 +++++++++++++++++++ tests/test_extraction_fixtures.py | 4 +++ 5 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json new file mode 100644 index 0000000..c53a7fb --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_typescript].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_TS", + "title": "Ts Title", + "type": "impl", + "links": { + "links": [ + "REQ_TS" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json new file mode 100644 index 0000000..b1d1f53 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-jsx_oneliner_tsx].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_TSX", + "title": "Tsx Title", + "type": "impl", + "links": { + "links": [ + "REQ_TSX" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index dd845c7..2dd95f1 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -12,7 +12,7 @@ Each `*.yaml` file in this directory is a map of `case_name → case`: ```yaml default_oneliner_cpp: - lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash + lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx config: default # "default", or an inline config block (see below) source: | // @My Title, IMPL_1, impl, [REQ_1] diff --git a/tests/data/extraction/oneline.yaml b/tests/data/extraction/oneline.yaml index 9a423ae..7976224 100644 --- a/tests/data/extraction/oneline.yaml +++ b/tests/data/extraction/oneline.yaml @@ -57,3 +57,28 @@ shebang_oneliner_bash: #!/bin/bash # @Bash Title, IMPL_BASH_SHEBANG, impl, [REQ_BASH] function greet { echo hi; } + +default_oneliner_typescript: + lang: typescript + config: default + source: | + // @Ts Title, IMPL_TS, impl, [REQ_TS] + function f(): void {} + +# a JSX comment ({/* ... */}) is an ordinary block-comment node in the TSX +# grammar; the marker inside one must extract like any /* */ comment. The +# marker sits on its own line: with the default newline end_sequence, a +# marker sharing a line with the closing */ swallows the */ into its last +# field (pre-existing engine behavior; deliberately not exercised by the +# shared fixtures) +jsx_oneliner_tsx: + lang: tsx + config: default + source: | + const App = () => ( +
+ {/* + @Tsx Title, IMPL_TSX, impl, [REQ_TSX] + */} +
+ ); diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 1a0e560..a51e2e0 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -36,6 +36,10 @@ "go": (CommentType.go, "go"), "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), + "typescript": (CommentType.ts, "ts"), + # `.tsx` runs the same TSX grammar; the distinct extension + JSX source + # exercises the superset-grammar decision end to end. + "tsx": (CommentType.ts, "tsx"), } From 0330c0b748e6dad4c8b077dfa681b0b34935dbef Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 7 Aug 2026 16:24:06 +0200 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20Trace=20TypeScript?= =?UTF-8?q?=20support=20(FE=5FTS=20feature=20and=20impl=20markers)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/components/features.rst | 26 +++++++++++++++++++ src/sphinx_codelinks/analyse/utils.py | 4 ++- .../source_discover/config.py | 1 + 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index f342bc4..9edea7a 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -266,6 +266,32 @@ Features .. fault:: Sphinx-codelinks hallucinates traceability objects in Bash :id: FAULT_BASH_2 +.. feature:: TypeScript Language Support + :id: FE_TS + + Support for defining traceability objects in TypeScript source files via one-line + comment annotations. + + The TypeScript language parser leverages tree-sitter to accurately identify and + extract comments from TypeScript sources, including single-line (``//``) and + multi-line (``/* */``) comment styles. All files are parsed with the TSX + grammar — a strict superset of the TypeScript grammar — so ``.tsx`` files + (including JSX comments such as ``{/* ... */}``) need no per-file grammar choice. + + Key capabilities: + + * Detection of inline and block comments + * Association of comments with function, class, and method declarations + * ``const``/``let``/``var`` declarations count as scopes only when they assign + a function or arrow function + * File extensions ``.ts`` and ``.tsx`` auto-discovered when ``comment_type = "ts"`` + + .. fault:: Traceability objects are not detected in TypeScript language + :id: FAULT_TS_1 + + .. fault:: Sphinx-codelinks hallucinates traceability objects in TypeScript + :id: FAULT_TS_2 + .. feature:: Preprocessor-Aware C/C++ Extraction :id: FE_PREPROC diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index fe4109a..6c5af64 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -28,6 +28,7 @@ # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP] CommentType.cpp: {"function_definition", "class_definition"}, CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"}, + # @TypeScript Scope Node Types, IMPL_TS_2, impl, [FE_TS] CommentType.ts: { "function_declaration", "class_declaration", @@ -73,6 +74,7 @@ """ CPP_QUERY = """(comment) @comment""" C_SHARP_QUERY = """(comment) @comment""" +# @TypeScript comment query for tree-sitter, IMPL_TS_3, impl, [FE_TS] TYPE_SCRIPT_QUERY = """(comment) @comment""" YAML_QUERY = """(comment) @comment""" RUST_QUERY = """ @@ -115,7 +117,7 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: return False -# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH] +# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH, FE_TS] def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: if comment_type == CommentType.cpp: import tree_sitter_cpp # noqa: PLC0415 diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 70fd864..ee4f382 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -27,6 +27,7 @@ class CommentType(str, Enum): python = "python" cpp = "cpp" cs = "cs" + # @Support TypeScript style comments, IMPL_TS_1, impl, [FE_TS]; ts = "ts" yaml = "yaml" # @Support Rust style comments, IMPL_RUST_1, impl, [FE_RUST]; From 2ceb5b4ea62b351e9101fa9d6662e2fe87c250c5 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 7 Aug 2026 16:51:19 +0200 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=94=A7=20MAINTAIN:=20Clarify=20tsx?= =?UTF-8?q?=20fixture=20comment=20and=20complete=20README=20lang=20list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/data/extraction/README.md | 2 +- tests/test_extraction_fixtures.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index 2dd95f1..7df7168 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -12,7 +12,7 @@ Each `*.yaml` file in this directory is a map of `case_name → case`: ```yaml default_oneliner_cpp: - lang: cpp # cpp | c | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx + lang: cpp # cpp | c | cpp_header | python | csharp | rust | yaml | go | jsonc | bash | typescript | tsx config: default # "default", or an inline config block (see below) source: | // @My Title, IMPL_1, impl, [REQ_1] diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index a51e2e0..881d753 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -37,8 +37,11 @@ "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), "typescript": (CommentType.ts, "ts"), - # `.tsx` runs the same TSX grammar; the distinct extension + JSX source - # exercises the superset-grammar decision end to end. + # `.tsx` is documentary: extraction never reads the file suffix, and the + # plain-TS and TSX grammars lex comments identically (the TSX-grammar + # choice is pinned by test_analyse_utils.py's has_error check instead). + # What this case pins: a marker on its own line inside a multi-line JSX + # block comment anchors to that line. "tsx": (CommentType.ts, "tsx"), } From 279c935577dd43d6700479a2a5a64292b029ea16 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 9 Aug 2026 20:21:40 +0200 Subject: [PATCH 10/11] =?UTF-8?q?=E2=9C=A8=20NEW:=20Widen=20ts=20comment?= =?UTF-8?q?=20type=20to=20full=20TS/JS=20family?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TSX grammar used for comment_type = "ts" is a strict superset of TypeScript, which is itself a superset of JavaScript, so it already parses .mts/.cts (TypeScript's own ESM/CJS variants) and the full .js/.jsx/.mjs/.cjs JavaScript family with no new grammar dependency. --- docs/source/components/analyse.rst | 2 +- docs/source/components/configuration.rst | 5 +++-- docs/source/components/features.rst | 16 +++++++++------- docs/source/development/change_log.rst | 6 ++++-- src/sphinx_codelinks/analyse/utils.py | 7 ++++--- src/sphinx_codelinks/source_discover/config.py | 7 ++++++- tests/data/discover_fixtures.json | 16 ++++++++++++++-- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 228163b..4a5dcec 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -47,7 +47,7 @@ Limitations **Current Limitations:** -- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript (``//``, ``/* */``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported +- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), TypeScript/JavaScript (``//``, ``/* */``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time Extraction Examples diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 8f81f3d..315d439 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -304,11 +304,12 @@ Specifies the comment syntax style used in the source code files. This determine ``/* */`` (multi-line), ``///`` (XML doc comments) - ``.cs`` - * - TypeScript + * - TypeScript / JavaScript - ``"ts"`` - ``//`` (single-line), ``/* */`` (multi-line) - - ``.ts``, ``.tsx`` + - ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, ``.mjs`` + and ``.cjs`` * - YAML - ``"yaml"`` - ``#`` (single-line) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index 9edea7a..021e12e 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -269,14 +269,15 @@ Features .. feature:: TypeScript Language Support :id: FE_TS - Support for defining traceability objects in TypeScript source files via one-line - comment annotations. + Support for defining traceability objects in TypeScript and JavaScript source + files via one-line comment annotations. The TypeScript language parser leverages tree-sitter to accurately identify and - extract comments from TypeScript sources, including single-line (``//``) and - multi-line (``/* */``) comment styles. All files are parsed with the TSX - grammar — a strict superset of the TypeScript grammar — so ``.tsx`` files - (including JSX comments such as ``{/* ... */}``) need no per-file grammar choice. + extract comments from TypeScript and JavaScript sources, including single-line + (``//``) and multi-line (``/* */``) comment styles. All files are parsed with + the TSX grammar — a strict superset of the TypeScript grammar, which is in turn + a superset of JavaScript — so ``.tsx`` files (including JSX comments such as + ``{/* ... */}``) and plain JavaScript sources need no per-file grammar choice. Key capabilities: @@ -284,7 +285,8 @@ Features * Association of comments with function, class, and method declarations * ``const``/``let``/``var`` declarations count as scopes only when they assign a function or arrow function - * File extensions ``.ts`` and ``.tsx`` auto-discovered when ``comment_type = "ts"`` + * File extensions ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, + ``.mjs`` and ``.cjs`` auto-discovered when ``comment_type = "ts"`` .. fault:: Traceability objects are not detected in TypeScript language :id: FAULT_TS_1 diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 4556b14..342d673 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -11,8 +11,10 @@ New and Improved - ✨ Added TypeScript comment type support for source discovery and analysis. - TypeScript files can now be processed using ``comment_type = "ts"``. - Source discovery supports both ``.ts`` and ``.tsx`` extensions by default. + TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``, + since the TSX grammar used to parse them is a superset of both languages. + Source discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, + ``.jsx``, ``.mjs`` and ``.cjs`` extensions by default. .. _`release:1.4.0`: diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 6c5af64..3bb4591 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -137,9 +137,10 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: elif comment_type == CommentType.ts: import tree_sitter_typescript # noqa: PLC0415 - # The TSX grammar is a strict superset of the TypeScript grammar (it also - # parses plain .ts fine), so use it for both to support .tsx files without - # needing a per-file grammar choice. + # The TSX grammar is a strict superset of the TypeScript grammar, which is + # itself a superset of JavaScript, so it also parses plain .ts and the + # whole JavaScript family fine. Use it for all of them to avoid needing a + # per-file grammar choice. parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index ee4f382..aa3a208 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -9,7 +9,12 @@ "cpp": ["c", "ci", "cpp", "cc", "cxx", "h", "hpp", "hxx", "hh", "ihl"], "python": ["py"], "cs": ["cs"], - "ts": ["ts", "tsx"], + # ".mts"/".cts" are TypeScript's own ESM/CJS module variants. ".js"/".jsx"/ + # ".mjs"/".cjs" are JavaScript, covered by the same comment type because the + # TSX grammar used to parse "ts" sources is a strict superset of the + # TypeScript grammar, which is itself a superset of JavaScript, so no + # separate grammar or comment_type value is needed. + "ts": ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"], "yaml": ["yml", "yaml"], "rust": ["rs"], "go": ["go"], diff --git a/tests/data/discover_fixtures.json b/tests/data/discover_fixtures.json index 5959cab..14516dd 100644 --- a/tests/data/discover_fixtures.json +++ b/tests/data/discover_fixtures.json @@ -399,11 +399,17 @@ }, { "name": "typescript_comment_type", - "description": "TypeScript comment type discovers .ts and .tsx files", + "description": "TypeScript comment type discovers the full TypeScript and JavaScript family", "git_init": false, "files": { "src/main.ts": "// main", "src/component.tsx": "// component", + "src/esm.mts": "// esm module", + "src/cjs.cts": "// cjs module", + "src/script.js": "// script", + "src/widget.jsx": "// widget", + "src/esm.mjs": "// esm js", + "src/legacy.cjs": "// legacy", "src/main.cpp": "// not ts", "src/util.py": "# not ts" }, @@ -415,8 +421,14 @@ "comment_type": "ts" }, "expected": [ + "src/cjs.cts", "src/component.tsx", - "src/main.ts" + "src/esm.mjs", + "src/esm.mts", + "src/legacy.cjs", + "src/main.ts", + "src/script.js", + "src/widget.jsx" ] } ] From 41fb7a9233ac70ac77003b8a673bad42700616a1 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Mon, 10 Aug 2026 14:26:22 +0200 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20Select=20TS/TSX=20g?= =?UTF-8?q?rammar=20per=20file=20suffix,=20not=20TSX=20for=20all?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TSX grammar is not a strict superset of TypeScript: a legacy angle-bracket type assertion (`x`), valid in `.ts`/`.mts`/`.cts`, parses as JSX under the TSX grammar and swallows the rest of the file into one `jsx_text` node, silently dropping every marker below it. Pick the tree-sitter grammar per file from its suffix instead: the plain TypeScript grammar for `.ts`/`.mts`/`.cts`, and the TSX grammar (safe for JavaScript and JSX) for everything else in the family. The parser/query pair is now built lazily per grammar in a small cache, so it is still built at most once per grammar actually used, not once per file. --- docs/source/components/features.rst | 12 +++-- docs/source/development/change_log.rst | 10 ++-- src/sphinx_codelinks/analyse/analyse.py | 23 ++++++++- src/sphinx_codelinks/analyse/utils.py | 49 ++++++++++++++++--- .../source_discover/config.py | 10 ++-- tests/test_analyse_utils.py | 39 ++++++++++++++- tests/test_extraction_fixtures.py | 13 +++-- 7 files changed, 130 insertions(+), 26 deletions(-) diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index 021e12e..ccfd31f 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -274,10 +274,14 @@ Features The TypeScript language parser leverages tree-sitter to accurately identify and extract comments from TypeScript and JavaScript sources, including single-line - (``//``) and multi-line (``/* */``) comment styles. All files are parsed with - the TSX grammar — a strict superset of the TypeScript grammar, which is in turn - a superset of JavaScript — so ``.tsx`` files (including JSX comments such as - ``{/* ... */}``) and plain JavaScript sources need no per-file grammar choice. + (``//``) and multi-line (``/* */``) comment styles. The grammar is chosen per + file from its extension: ``.ts``, ``.mts``, and ``.cts`` — TypeScript's own + module variants — are parsed with the plain TypeScript grammar, since a legacy + angle-bracket type assertion (``x``) is valid there but is JSX syntax + under the TSX grammar. Every other extension (``.tsx``, ``.jsx``, ``.js``, + ``.mjs``, ``.cjs``) is parsed with the TSX grammar, which is safe for plain + JavaScript and additionally handles JSX (including JSX comments such as + ``{/* ... */}``) embedded in ``.tsx`` or ``.js`` sources. Key capabilities: diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 342d673..b63bad0 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -11,10 +11,12 @@ New and Improved - ✨ Added TypeScript comment type support for source discovery and analysis. - TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``, - since the TSX grammar used to parse them is a superset of both languages. - Source discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, - ``.jsx``, ``.mjs`` and ``.cjs`` extensions by default. + TypeScript and JavaScript files can now be processed using ``comment_type = "ts"``. + The tree-sitter grammar is chosen per file from its extension: ``.ts``, ``.mts``, + and ``.cts`` use the plain TypeScript grammar, and everything else (``.tsx``, + ``.jsx``, ``.js``, ``.mjs``, ``.cjs``) falls back to the TSX grammar. Source + discovery supports ``.ts``, ``.tsx``, ``.mts``, ``.cts``, ``.js``, ``.jsx``, + ``.mjs`` and ``.cjs`` extensions by default. .. _`release:1.4.0`: diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index c2c4f7a..a26a7bf 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -5,6 +5,7 @@ from typing import Any, TypedDict, cast from tree_sitter import Node as TreeSitterNode +from tree_sitter import Parser, Query from sphinx_codelinks.analyse import utils from sphinx_codelinks.analyse.models import ( @@ -97,9 +98,29 @@ def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: yield src_path, text.encode("utf-8") def create_src_objects(self) -> None: - parser, query = utils.init_tree_sitter(self.analyse_config.comment_type) + comment_type = self.analyse_config.comment_type + # One (parser, query) pair per distinct grammar actually needed, built + # lazily so a parser is never rebuilt per file. Every comment type + # except TypeScript uses a single grammar for the whole run; + # TypeScript alone varies its grammar per file (utils.ts_grammar_key) + # because a legacy TypeScript-only cast parses as JSX under the wrong + # grammar — see the CommentType.ts branch of utils.init_tree_sitter. + parser_cache: dict[str, tuple[Parser, Query]] = {} for src_path, src_string in self.get_src_strings(): + # `comment_type` is normally a CommentType member, but a few call + # sites carry it as a plain (possibly invalid) str instead — see + # SourceAnalyseConfig.comment_type — so key on `str(comment_type)` + # rather than `.value`, which only the enum has. + cache_key = ( + utils.ts_grammar_key(src_path) + if comment_type == CommentType.ts + else str(comment_type) + ) + if cache_key not in parser_cache: + parser_cache[cache_key] = utils.init_tree_sitter(comment_type, src_path) + parser, query = parser_cache[cache_key] + comments: list[TreeSitterNode] | None = utils.extract_comments( src_string, parser, query ) diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 3bb4591..6ed904f 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -101,6 +101,25 @@ "null", } +# TypeScript's own module variants. Legacy angle-bracket type assertions +# (``x``) are valid syntax only here, not under the TSX grammar (see the +# CommentType.ts branch of init_tree_sitter for what goes wrong otherwise), so +# these three suffixes get the plain TypeScript grammar and everything else in +# the JS/TS family falls back to TSX. +TS_STRICT_GRAMMAR_SUFFIXES = {".ts", ".mts", ".cts"} + + +def ts_grammar_key(src_path: Path) -> str: + """Return which tree-sitter-typescript grammar ``src_path`` needs. + + ``"typescript"`` for TypeScript's own module variants (``.ts``, ``.mts``, + ``.cts``); ``"tsx"`` for the rest of the JavaScript/TypeScript family + (``.tsx``, ``.jsx``, ``.js``, ``.mjs``, ``.cjs``). Used both to pick the + grammar in ``init_tree_sitter`` and, by callers that parse many files, to + cache one parser per grammar instead of rebuilding one per file. + """ + return "typescript" if src_path.suffix in TS_STRICT_GRAMMAR_SUFFIXES else "tsx" + def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: """Return True if file is likely text, False if binary.""" @@ -118,7 +137,17 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: # @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH, FE_TS] -def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: +def init_tree_sitter( + comment_type: CommentType, src_path: Path | None = None +) -> tuple[Parser, Query]: + """Build the (parser, query) pair for ``comment_type``. + + ``src_path`` only matters for ``CommentType.ts``, whose grammar varies by + file suffix (see ``ts_grammar_key``); every other comment type ignores it + and uses a single grammar. When ``src_path`` is omitted the TSX grammar is + assumed, which is the safe default for the whole JS/TS family except + TypeScript's own module variants. + """ if comment_type == CommentType.cpp: import tree_sitter_cpp # noqa: PLC0415 @@ -137,11 +166,19 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: elif comment_type == CommentType.ts: import tree_sitter_typescript # noqa: PLC0415 - # The TSX grammar is a strict superset of the TypeScript grammar, which is - # itself a superset of JavaScript, so it also parses plain .ts and the - # whole JavaScript family fine. Use it for all of them to avoid needing a - # per-file grammar choice. - parsed_language = Language(tree_sitter_typescript.language_tsx()) + # Legacy angle-bracket type assertions (``x``) are valid TypeScript + # syntax in .ts/.mts/.cts, but the same text is JSX syntax under the TSX + # grammar: ``x`` parses as a jsx_opening_element and swallows the + # rest of the file into a single jsx_text node, silently dropping every + # marker after it. So the plain TypeScript grammar is required for those + # three suffixes. The TSX grammar remains the fallback for the rest of + # the JS/TS family (.tsx, .jsx, .js, .mjs, .cjs): JavaScript has no such + # cast syntax, so TSX is safe there, and it additionally handles JSX + # embedded in plain .js. + if src_path is not None and ts_grammar_key(src_path) == "typescript": + parsed_language = Language(tree_sitter_typescript.language_typescript()) + else: + parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, TYPE_SCRIPT_QUERY) elif comment_type == CommentType.yaml: import tree_sitter_yaml # noqa: PLC0415 diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index aa3a208..ca14515 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -10,10 +10,12 @@ "python": ["py"], "cs": ["cs"], # ".mts"/".cts" are TypeScript's own ESM/CJS module variants. ".js"/".jsx"/ - # ".mjs"/".cjs" are JavaScript, covered by the same comment type because the - # TSX grammar used to parse "ts" sources is a strict superset of the - # TypeScript grammar, which is itself a superset of JavaScript, so no - # separate grammar or comment_type value is needed. + # ".mjs"/".cjs" are JavaScript. All of these share the "ts" comment type: + # comment syntax is identical across the family, and the analyse stage + # picks the actual tree-sitter grammar per file from the suffix (the plain + # TypeScript grammar for ".ts"/".mts"/".cts", the TSX grammar for + # everything else — see utils.ts_grammar_key), so no separate + # comment_type value is needed here. "ts": ["ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs"], "yaml": ["yml", "yaml"], "rust": ["rs"], diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index 0a03192..1833acb 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -63,8 +63,11 @@ def init_rust_tree_sitter() -> tuple[Parser, Query]: @pytest.fixture(scope="session") def init_typescript_tree_sitter() -> tuple[Parser, Query]: - # TSX grammar is a superset of the TypeScript grammar (parses plain .ts too), - # matching what utils.init_tree_sitter uses for CommentType.ts. + # The TSX grammar, matching what utils.init_tree_sitter picks for + # CommentType.ts when the file isn't one of TypeScript's own module + # variants (.ts/.mts/.cts) — see utils.ts_grammar_key. Fine for the plain + # TS fixtures below too, since none of them use a legacy angle-bracket + # cast (the one construct where the two grammars disagree). parsed_language = Language(tree_sitter_typescript.language_tsx()) query = Query(parsed_language, utils.TYPE_SCRIPT_QUERY) parser = Parser(parsed_language) @@ -794,6 +797,38 @@ def test_find_associated_scope_typescript_jsx_no_parse_error( assert not tree.root_node.has_error +def test_typescript_ts_suffix_recovers_markers_around_angle_bracket_cast(): + """A ``.ts`` file must use the plain TypeScript grammar, not TSX. + + ``x`` is a legacy angle-bracket type assertion: valid TypeScript + syntax, but JSX syntax under the TSX grammar. There it parses as a + ``jsx_opening_element`` and swallows the rest of the file into a single + ``jsx_text`` node, silently dropping every marker after it (``has_error`` + is also set). ``init_tree_sitter`` must pick the plain TypeScript grammar + for a ``.ts`` path, via ``ts_grammar_key``, so markers both above and + below the cast all survive. + """ + code = b"""// @Top, IMPL_TOP +const v = x; +// @Bottom, IMPL_BOTTOM +const y = 2; +// @Third, IMPL_THIRD +""" + parser, query = utils.init_tree_sitter(CommentType.ts, Path("dummy.ts")) + tree = parser.parse(code) + assert not tree.root_node.has_error + + comments = utils.extract_comments(code, parser, query) + assert comments is not None + comments.sort(key=lambda node: node.start_point.row) + texts = [node.text.decode("utf-8") for node in comments if node.text] + assert texts == [ + "// @Top, IMPL_TOP", + "// @Bottom, IMPL_BOTTOM", + "// @Third, IMPL_THIRD", + ] + + @pytest.mark.parametrize( ("code", "result"), [ diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 881d753..1e4a806 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -37,11 +37,14 @@ "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), "typescript": (CommentType.ts, "ts"), - # `.tsx` is documentary: extraction never reads the file suffix, and the - # plain-TS and TSX grammars lex comments identically (the TSX-grammar - # choice is pinned by test_analyse_utils.py's has_error check instead). - # What this case pins: a marker on its own line inside a multi-line JSX - # block comment anchors to that line. + # `.tsx` matters here: extraction now picks the tree-sitter grammar per + # file from the suffix (utils.ts_grammar_key), and `.tsx` is one of the + # suffixes that gets the TSX grammar rather than the plain TypeScript one + # (the `.ts`/`.mts`/`.cts` suffixes get the latter — see + # utils.init_tree_sitter). What this case pins: a marker on its own line + # inside a multi-line JSX block comment anchors to that line, which + # requires the source (an arrow function returning JSX) to parse cleanly + # under the TSX grammar in the first place. "tsx": (CommentType.ts, "tsx"), }