From 1c0402c69591aaa004f2c4ac0cc076fef9e2c6de Mon Sep 17 00:00:00 2001 From: hartmannnico Date: Wed, 22 Jul 2026 07:00:58 +0200 Subject: [PATCH 1/2] feat(markdown): add CommentType.markdown backed by tree-sitter-markdown Add support for Markdown HTML-comment traceability markers: Changes: - source_discover/config.py: add 'markdown' to COMMENT_FILETYPE (.md/.markdown) and CommentType.markdown enum value - analyse/utils.py: add MARKDOWN_QUERY '(html_block) @comment', wire tree_sitter_markdown in init_tree_sitter(); no SCOPE_NODE_TYPES entry (markdown has no function/class scopes; oneline-only mode) - pyproject.toml: add tree-sitter-markdown>=0.5.1 dependency Note: callers must set end_sequence=' -->' (not the default newline) because the html_block node text includes the full '' delimiters. Closes #NNN. Template: PR-82 (TypeScript). --- pyproject.toml | 1 + src/sphinx_codelinks/analyse/utils.py | 16 +++++++++++++++- src/sphinx_codelinks/source_discover/config.py | 8 ++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3e0d54c..c94a1ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ dependencies = [ "tree-sitter-go>=0.23.0", "tree-sitter-json>=0.24.8", "tree-sitter-bash>=0.25.1", + "tree-sitter-markdown>=0.5.1", ] [project.optional-dependencies] diff --git a/src/sphinx_codelinks/analyse/utils.py b/src/sphinx_codelinks/analyse/utils.py index 48e726a..ed0e5d5 100644 --- a/src/sphinx_codelinks/analyse/utils.py +++ b/src/sphinx_codelinks/analyse/utils.py @@ -46,6 +46,11 @@ }, # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH] CommentType.bash: {"function_definition"}, + # Markdown has no function/class scopes relevant for code traceability. + # oneline markers in Markdown are always standalone html_block nodes; + # scope association (find_enclosing_scope / find_next_scope) is never + # invoked when get_oneline_needs=True and get_need_id_refs=False. + # CommentType.markdown is intentionally absent from this table. } logger = get_logger(__name__) @@ -78,6 +83,10 @@ JSONC_QUERY = """(comment) @comment""" # @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH] BASH_QUERY = """(comment) @comment""" +# @Markdown HTML-comment query for tree-sitter, IMPL_MD_3, impl, [FE_MARKDOWN] +# Captures block-level HTML nodes () as @comment. Inline HTML comments +# inside paragraphs are not captured — only standalone html_block elements. +MARKDOWN_QUERY = """(html_block) @comment""" # JSON value node types that can be associated with a comment. JSON_STRUCTURE_TYPES = { @@ -107,7 +116,7 @@ def is_text_file(filepath: Path, sample_size: int = 2048) -> bool: return False -# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH] +# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH, FE_MARKDOWN] def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: if comment_type == CommentType.cpp: import tree_sitter_cpp # noqa: PLC0415 @@ -149,6 +158,11 @@ def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]: parsed_language = Language(tree_sitter_bash.language()) query = Query(parsed_language, BASH_QUERY) + elif comment_type == CommentType.markdown: + import tree_sitter_markdown # noqa: PLC0415 + + parsed_language = Language(tree_sitter_markdown.language()) + query = Query(parsed_language, MARKDOWN_QUERY) else: raise ValueError(f"Unsupported comment style: {comment_type}") parser = Parser(parsed_language) diff --git a/src/sphinx_codelinks/source_discover/config.py b/src/sphinx_codelinks/source_discover/config.py index 0c6d295..2e1c6fd 100644 --- a/src/sphinx_codelinks/source_discover/config.py +++ b/src/sphinx_codelinks/source_discover/config.py @@ -19,6 +19,12 @@ # PyPI, so it cannot be wired in here. Track fish separately if a PyPI # grammar becomes available. "bash": ["sh", "bash", "zsh", "ksh"], + # Markdown uses block-level HTML comments `` as traceability + # markers. tree-sitter-markdown captures them as `html_block` nodes. + # NOTE: because the node text includes the `` delimiters, callers + # must set `end_sequence: " -->"` (not the default `"\n"`) in their + # oneline_comment_style config to prevent `-->` from leaking into parsed fields. + "markdown": ["md", "markdown"], } @@ -35,6 +41,8 @@ class CommentType(str, Enum): jsonc = "jsonc" # @Support Bash style comments, IMPL_BASH_1, impl, [FE_BASH]; bash = "bash" + # @Support Markdown HTML-comment style, IMPL_MD_1, impl, [FE_MARKDOWN] + markdown = "markdown" class SourceDiscoverSectionConfigType(TypedDict, total=False): From 09585c9ab271bfd42efe0ddaccaf07b6d642cd67 Mon Sep 17 00:00:00 2001 From: hartmannnico Date: Wed, 22 Jul 2026 23:02:53 +0200 Subject: [PATCH 2/2] test(markdown): add tests, fixtures, docs following PR #92 pattern - Fix test_source_discover + test_src_trace: add 'markdown' to expected comment_type validation error message list - Add FE_MARKDOWN feature need to docs/source/components/features.rst (fixes docs build warnings for IMPL_MD_1/IMPL_MD_3/IMPL_LANG_1) - Add default_oneliner_markdown fixture to tests/data/extraction/oneline.yaml with end_sequence: " -->" config - Generate snapshot for the markdown extraction fixture - Add init_markdown_tree_sitter fixture + test_extract_comments_markdown + test_init_tree_sitter_markdown to tests/test_analyse_utils.py - Add 'markdown' entry to LANG_MAP in tests/test_extraction_fixtures.py - Update docs: configuration.rst (supported values + table row), analyse.rst (language support list), change_log.rst (Unreleased entry) --- docs/source/components/analyse.rst | 2 +- docs/source/components/configuration.rst | 6 ++- docs/source/components/features.rst | 35 ++++++++++++++ docs/source/development/change_log.rst | 15 ++++++ ...re[oneline-default_oneliner_markdown].json | 19 ++++++++ tests/data/extraction/oneline.yaml | 7 +++ tests/test_analyse_utils.py | 48 +++++++++++++++++++ tests/test_extraction_fixtures.py | 1 + tests/test_source_discover.py | 3 +- tests/test_src_trace.py | 2 +- 10 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_markdown].json diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 0f411ed..8bf4837 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -47,7 +47,7 @@ Limitations **Current Limitations:** -- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``) and Bash (``#``) comment styles are supported +- **Language Support**: C/C++ (``//``, ``/* */``), C# (``//``, ``/* */``, ``///``), Python (``#``), YAML (``#``), Rust (``//``, ``/* */``, ``///``), Go (``//``, ``/* */``), JSONC (``//``, ``/* */``), Bash (``#``) and Markdown (````) comment styles are supported - **Single Comment Style**: Each analysis run processes only one comment style at a time Extraction Examples diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 2b2b10a..e243a54 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -271,7 +271,7 @@ Specifies the comment syntax style used in the source code files. This determine **Type:** ``str`` **Default:** ``"cpp"`` -**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"yaml"``, ``"rust"``, ``"go"``, ``"jsonc"``, ``"bash"`` +**Supported values:** ``"cpp"``, ``"python"``, ``"cs"``, ``"yaml"``, ``"rust"``, ``"go"``, ``"jsonc"``, ``"bash"``, ``"markdown"`` .. code-block:: toml @@ -330,6 +330,10 @@ Specifies the comment syntax style used in the source code files. This determine - ``"bash"`` - ``#`` (single-line) - ``.sh``, ``.bash``, ``.zsh``, ``.ksh`` + * - Markdown + - ``"markdown"`` + - ```` (HTML-comment block) + - ``.md``, ``.markdown`` .. note:: Future versions may support additional programming languages. diff --git a/docs/source/components/features.rst b/docs/source/components/features.rst index f342bc4..2e6683e 100644 --- a/docs/source/components/features.rst +++ b/docs/source/components/features.rst @@ -266,6 +266,41 @@ Features .. fault:: Sphinx-codelinks hallucinates traceability objects in Bash :id: FAULT_BASH_2 +.. feature:: Markdown Language Support + :id: FE_MARKDOWN + + Support for defining traceability objects in Markdown files using + HTML-comment markers (````). + + The Markdown language parser leverages tree-sitter to identify and extract + standalone HTML-comment blocks from Markdown documents, enabling + requirements traceability in agent definition files, skill files, and other + Markdown-based implementation artefacts. + + ``.md`` and ``.markdown`` files are auto-discovered when + ``comment_type = "markdown"``. Because tree-sitter-markdown exposes + standalone HTML comments as ``html_block`` nodes, only block-level + ```` markers are captured; inline HTML comments inside paragraphs + are not. + + Key capabilities: + + * HTML-comment (````) detection via tree-sitter + * Auto-discovery of ``.md`` and ``.markdown`` files + * Oneline-only mode (no scope association needed) + + .. note:: + + Because the captured node text includes the full ```` + delimiters, callers must set ``end_sequence: " -->"`` (not the default + ``"\n"``) in their ``oneline_comment_style`` config. + + .. fault:: Traceability objects are not detected in Markdown + :id: FAULT_MARKDOWN_1 + + .. fault:: Sphinx-codelinks hallucinates traceability objects in Markdown + :id: FAULT_MARKDOWN_2 + .. feature:: Preprocessor-Aware C/C++ Extraction :id: FE_PREPROC diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index 420cdd8..a368539 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -3,6 +3,21 @@ Changelog ========= +Unreleased +---------- + +New and Improved +................ + +- ✨ Added Markdown language support for the ``analyse`` module. + + Standalone HTML-comment blocks (````) in Markdown files are + now parsed for need ID references and one-line need definitions. ``.md`` and + ``.markdown`` files are discovered when ``comment_type = "markdown"``. + Because ``tree-sitter-markdown`` exposes HTML comments as ``html_block`` + nodes, callers must set ``end_sequence: " -->"`` in their + ``oneline_comment_style`` configuration. + .. _`release:1.4.0`: 1.4.0 diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_markdown].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_markdown].json new file mode 100644 index 0000000..333040a --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[oneline-default_oneliner_markdown].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_MD", + "title": "Md Title", + "type": "impl", + "links": { + "links": [ + "REQ_MD" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/oneline.yaml b/tests/data/extraction/oneline.yaml index 9a423ae..52baeed 100644 --- a/tests/data/extraction/oneline.yaml +++ b/tests/data/extraction/oneline.yaml @@ -57,3 +57,10 @@ shebang_oneliner_bash: #!/bin/bash # @Bash Title, IMPL_BASH_SHEBANG, impl, [REQ_BASH] function greet { echo hi; } + +default_oneliner_markdown: + lang: markdown + config: + end_sequence: " -->" + source: | + diff --git a/tests/test_analyse_utils.py b/tests/test_analyse_utils.py index bbf348f..f418da9 100644 --- a/tests/test_analyse_utils.py +++ b/tests/test_analyse_utils.py @@ -11,6 +11,7 @@ import tree_sitter_cpp import tree_sitter_go import tree_sitter_json +import tree_sitter_markdown import tree_sitter_python import tree_sitter_rust import tree_sitter_yaml @@ -84,6 +85,53 @@ def init_bash_tree_sitter() -> tuple[Parser, Query]: return parser, query +@pytest.fixture(scope="session") +def init_markdown_tree_sitter() -> tuple[Parser, Query]: + parsed_language = Language(tree_sitter_markdown.language()) + query = Query(parsed_language, utils.MARKDOWN_QUERY) + parser = Parser(parsed_language) + return parser, query + + +@pytest.mark.parametrize( + ("code", "expected_count"), + [ + # standalone block HTML comment is captured + ( + b"\n", + 1, + ), + # multiple HTML comment blocks are each captured + ( + b"\n\nSome paragraph.\n\n\n", + 2, + ), + # paragraph text without HTML comment produces no comments + ( + b"# Heading\n\nJust a paragraph with no markers.\n", + 0, + ), + ], +) +def test_extract_comments_markdown(code, expected_count, init_markdown_tree_sitter): + parser, query = init_markdown_tree_sitter + comments = utils.extract_comments(code, parser, query) or [] + assert len(comments) == expected_count + + +@pytest.mark.parametrize( + "code", + [ + b"\n", + ], +) +def test_init_tree_sitter_markdown(code): + """init_tree_sitter returns a working parser/query pair for markdown.""" + parser, query = utils.init_tree_sitter(CommentType.markdown) + comments = utils.extract_comments(code, parser, query) + assert len(comments) == 1 + + @pytest.mark.parametrize( ("code", "result"), [ diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 1a0e560..5190335 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -36,6 +36,7 @@ "go": (CommentType.go, "go"), "jsonc": (CommentType.jsonc, "jsonc"), "bash": (CommentType.bash, "sh"), + "markdown": (CommentType.markdown, "md"), } diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 4e8ec8a..c0e7062 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -49,7 +49,7 @@ "comment_type": "java", }, [ - "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'python', 'rust', 'yaml']" + "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'markdown', 'python', 'rust', 'yaml']" ], ), ( @@ -183,6 +183,7 @@ def create_source_files(tmp_path: Path) -> Path: ("cpp", len(COMMENT_FILETYPE["cpp"])), ("python", len(COMMENT_FILETYPE["python"])), ("bash", len(COMMENT_FILETYPE["bash"])), + ("markdown", len(COMMENT_FILETYPE["markdown"])), ], ) def test_comment_filetype( diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 7389e81..5e377d2 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -59,7 +59,7 @@ [ "Project 'dcdc' has the following errors:", "Schema validation error in field 'exclude': 123 is not of type 'string'", - "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'python', 'rust', 'yaml']", + "Schema validation error in field 'comment_type': 'java' is not one of ['bash', 'cpp', 'cs', 'go', 'jsonc', 'markdown', 'python', 'rust', 'yaml']", "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", "Schema validation error in field 'include': 345 is not of type 'string'", "Schema validation error in field 'src_dir': ['../dcdc'] is not of type 'string'",