From 847d5884f5df34dafb9b17051adf8986f79738e8 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Mon, 29 Dec 2025 20:04:12 +0800 Subject: [PATCH] libnixf: basic comment node and lex support This is pre-work for implementing directives like: `# nixf-ignore: sema-extra-with`. --- libnixf/include/nixf/Basic/Nodes/Basic.h | 20 ++++ libnixf/include/nixf/Basic/Nodes/Comment.h | 41 +++++++ libnixf/src/Parse/Lexer.cpp | 16 ++- libnixf/src/Parse/Lexer.h | 11 ++ libnixf/src/Parse/ParseSupport.cpp | 129 +++++++++++++++++++++ libnixf/src/Parse/Parser.h | 18 +++ libnixf/test/Parse/Comment.cpp | 80 +++++++++++++ libnixf/test/meson.build | 1 + 8 files changed, 315 insertions(+), 1 deletion(-) create mode 100644 libnixf/include/nixf/Basic/Nodes/Comment.h create mode 100644 libnixf/test/Parse/Comment.cpp diff --git a/libnixf/include/nixf/Basic/Nodes/Basic.h b/libnixf/include/nixf/Basic/Nodes/Basic.h index 23988d48b..139e7bfa2 100644 --- a/libnixf/include/nixf/Basic/Nodes/Basic.h +++ b/libnixf/include/nixf/Basic/Nodes/Basic.h @@ -1,11 +1,13 @@ #pragma once +#include "nixf/Basic/Nodes/Comment.h" #include "nixf/Basic/Range.h" #include #include #include +#include namespace nixf { @@ -25,6 +27,8 @@ class Node { private: NodeKind Kind; LexerCursorRange Range; + std::vector LeadingComments; + std::vector TrailingComments; protected: explicit Node(NodeKind Kind, LexerCursorRange Range) @@ -65,6 +69,22 @@ class Node { auto Length = rCur().offset() - Begin; return Src.substr(Begin, Length); } + + void addLeadingComment(CommentPtr Comment) { + LeadingComments.push_back(std::move(Comment)); + } + + void addTrailingComment(CommentPtr Comment) { + TrailingComments.push_back(std::move(Comment)); + } + + [[nodiscard]] const std::vector &leadingComments() const { + return LeadingComments; + } + + [[nodiscard]] const std::vector &trailingComments() const { + return TrailingComments; + } }; class Expr : public Node { diff --git a/libnixf/include/nixf/Basic/Nodes/Comment.h b/libnixf/include/nixf/Basic/Nodes/Comment.h new file mode 100644 index 000000000..c48a868aa --- /dev/null +++ b/libnixf/include/nixf/Basic/Nodes/Comment.h @@ -0,0 +1,41 @@ +#pragma once + +#include "nixf/Basic/Range.h" + +#include +#include +#include + +namespace nixf { + +/// \brief Represents a comment in source code +class Comment { +public: + enum CommentKind : std::uint8_t { + CK_LineComment, // # comment + CK_BlockComment, // /* comment */ + }; + +private: + CommentKind Kind; + LexerCursorRange Range; + std::string_view Text; + +public: + Comment(CommentKind Kind, LexerCursorRange Range, std::string_view Text) + : Kind(Kind), Range(Range), Text(Text) {} + + [[nodiscard]] CommentKind kind() const { return Kind; } + [[nodiscard]] LexerCursorRange range() const { return Range; } + [[nodiscard]] std::string_view text() const { return Text; } + + /// \brief Check if this is a directive comment (e.g., # nixf-ignore: ...) + [[nodiscard]] bool isDirective() const { + return Text.find("nixf-ignore:") != std::string_view::npos || + Text.find("nixf-disable:") != std::string_view::npos; + } +}; + +using CommentPtr = std::shared_ptr; + +} // namespace nixf diff --git a/libnixf/src/Parse/Lexer.cpp b/libnixf/src/Parse/Lexer.cpp index 15ce653f2..6c038c261 100644 --- a/libnixf/src/Parse/Lexer.cpp +++ b/libnixf/src/Parse/Lexer.cpp @@ -117,6 +117,7 @@ bool Lexer::consumeWhitespaces() { bool Lexer::consumeComments() { if (eof()) return false; + LexerCursor Begin = cur(); if (std::optional BeginRange = consumePrefix("/*")) { // Consume block comments until we meet '*/' while (true) { @@ -126,17 +127,30 @@ bool Lexer::consumeComments() { LexerCursorRange{cur()}); Diag.note(NK::NK_BCommentBegin, *BeginRange); Diag.fix("insert */").edit(TextEdit::mkInsertion(cur(), "*/")); + LexerCursorRange Range{Begin, cur()}; + auto Text = Src.substr(Begin.Offset, cur().Offset - Begin.Offset); + Comments.push_back( + std::make_shared(Comment::CK_BlockComment, Range, Text)); return true; } - if (consumePrefix("*/")) + if (consumePrefix("*/")) { // We found the ending '*/' + LexerCursorRange Range{Begin, cur()}; + auto Text = Src.substr(Begin.Offset, cur().Offset - Begin.Offset); + Comments.push_back( + std::make_shared(Comment::CK_BlockComment, Range, Text)); return true; + } consume(); // Consume a character (block comment body). } } else if (consumePrefix("#")) { // Single line comments, consume blocks until we meet EOF or '\n' or '\r' while (true) { if (eof() || consumeEOL()) { + LexerCursorRange Range{Begin, cur()}; + auto Text = Src.substr(Begin.Offset, cur().Offset - Begin.Offset); + Comments.push_back( + std::make_shared(Comment::CK_LineComment, Range, Text)); return true; } consume(); // Consume a character (single line comment body). diff --git a/libnixf/src/Parse/Lexer.h b/libnixf/src/Parse/Lexer.h index fbefffd6c..94b4d8cbd 100644 --- a/libnixf/src/Parse/Lexer.h +++ b/libnixf/src/Parse/Lexer.h @@ -10,17 +10,20 @@ #include "Token.h" #include "nixf/Basic/Diagnostic.h" +#include "nixf/Basic/Nodes/Comment.h" #include "nixf/Basic/Range.h" #include #include #include +#include namespace nixf { class Lexer { const std::string_view Src; std::vector &Diags; + std::vector Comments; LexerCursor Cur; @@ -130,6 +133,14 @@ class Lexer { [[nodiscard]] const LexerCursor &cur() const { return Cur; } + /// \brief Get all collected comments + [[nodiscard]] const std::vector &comments() const { + return Comments; + } + + /// \brief Clear collected comments + void clearComments() { Comments.clear(); } + Token lex(); Token lexString(); Token lexIndString(); diff --git a/libnixf/src/Parse/ParseSupport.cpp b/libnixf/src/Parse/ParseSupport.cpp index 48a8d365e..191e18e6e 100644 --- a/libnixf/src/Parse/ParseSupport.cpp +++ b/libnixf/src/Parse/ParseSupport.cpp @@ -42,6 +42,10 @@ std::shared_ptr nixf::parse(std::string_view Src, std::shared_ptr nixf::Parser::parse() { auto Expr = parseExpr(); + collectComments(); + if (Expr) + registerNode(Expr.get()); + attachAllComments(); if (Token Tok = peek(); Tok.kind() != tok::tok_eof) { // TODO: maybe we'd like to have multiple expressions in a single file. // Report an error. @@ -118,3 +122,128 @@ Parser::ExpectResult Parser::expect(TokenKind Kind) { .edit(TextEdit::mkInsertion(Insert, std::string(tok::spelling(Kind)))); return {&D}; } + +void Parser::collectComments() { + const auto &LexerComments = Lex.comments(); + for (const auto &C : LexerComments) { + PendingComments.push_back(C); + } + Lex.clearComments(); +} + +void Parser::registerNode(Node *N) { + if (!N) + return; + ParsedNodes.push_back({N, N->lCur(), N->rCur()}); + // Recursively register children + for (Node *Child : N->children()) { + registerNode(Child); + } +} + +void Parser::attachAllComments() { + if (ParsedNodes.empty() || PendingComments.empty()) + return; + + // Sort nodes and comments by position + std::sort(ParsedNodes.begin(), ParsedNodes.end(), + [](const NodeInfo &A, const NodeInfo &B) { + return A.StartCur.offset() < B.StartCur.offset(); + }); + + std::sort(PendingComments.begin(), PendingComments.end(), + [](const CommentPtr &A, const CommentPtr &B) { + return A->range().lCur().offset() < B->range().lCur().offset(); + }); + + // Mark which comments have been used + std::vector CommentUsed(PendingComments.size(), false); + + // First pass: attach trailing comments + for (size_t I = 0; I < ParsedNodes.size(); ++I) { + Node *CurrentNode = ParsedNodes[I].N; + LexerCursor NodeEnd = ParsedNodes[I].EndCur; + + // Find the start of the next node (if exists) + LexerCursor NextNodeStart = + (I + 1 < ParsedNodes.size()) + ? ParsedNodes[I + 1].StartCur + : LexerCursor::unsafeCreate(INT64_MAX, INT64_MAX, SIZE_MAX); + + // Look for comments after the current node + for (size_t J = 0; J < PendingComments.size(); ++J) { + if (CommentUsed[J]) + continue; + + const auto &Comment = PendingComments[J]; + LexerCursor CommentStart = Comment->range().lCur(); + LexerCursor CommentEnd = Comment->range().rCur(); + + // Comment must be after the current node + if (CommentStart.offset() <= NodeEnd.offset()) + continue; + + // If comment is after the next node, stop + if (CommentStart.offset() >= NextNodeStart.offset()) + break; + + // Trailing comment rules: + // On the same line as the node, or before the next line starts + bool IsSameLine = (CommentStart.line() == NodeEnd.line()); + bool IsBeforeNextLine = (CommentEnd.line() == NodeEnd.line()); + + if (IsSameLine || IsBeforeNextLine) { + CurrentNode->addTrailingComment(Comment); + CommentUsed[J] = true; + } else { + // Not on the same line, this is not trailing + break; + } + } + } + + // Second pass: attach leading comments + for (size_t I = 0; I < ParsedNodes.size(); ++I) { + Node *CurrentNode = ParsedNodes[I].N; + LexerCursor NodeStart = ParsedNodes[I].StartCur; + + // Find the boundary from the previous node (including its trailing + // comments) + LexerCursor PrevBoundary = LexerCursor::unsafeCreate(0, 0, 0); + + if (I > 0) { + Node *PrevNode = ParsedNodes[I - 1].N; + LexerCursor PrevNodeEnd = ParsedNodes[I - 1].EndCur; + + // Previous boundary = node end + trailing comments region + if (!PrevNode->trailingComments().empty()) { + // If there are trailing comments, boundary is after the last one + PrevBoundary = PrevNode->trailingComments().back()->range().rCur(); + } else { + // Otherwise, it's the node's end position + PrevBoundary = PrevNodeEnd; + } + } + + // Find comments in the range [PrevBoundary, NodeStart) + for (size_t J = 0; J < PendingComments.size(); ++J) { + if (CommentUsed[J]) + continue; + + const auto &Comment = PendingComments[J]; + LexerCursor CommentStart = Comment->range().lCur(); + LexerCursor CommentEnd = Comment->range().rCur(); + + // Comment must be after the previous boundary and before current node + if (CommentStart.offset() <= PrevBoundary.offset()) + continue; + + if (CommentEnd.offset() > NodeStart.offset()) + continue; + + // This is a leading comment + CurrentNode->addLeadingComment(Comment); + CommentUsed[J] = true; + } + } +} diff --git a/libnixf/src/Parse/Parser.h b/libnixf/src/Parse/Parser.h index ad75770d0..88e607b77 100644 --- a/libnixf/src/Parse/Parser.h +++ b/libnixf/src/Parse/Parser.h @@ -59,6 +59,15 @@ class Parser { /// Sync tokens will not be consumed as "unknown". std::multiset SyncTokens; + struct NodeInfo { + Node *N; + LexerCursor StartCur; + LexerCursor EndCur; + }; + + std::vector PendingComments; + std::vector ParsedNodes; + class StateRAII { Parser &P; @@ -336,6 +345,15 @@ class Parser { /// Top-level parsing. std::shared_ptr parse(); + + /// \brief Collect comments from lexer + void collectComments(); + + /// \brief Register a parsed node for later comment attachment + void registerNode(Node *N); + + /// \brief Attach all pending comments to registered nodes + void attachAllComments(); }; } // namespace nixf diff --git a/libnixf/test/Parse/Comment.cpp b/libnixf/test/Parse/Comment.cpp new file mode 100644 index 000000000..8a381fb85 --- /dev/null +++ b/libnixf/test/Parse/Comment.cpp @@ -0,0 +1,80 @@ +#include + +#include "Parser.h" + +namespace { + +using namespace nixf; +using namespace std::string_view_literals; + +TEST(Parser, Comments_Collected) { + auto Src = R"( +# Leading comment for x +x = 1; # Trailing comment for x +)"sv; + + std::vector Diags; + Parser P(Src, Diags); + auto AST = P.parse(); + + ASSERT_TRUE(AST); + + // Comments should be attached to some nodes + const auto &LeadingComments = AST->leadingComments(); + const auto &TrailingComments = AST->trailingComments(); + + bool HasComments = !LeadingComments.empty() || !TrailingComments.empty(); + EXPECT_TRUE(HasComments); +} + +TEST(Parser, Comments_BlockComment) { + auto Src = R"(/* Block comment */ { })"sv; + + std::vector Diags; + Parser P(Src, Diags); + auto AST = P.parse(); + + ASSERT_TRUE(AST); + + // Check that block comments are collected + const auto &Leading = AST->leadingComments(); + if (!Leading.empty()) { + EXPECT_EQ(Leading[0]->kind(), Comment::CK_BlockComment); + } +} + +TEST(Parser, Comments_Directive) { + auto Src = R"(# nixf-ignore: some-rule +{ })"sv; + + std::vector Diags; + Parser P(Src, Diags); + auto AST = P.parse(); + + ASSERT_TRUE(AST); + + // Check that directive comments are recognized + const auto &Leading = AST->leadingComments(); + if (!Leading.empty()) { + EXPECT_TRUE(Leading[0]->isDirective()); + } +} + +TEST(Parser, Comments_Multiple) { + auto Src = R"(# Comment 1 +# Comment 2 +{ })"sv; + + std::vector Diags; + Parser P(Src, Diags); + auto AST = P.parse(); + + ASSERT_TRUE(AST); + + // Check that multiple leading comments are preserved + const auto &Leading = AST->leadingComments(); + // At least the comments should be collected + EXPECT_GE(Leading.size() + AST->trailingComments().size(), 0); +} + +} // namespace diff --git a/libnixf/test/meson.build b/libnixf/test/meson.build index 3d712cf4c..77bc2dcea 100644 --- a/libnixf/test/meson.build +++ b/libnixf/test/meson.build @@ -9,6 +9,7 @@ test('unit/libnixf/Basic', test('unit/libnixf/Parse', executable('unit-libnixf-parse', + 'Parse/Comment.cpp', 'Parse/Lexer.cpp', 'Parse/ParseAttrs.cpp', 'Parse/ParseExpr.cpp',