From 46c904a7ffb7489966374dbdcb56082bf949c6fb Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 1 Jul 2026 14:28:24 -0400 Subject: [PATCH] Fix '#' inside string literals being lexed as a comment The comment token runs to end of line, so at a position like the content of "#fff" it outmatched the string-content token on length and the string interior was consumed as a comment, corrupting the parse of everything after it (with a later string in the file the quotes re-pair across rules). Give the quoted_string and raw_string content tokens lexical precedence over the comment token, mirroring the precedence the two interpolated string forms already had. Comments after strings on the same line still lex normally, since the string-content tokens are not valid outside string interiors. Adds corpus tests for '#' in quoted and raw strings and for a real comment following a string that contains '#'. Fixes #21 --- grammar.js | 8 +++- src/grammar.json | 18 ++++++-- src/node-types.json | 2 +- src/parser.c | 100 ++++++++++++++++++++-------------------- test/corpus/strings.txt | 75 ++++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+), 58 deletions(-) create mode 100644 test/corpus/strings.txt diff --git a/grammar.js b/grammar.js index e2d06d6..9d72ab9 100644 --- a/grammar.js +++ b/grammar.js @@ -352,9 +352,13 @@ module.exports = grammar({ ), // quoted-string = '"' { CHAR } '"' + // String content needs lexical precedence over the comment token: + // without it, a '#' in the content lets the comment (which runs to end + // of line) win the longest-match rule, so `"#fff"` lexed as a comment + // and corrupted the rest of the parse. quoted_string: $ => seq( '"', - optional(alias(token.immediate(/([^\\"\n]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})+/), 'string_content')), + optional(alias(token.immediate(prec(1, /([^\\"\n]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})+/)), 'string_content')), '"', ), @@ -363,7 +367,7 @@ module.exports = grammar({ seq( '`', repeat( - token.immediate(/[^`]+/), + token.immediate(prec(1, /[^`]+/)), ), '`', ), diff --git a/src/grammar.json b/src/grammar.json index 6ba4cea..c6ab271 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -1271,8 +1271,12 @@ "content": { "type": "IMMEDIATE_TOKEN", "content": { - "type": "PATTERN", - "value": "([^\\\\\"\\n]|\\\\[\"\\\\/bfnrt]|\\\\u[0-9a-fA-F]{4})+" + "type": "PREC", + "value": 1, + "content": { + "type": "PATTERN", + "value": "([^\\\\\"\\n]|\\\\[\"\\\\/bfnrt]|\\\\u[0-9a-fA-F]{4})+" + } } }, "named": false, @@ -1301,8 +1305,12 @@ "content": { "type": "IMMEDIATE_TOKEN", "content": { - "type": "PATTERN", - "value": "[^`]+" + "type": "PREC", + "value": 1, + "content": { + "type": "PATTERN", + "value": "[^`]+" + } } } }, @@ -1829,4 +1837,4 @@ "inline": [], "supertypes": [], "reserved": {} -} +} \ No newline at end of file diff --git a/src/node-types.json b/src/node-types.json index 434a1b2..79c5719 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -1334,4 +1334,4 @@ "type": "}", "named": false } -] +] \ No newline at end of file diff --git a/src/parser.c b/src/parser.c index 31589f0..c97f12a 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1,4 +1,4 @@ -/* Automatically @generated by tree-sitter v0.25.10 */ +/* Automatically @generated by tree-sitter */ #include "tree_sitter/parser.h" @@ -2017,10 +2017,10 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 4: if (lookahead == '\n') SKIP(17); if (lookahead == '"') ADVANCE(130); - if (lookahead == '#') ADVANCE(131); + if (lookahead == '#') ADVANCE(133); if (lookahead == '\\') ADVANCE(31); - if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(132); - if (lookahead != 0) ADVANCE(133); + if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(131); + if (lookahead != 0) ADVANCE(132); END_STATE(); case 5: if (lookahead == '\n') ADVANCE(55); @@ -2351,14 +2351,14 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 31: ADVANCE_MAP( 'u', 40, - '"', 133, - '/', 133, - '\\', 133, - 'b', 133, - 'f', 133, - 'n', 133, - 'r', 133, - 't', 133, + '"', 132, + '/', 132, + '\\', 132, + 'b', 132, + 'f', 132, + 'n', 132, + 'r', 132, + 't', 132, ); END_STATE(); case 32: @@ -2372,22 +2372,22 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 34: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(133); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(33); END_STATE(); case 35: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(33); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(132); END_STATE(); case 36: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(34); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(35); END_STATE(); case 37: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(35); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(34); END_STATE(); case 38: if (('0' <= lookahead && lookahead <= '9') || @@ -3163,24 +3163,24 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 131: ACCEPT_TOKEN(aux_sym_quoted_string_token1); - if (lookahead == '"') ADVANCE(158); - if (lookahead == '\\') ADVANCE(153); + if (lookahead == '#') ADVANCE(133); + if (lookahead == '\\') ADVANCE(31); + if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(131); if (lookahead != 0 && - lookahead != '\n') ADVANCE(131); + (lookahead < '\t' || '\r' < lookahead) && + lookahead != '"' && + lookahead != '#') ADVANCE(132); END_STATE(); case 132: ACCEPT_TOKEN(aux_sym_quoted_string_token1); - if (lookahead == '#') ADVANCE(131); if (lookahead == '\\') ADVANCE(31); - if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(132); if (lookahead != 0 && - (lookahead < '\t' || '\r' < lookahead) && - lookahead != '"' && - lookahead != '#') ADVANCE(133); + lookahead != '\n' && + lookahead != '"') ADVANCE(132); END_STATE(); case 133: ACCEPT_TOKEN(aux_sym_quoted_string_token1); - if (lookahead == '\\') ADVANCE(31); + if (lookahead == '\\') ADVANCE(153); if (lookahead != 0 && lookahead != '\n' && lookahead != '"') ADVANCE(133); @@ -3191,8 +3191,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 135: ACCEPT_TOKEN(aux_sym_raw_string_token1); if (lookahead == '\n') ADVANCE(137); - if (lookahead == '`') ADVANCE(158); - if (lookahead != 0) ADVANCE(135); + if (lookahead != 0 && + lookahead != '`') ADVANCE(135); END_STATE(); case 136: ACCEPT_TOKEN(aux_sym_raw_string_token1); @@ -3285,14 +3285,14 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { ACCEPT_TOKEN(sym_comment); ADVANCE_MAP( 'u', 157, - '"', 131, - '/', 131, - '\\', 131, - 'b', 131, - 'f', 131, - 'n', 131, - 'r', 131, - 't', 131, + '"', 133, + '/', 133, + '\\', 133, + 'b', 133, + 'f', 133, + 'n', 133, + 'r', 133, + 't', 133, ); if (lookahead != 0 && lookahead != '\n') ADVANCE(158); @@ -3301,7 +3301,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { ACCEPT_TOKEN(sym_comment); if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(131); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(133); if (lookahead != 0 && lookahead != '\n') ADVANCE(158); END_STATE(); @@ -35951,13 +35951,13 @@ static const TSParseActionEntry ts_parse_actions[] = { [1271] = {.entry = {.count = 1, .reusable = true}}, SHIFT(187), [1273] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_fn_args_repeat1, 2, 0, 0), SHIFT_REPEAT(73), [1276] = {.entry = {.count = 1, .reusable = false}}, SHIFT(390), - [1278] = {.entry = {.count = 1, .reusable = false}}, SHIFT(776), + [1278] = {.entry = {.count = 1, .reusable = true}}, SHIFT(776), [1280] = {.entry = {.count = 1, .reusable = false}}, SHIFT(381), - [1282] = {.entry = {.count = 1, .reusable = false}}, SHIFT(779), + [1282] = {.entry = {.count = 1, .reusable = true}}, SHIFT(779), [1284] = {.entry = {.count = 1, .reusable = true}}, SHIFT(183), [1286] = {.entry = {.count = 1, .reusable = true}}, SHIFT(201), [1288] = {.entry = {.count = 1, .reusable = false}}, REDUCE(aux_sym_raw_string_repeat1, 2, 0, 0), - [1290] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_raw_string_repeat1, 2, 0, 0), SHIFT_REPEAT(779), + [1290] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_raw_string_repeat1, 2, 0, 0), SHIFT_REPEAT(779), [1293] = {.entry = {.count = 1, .reusable = false}}, SHIFT(149), [1295] = {.entry = {.count = 1, .reusable = true}}, SHIFT(447), [1297] = {.entry = {.count = 1, .reusable = false}}, SHIFT(480), @@ -35966,23 +35966,23 @@ static const TSParseActionEntry ts_parse_actions[] = { [1304] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_expr_every_repeat1, 2, 0, 0), [1306] = {.entry = {.count = 1, .reusable = true}}, SHIFT(257), [1308] = {.entry = {.count = 1, .reusable = false}}, SHIFT(557), - [1310] = {.entry = {.count = 1, .reusable = false}}, SHIFT(791), + [1310] = {.entry = {.count = 1, .reusable = true}}, SHIFT(791), [1312] = {.entry = {.count = 1, .reusable = false}}, SHIFT(442), - [1314] = {.entry = {.count = 1, .reusable = false}}, SHIFT(801), + [1314] = {.entry = {.count = 1, .reusable = true}}, SHIFT(801), [1316] = {.entry = {.count = 1, .reusable = false}}, SHIFT(515), [1318] = {.entry = {.count = 1, .reusable = true}}, SHIFT(875), [1320] = {.entry = {.count = 1, .reusable = true}}, SHIFT(85), [1322] = {.entry = {.count = 1, .reusable = true}}, SHIFT(102), [1324] = {.entry = {.count = 1, .reusable = false}}, SHIFT(558), [1326] = {.entry = {.count = 1, .reusable = false}}, SHIFT(508), - [1328] = {.entry = {.count = 1, .reusable = false}}, SHIFT(788), + [1328] = {.entry = {.count = 1, .reusable = true}}, SHIFT(788), [1330] = {.entry = {.count = 1, .reusable = true}}, SHIFT(271), [1332] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_object_repeat1, 2, 0, 0), SHIFT_REPEAT(273), [1335] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_object_repeat1, 2, 0, 0), [1337] = {.entry = {.count = 1, .reusable = true}}, SHIFT(80), [1339] = {.entry = {.count = 1, .reusable = true}}, SHIFT(185), [1341] = {.entry = {.count = 1, .reusable = false}}, SHIFT(463), - [1343] = {.entry = {.count = 1, .reusable = false}}, SHIFT(782), + [1343] = {.entry = {.count = 1, .reusable = true}}, SHIFT(782), [1345] = {.entry = {.count = 1, .reusable = true}}, SHIFT(192), [1347] = {.entry = {.count = 1, .reusable = true}}, SHIFT(95), [1349] = {.entry = {.count = 1, .reusable = true}}, SHIFT(96), @@ -35996,30 +35996,30 @@ static const TSParseActionEntry ts_parse_actions[] = { [1365] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_fn_args, 2, 0, 0), [1367] = {.entry = {.count = 1, .reusable = true}}, SHIFT(203), [1369] = {.entry = {.count = 1, .reusable = false}}, SHIFT(147), - [1371] = {.entry = {.count = 1, .reusable = false}}, SHIFT(780), + [1371] = {.entry = {.count = 1, .reusable = true}}, SHIFT(780), [1373] = {.entry = {.count = 1, .reusable = true}}, SHIFT(105), [1375] = {.entry = {.count = 1, .reusable = true}}, SHIFT(106), [1377] = {.entry = {.count = 1, .reusable = true}}, SHIFT(65), [1379] = {.entry = {.count = 1, .reusable = true}}, SHIFT(178), [1381] = {.entry = {.count = 1, .reusable = false}}, SHIFT(507), - [1383] = {.entry = {.count = 1, .reusable = false}}, SHIFT(889), + [1383] = {.entry = {.count = 1, .reusable = true}}, SHIFT(889), [1385] = {.entry = {.count = 1, .reusable = true}}, SHIFT(230), [1387] = {.entry = {.count = 1, .reusable = true}}, SHIFT(272), [1389] = {.entry = {.count = 1, .reusable = true}}, SHIFT(529), [1391] = {.entry = {.count = 1, .reusable = true}}, SHIFT(552), [1393] = {.entry = {.count = 1, .reusable = false}}, SHIFT(460), - [1395] = {.entry = {.count = 1, .reusable = false}}, SHIFT(886), + [1395] = {.entry = {.count = 1, .reusable = true}}, SHIFT(886), [1397] = {.entry = {.count = 1, .reusable = true}}, SHIFT(173), [1399] = {.entry = {.count = 1, .reusable = true}}, SHIFT(239), [1401] = {.entry = {.count = 1, .reusable = false}}, SHIFT(389), - [1403] = {.entry = {.count = 1, .reusable = false}}, SHIFT(909), + [1403] = {.entry = {.count = 1, .reusable = true}}, SHIFT(909), [1405] = {.entry = {.count = 1, .reusable = true}}, SHIFT(264), [1407] = {.entry = {.count = 1, .reusable = false}}, SHIFT(437), - [1409] = {.entry = {.count = 1, .reusable = false}}, SHIFT(913), + [1409] = {.entry = {.count = 1, .reusable = true}}, SHIFT(913), [1411] = {.entry = {.count = 1, .reusable = false}}, SHIFT(146), - [1413] = {.entry = {.count = 1, .reusable = false}}, SHIFT(915), + [1413] = {.entry = {.count = 1, .reusable = true}}, SHIFT(915), [1415] = {.entry = {.count = 1, .reusable = false}}, SHIFT(556), - [1417] = {.entry = {.count = 1, .reusable = false}}, SHIFT(918), + [1417] = {.entry = {.count = 1, .reusable = true}}, SHIFT(918), [1419] = {.entry = {.count = 1, .reusable = true}}, SHIFT(542), [1421] = {.entry = {.count = 1, .reusable = true}}, SHIFT(232), [1423] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_module, 3, 0, 0), diff --git a/test/corpus/strings.txt b/test/corpus/strings.txt new file mode 100644 index 0000000..08f4206 --- /dev/null +++ b/test/corpus/strings.txt @@ -0,0 +1,75 @@ +================================================================================ +hash inside quoted string +================================================================================ + +package test + +color := "#fff" + +-------------------------------------------------------------------------------- +(source_file + (module + (package) + (ref + (var)) + (policy + (rule + (rule_head + (var)) + (rule_body + (assignment) + (term + (scalar + (string + (quoted_string))))))))) + +================================================================================ +hash inside raw string +================================================================================ + +package test + +tag := `#hash` + +-------------------------------------------------------------------------------- +(source_file + (module + (package) + (ref + (var)) + (policy + (rule + (rule_head + (var)) + (rule_body + (assignment) + (term + (scalar + (string + (raw_string))))))))) + +================================================================================ +comment after string containing hash +================================================================================ + +package test + +x := "#a" # real comment + +-------------------------------------------------------------------------------- +(source_file + (module + (package) + (ref + (var)) + (policy + (rule + (rule_head + (var)) + (rule_body + (assignment) + (term + (scalar + (string + (quoted_string)))))))) + (comment))