diff --git a/grammar.js b/grammar.js index e2d06d6..9d72ab9 100644 --- a/grammar.js +++ b/grammar.js @@ -352,9 +352,13 @@ module.exports = grammar({ ), // quoted-string = '"' { CHAR } '"' + // String content needs lexical precedence over the comment token: + // without it, a '#' in the content lets the comment (which runs to end + // of line) win the longest-match rule, so `"#fff"` lexed as a comment + // and corrupted the rest of the parse. quoted_string: $ => seq( '"', - optional(alias(token.immediate(/([^\\"\n]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})+/), 'string_content')), + optional(alias(token.immediate(prec(1, /([^\\"\n]|\\["\\/bfnrt]|\\u[0-9a-fA-F]{4})+/)), 'string_content')), '"', ), @@ -363,7 +367,7 @@ module.exports = grammar({ seq( '`', repeat( - token.immediate(/[^`]+/), + token.immediate(prec(1, /[^`]+/)), ), '`', ), diff --git a/src/grammar.json b/src/grammar.json index 6ba4cea..c6ab271 100644 --- a/src/grammar.json +++ b/src/grammar.json @@ -1271,8 +1271,12 @@ "content": { "type": "IMMEDIATE_TOKEN", "content": { - "type": "PATTERN", - "value": "([^\\\\\"\\n]|\\\\[\"\\\\/bfnrt]|\\\\u[0-9a-fA-F]{4})+" + "type": "PREC", + "value": 1, + "content": { + "type": "PATTERN", + "value": "([^\\\\\"\\n]|\\\\[\"\\\\/bfnrt]|\\\\u[0-9a-fA-F]{4})+" + } } }, "named": false, @@ -1301,8 +1305,12 @@ "content": { "type": "IMMEDIATE_TOKEN", "content": { - "type": "PATTERN", - "value": "[^`]+" + "type": "PREC", + "value": 1, + "content": { + "type": "PATTERN", + "value": "[^`]+" + } } } }, @@ -1829,4 +1837,4 @@ "inline": [], "supertypes": [], "reserved": {} -} +} \ No newline at end of file diff --git a/src/node-types.json b/src/node-types.json index 434a1b2..79c5719 100644 --- a/src/node-types.json +++ b/src/node-types.json @@ -1334,4 +1334,4 @@ "type": "}", "named": false } -] +] \ No newline at end of file diff --git a/src/parser.c b/src/parser.c index 31589f0..c97f12a 100644 --- a/src/parser.c +++ b/src/parser.c @@ -1,4 +1,4 @@ -/* Automatically @generated by tree-sitter v0.25.10 */ +/* Automatically @generated by tree-sitter */ #include "tree_sitter/parser.h" @@ -2017,10 +2017,10 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 4: if (lookahead == '\n') SKIP(17); if (lookahead == '"') ADVANCE(130); - if (lookahead == '#') ADVANCE(131); + if (lookahead == '#') ADVANCE(133); if (lookahead == '\\') ADVANCE(31); - if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(132); - if (lookahead != 0) ADVANCE(133); + if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(131); + if (lookahead != 0) ADVANCE(132); END_STATE(); case 5: if (lookahead == '\n') ADVANCE(55); @@ -2351,14 +2351,14 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 31: ADVANCE_MAP( 'u', 40, - '"', 133, - '/', 133, - '\\', 133, - 'b', 133, - 'f', 133, - 'n', 133, - 'r', 133, - 't', 133, + '"', 132, + '/', 132, + '\\', 132, + 'b', 132, + 'f', 132, + 'n', 132, + 'r', 132, + 't', 132, ); END_STATE(); case 32: @@ -2372,22 +2372,22 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 34: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(133); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(33); END_STATE(); case 35: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(33); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(132); END_STATE(); case 36: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(34); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(35); END_STATE(); case 37: if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(35); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(34); END_STATE(); case 38: if (('0' <= lookahead && lookahead <= '9') || @@ -3163,24 +3163,24 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { END_STATE(); case 131: ACCEPT_TOKEN(aux_sym_quoted_string_token1); - if (lookahead == '"') ADVANCE(158); - if (lookahead == '\\') ADVANCE(153); + if (lookahead == '#') ADVANCE(133); + if (lookahead == '\\') ADVANCE(31); + if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(131); if (lookahead != 0 && - lookahead != '\n') ADVANCE(131); + (lookahead < '\t' || '\r' < lookahead) && + lookahead != '"' && + lookahead != '#') ADVANCE(132); END_STATE(); case 132: ACCEPT_TOKEN(aux_sym_quoted_string_token1); - if (lookahead == '#') ADVANCE(131); if (lookahead == '\\') ADVANCE(31); - if (set_contains(extras_character_set_3, 10, lookahead)) ADVANCE(132); if (lookahead != 0 && - (lookahead < '\t' || '\r' < lookahead) && - lookahead != '"' && - lookahead != '#') ADVANCE(133); + lookahead != '\n' && + lookahead != '"') ADVANCE(132); END_STATE(); case 133: ACCEPT_TOKEN(aux_sym_quoted_string_token1); - if (lookahead == '\\') ADVANCE(31); + if (lookahead == '\\') ADVANCE(153); if (lookahead != 0 && lookahead != '\n' && lookahead != '"') ADVANCE(133); @@ -3191,8 +3191,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { case 135: ACCEPT_TOKEN(aux_sym_raw_string_token1); if (lookahead == '\n') ADVANCE(137); - if (lookahead == '`') ADVANCE(158); - if (lookahead != 0) ADVANCE(135); + if (lookahead != 0 && + lookahead != '`') ADVANCE(135); END_STATE(); case 136: ACCEPT_TOKEN(aux_sym_raw_string_token1); @@ -3285,14 +3285,14 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { ACCEPT_TOKEN(sym_comment); ADVANCE_MAP( 'u', 157, - '"', 131, - '/', 131, - '\\', 131, - 'b', 131, - 'f', 131, - 'n', 131, - 'r', 131, - 't', 131, + '"', 133, + '/', 133, + '\\', 133, + 'b', 133, + 'f', 133, + 'n', 133, + 'r', 133, + 't', 133, ); if (lookahead != 0 && lookahead != '\n') ADVANCE(158); @@ -3301,7 +3301,7 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) { ACCEPT_TOKEN(sym_comment); if (('0' <= lookahead && lookahead <= '9') || ('A' <= lookahead && lookahead <= 'F') || - ('a' <= lookahead && lookahead <= 'f')) ADVANCE(131); + ('a' <= lookahead && lookahead <= 'f')) ADVANCE(133); if (lookahead != 0 && lookahead != '\n') ADVANCE(158); END_STATE(); @@ -35951,13 +35951,13 @@ static const TSParseActionEntry ts_parse_actions[] = { [1271] = {.entry = {.count = 1, .reusable = true}}, SHIFT(187), [1273] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_fn_args_repeat1, 2, 0, 0), SHIFT_REPEAT(73), [1276] = {.entry = {.count = 1, .reusable = false}}, SHIFT(390), - [1278] = {.entry = {.count = 1, .reusable = false}}, SHIFT(776), + [1278] = {.entry = {.count = 1, .reusable = true}}, SHIFT(776), [1280] = {.entry = {.count = 1, .reusable = false}}, SHIFT(381), - [1282] = {.entry = {.count = 1, .reusable = false}}, SHIFT(779), + [1282] = {.entry = {.count = 1, .reusable = true}}, SHIFT(779), [1284] = {.entry = {.count = 1, .reusable = true}}, SHIFT(183), [1286] = {.entry = {.count = 1, .reusable = true}}, SHIFT(201), [1288] = {.entry = {.count = 1, .reusable = false}}, REDUCE(aux_sym_raw_string_repeat1, 2, 0, 0), - [1290] = {.entry = {.count = 2, .reusable = false}}, REDUCE(aux_sym_raw_string_repeat1, 2, 0, 0), SHIFT_REPEAT(779), + [1290] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_raw_string_repeat1, 2, 0, 0), SHIFT_REPEAT(779), [1293] = {.entry = {.count = 1, .reusable = false}}, SHIFT(149), [1295] = {.entry = {.count = 1, .reusable = true}}, SHIFT(447), [1297] = {.entry = {.count = 1, .reusable = false}}, SHIFT(480), @@ -35966,23 +35966,23 @@ static const TSParseActionEntry ts_parse_actions[] = { [1304] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_expr_every_repeat1, 2, 0, 0), [1306] = {.entry = {.count = 1, .reusable = true}}, SHIFT(257), [1308] = {.entry = {.count = 1, .reusable = false}}, SHIFT(557), - [1310] = {.entry = {.count = 1, .reusable = false}}, SHIFT(791), + [1310] = {.entry = {.count = 1, .reusable = true}}, SHIFT(791), [1312] = {.entry = {.count = 1, .reusable = false}}, SHIFT(442), - [1314] = {.entry = {.count = 1, .reusable = false}}, SHIFT(801), + [1314] = {.entry = {.count = 1, .reusable = true}}, SHIFT(801), [1316] = {.entry = {.count = 1, .reusable = false}}, SHIFT(515), [1318] = {.entry = {.count = 1, .reusable = true}}, SHIFT(875), [1320] = {.entry = {.count = 1, .reusable = true}}, SHIFT(85), [1322] = {.entry = {.count = 1, .reusable = true}}, SHIFT(102), [1324] = {.entry = {.count = 1, .reusable = false}}, SHIFT(558), [1326] = {.entry = {.count = 1, .reusable = false}}, SHIFT(508), - [1328] = {.entry = {.count = 1, .reusable = false}}, SHIFT(788), + [1328] = {.entry = {.count = 1, .reusable = true}}, SHIFT(788), [1330] = {.entry = {.count = 1, .reusable = true}}, SHIFT(271), [1332] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_object_repeat1, 2, 0, 0), SHIFT_REPEAT(273), [1335] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_object_repeat1, 2, 0, 0), [1337] = {.entry = {.count = 1, .reusable = true}}, SHIFT(80), [1339] = {.entry = {.count = 1, .reusable = true}}, SHIFT(185), [1341] = {.entry = {.count = 1, .reusable = false}}, SHIFT(463), - [1343] = {.entry = {.count = 1, .reusable = false}}, SHIFT(782), + [1343] = {.entry = {.count = 1, .reusable = true}}, SHIFT(782), [1345] = {.entry = {.count = 1, .reusable = true}}, SHIFT(192), [1347] = {.entry = {.count = 1, .reusable = true}}, SHIFT(95), [1349] = {.entry = {.count = 1, .reusable = true}}, SHIFT(96), @@ -35996,30 +35996,30 @@ static const TSParseActionEntry ts_parse_actions[] = { [1365] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_fn_args, 2, 0, 0), [1367] = {.entry = {.count = 1, .reusable = true}}, SHIFT(203), [1369] = {.entry = {.count = 1, .reusable = false}}, SHIFT(147), - [1371] = {.entry = {.count = 1, .reusable = false}}, SHIFT(780), + [1371] = {.entry = {.count = 1, .reusable = true}}, SHIFT(780), [1373] = {.entry = {.count = 1, .reusable = true}}, SHIFT(105), [1375] = {.entry = {.count = 1, .reusable = true}}, SHIFT(106), [1377] = {.entry = {.count = 1, .reusable = true}}, SHIFT(65), [1379] = {.entry = {.count = 1, .reusable = true}}, SHIFT(178), [1381] = {.entry = {.count = 1, .reusable = false}}, SHIFT(507), - [1383] = {.entry = {.count = 1, .reusable = false}}, SHIFT(889), + [1383] = {.entry = {.count = 1, .reusable = true}}, SHIFT(889), [1385] = {.entry = {.count = 1, .reusable = true}}, SHIFT(230), [1387] = {.entry = {.count = 1, .reusable = true}}, SHIFT(272), [1389] = {.entry = {.count = 1, .reusable = true}}, SHIFT(529), [1391] = {.entry = {.count = 1, .reusable = true}}, SHIFT(552), [1393] = {.entry = {.count = 1, .reusable = false}}, SHIFT(460), - [1395] = {.entry = {.count = 1, .reusable = false}}, SHIFT(886), + [1395] = {.entry = {.count = 1, .reusable = true}}, SHIFT(886), [1397] = {.entry = {.count = 1, .reusable = true}}, SHIFT(173), [1399] = {.entry = {.count = 1, .reusable = true}}, SHIFT(239), [1401] = {.entry = {.count = 1, .reusable = false}}, SHIFT(389), - [1403] = {.entry = {.count = 1, .reusable = false}}, SHIFT(909), + [1403] = {.entry = {.count = 1, .reusable = true}}, SHIFT(909), [1405] = {.entry = {.count = 1, .reusable = true}}, SHIFT(264), [1407] = {.entry = {.count = 1, .reusable = false}}, SHIFT(437), - [1409] = {.entry = {.count = 1, .reusable = false}}, SHIFT(913), + [1409] = {.entry = {.count = 1, .reusable = true}}, SHIFT(913), [1411] = {.entry = {.count = 1, .reusable = false}}, SHIFT(146), - [1413] = {.entry = {.count = 1, .reusable = false}}, SHIFT(915), + [1413] = {.entry = {.count = 1, .reusable = true}}, SHIFT(915), [1415] = {.entry = {.count = 1, .reusable = false}}, SHIFT(556), - [1417] = {.entry = {.count = 1, .reusable = false}}, SHIFT(918), + [1417] = {.entry = {.count = 1, .reusable = true}}, SHIFT(918), [1419] = {.entry = {.count = 1, .reusable = true}}, SHIFT(542), [1421] = {.entry = {.count = 1, .reusable = true}}, SHIFT(232), [1423] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_module, 3, 0, 0), diff --git a/test/corpus/strings.txt b/test/corpus/strings.txt new file mode 100644 index 0000000..08f4206 --- /dev/null +++ b/test/corpus/strings.txt @@ -0,0 +1,75 @@ +================================================================================ +hash inside quoted string +================================================================================ + +package test + +color := "#fff" + +-------------------------------------------------------------------------------- +(source_file + (module + (package) + (ref + (var)) + (policy + (rule + (rule_head + (var)) + (rule_body + (assignment) + (term + (scalar + (string + (quoted_string))))))))) + +================================================================================ +hash inside raw string +================================================================================ + +package test + +tag := `#hash` + +-------------------------------------------------------------------------------- +(source_file + (module + (package) + (ref + (var)) + (policy + (rule + (rule_head + (var)) + (rule_body + (assignment) + (term + (scalar + (string + (raw_string))))))))) + +================================================================================ +comment after string containing hash +================================================================================ + +package test + +x := "#a" # real comment + +-------------------------------------------------------------------------------- +(source_file + (module + (package) + (ref + (var)) + (policy + (rule + (rule_head + (var)) + (rule_body + (assignment) + (term + (scalar + (string + (quoted_string)))))))) + (comment))