Parser: Re-parse on unexpected ";" error for more accurate errors

javascript/packages/browser/test/browser.test.ts

@@ -34,7 +34,7 @@ describe("@herb-tools/browser", () => {
     const simpleHtml = '<div><%= "Hello World" %></div>'
     const ruby = Herb.extractRuby(simpleHtml)
     expect(ruby).toBeDefined()
-    expect(ruby).toBe('         "Hello World"         ')
+    expect(ruby).toBe('         "Hello World"  ;      ')
   })
 
   test("extractHTML() extracts HTML content", async () => {

javascript/packages/linter/test/rules/parser-no-errors.test.ts

@@ -57,7 +57,6 @@ describe("ParserNoErrorsRule", () => {
 
   test("should report Ruby parse errors in ERB tags", () => {
     expectError("expect_expression_after_operator: unexpected end-of-input; expected an expression after the operator (`RUBY_PARSE_ERROR`)")
-    expectError("unexpected_token_close_context: unexpected end-of-input, assuming it is closing the parent top level context (`RUBY_PARSE_ERROR`)")
     assertOffenses(`<%= 1 + %>`)
   })
 

javascript/packages/node-wasm/test/node-wasm.test.ts

@@ -34,7 +34,7 @@ describe("@herb-tools/node-wasm", () => {
     const simpleHtml = '<div><%= "Hello World" %></div>'
     const ruby = Herb.extractRuby(simpleHtml)
     expect(ruby).toBeDefined()
-    expect(ruby).toBe('         "Hello World"         ')
+    expect(ruby).toBe('         "Hello World"  ;      ')
   })
 
   test("extractHTML() extracts HTML content", async () => {

javascript/packages/node/binding.gyp

@@ -30,6 +30,7 @@
         "./extension/libherb/parser_helpers.c",
         "./extension/libherb/parser_match_tags.c",
         "./extension/libherb/parser.c",
+        "./extension/libherb/position.c",
         "./extension/libherb/pretty_print.c",
         "./extension/libherb/prism_helpers.c",
         "./extension/libherb/range.c",

javascript/packages/node/test/node.test.ts

@@ -34,7 +34,7 @@ describe("@herb-tools/node", () => {
     const simpleHtml = '<div><%= "Hello World" %></div>'
     const ruby = Herb.extractRuby(simpleHtml)
     expect(ruby).toBeDefined()
-    expect(ruby).toBe('         "Hello World"         ')
+    expect(ruby).toBe('         "Hello World"  ;      ')
   })
 
   test("extractHTML() extracts HTML content", async () => {

rust/tests/cli_commands_test.rs

@@ -12,7 +12,7 @@ fn test_version_functions() {
 fn test_extract_ruby() {
   let source = "<div><%= name %></div>";
   let ruby = extract_ruby(source).unwrap();
-  assert_eq!(ruby, "         name         ");
+  assert_eq!(ruby, "         name  ;      ");
 }
 
 #[test]
@@ -32,7 +32,7 @@ fn test_extract_ruby_complex() {
   let ruby = extract_ruby(source).unwrap();
   assert_eq!(
     ruby,
-    "     \n     users.each do |user|   \n           user.name       \n     end   \n      "
+    "     \n     users.each do |user|  ;\n           user.name  ;    \n     end  ;\n      "
   );
 }
 

src/analyze.c

@@ -1409,6 +1409,35 @@ void herb_analyze_parse_tree(AST_DOCUMENT_NODE_T* document, const char* source)
   free(invalid_context);
 }
 
+static void parse_erb_content_errors(AST_NODE_T* erb_node, const char* source) {
+  if (!erb_node || erb_node->type != AST_ERB_CONTENT_NODE) { return; }
+  AST_ERB_CONTENT_NODE_T* content_node = (AST_ERB_CONTENT_NODE_T*) erb_node;
+
+  if (!content_node->content || !content_node->content->value) { return; }
+
+  const char* content = content_node->content->value;
+  if (strlen(content) == 0) { return; }
+
+  pm_parser_t parser;
+  pm_options_t options = { 0, .partial_script = true };
+  pm_parser_init(&parser, (const uint8_t*) content, strlen(content), &options);
+
+  pm_node_t* root = pm_parse(&parser);
+
+  const pm_diagnostic_t* error = (const pm_diagnostic_t*) parser.error_list.head;
+
+  if (error != NULL) {
+    RUBY_PARSE_ERROR_T* parse_error =
+      ruby_parse_error_from_prism_error_with_positions(error, erb_node->location.start, erb_node->location.end);
+
+    hb_array_append(erb_node->errors, parse_error);
+  }
+
+  pm_node_destroy(&parser, root);
+  pm_parser_free(&parser);
+  pm_options_free(&options);
+}
+
 void herb_analyze_parse_errors(AST_DOCUMENT_NODE_T* document, const char* source) {
   char* extracted_ruby = herb_extract_ruby_with_semicolons(source);
 
@@ -1422,6 +1451,19 @@ void herb_analyze_parse_errors(AST_DOCUMENT_NODE_T* document, const char* source
 
   for (const pm_diagnostic_t* error = (const pm_diagnostic_t*) parser.error_list.head; error != NULL;
        error = (const pm_diagnostic_t*) error->node.next) {
+    size_t error_offset = (size_t) (error->location.start - parser.start);
+
+    if (strstr(error->message, "unexpected ';'") != NULL) {
+      if (error_offset < strlen(extracted_ruby) && extracted_ruby[error_offset] == ';') {
+        if (error_offset >= strlen(source) || source[error_offset] != ';') {
+          AST_NODE_T* erb_node = find_erb_content_at_offset(document, source, error_offset);
+
+          if (erb_node) { parse_erb_content_errors(erb_node, source); }
+
+          continue;
+        }
+      }
+    }
 
     RUBY_PARSE_ERROR_T* parse_error = ruby_parse_error_from_prism_error(error, (AST_NODE_T*) document, source, &parser);
     hb_array_append(document->base.errors, parse_error);

src/ast_node.c

@@ -1,8 +1,10 @@
 #include "include/ast_node.h"
 #include "include/ast_nodes.h"
 #include "include/errors.h"
+#include "include/position.h"
 #include "include/token.h"
 #include "include/util.h"
+#include "include/visitor.h"
 
 #include <prism.h>
 #include <stdio.h>
@@ -76,3 +78,30 @@ void ast_node_set_positions_from_token(AST_NODE_T* node, const token_T* token) {
 bool ast_node_is(const AST_NODE_T* node, const ast_node_type_T type) {
   return node->type == type;
 }
+
+typedef struct {
+  position_T position;
+  AST_NODE_T* found_node;
+} find_erb_at_position_context_T;
+
+static bool find_erb_at_position_visitor(const AST_NODE_T* node, void* data) {
+  find_erb_at_position_context_T* context = (find_erb_at_position_context_T*) data;
+
+  if (node->type == AST_ERB_CONTENT_NODE) {
+    if (position_is_within_range(context->position, node->location.start, node->location.end)) {
+      context->found_node = (AST_NODE_T*) node;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+AST_NODE_T* find_erb_content_at_offset(AST_DOCUMENT_NODE_T* document, const char* source, size_t offset) {
+  position_T position = position_from_source_with_offset(source, offset);
+  find_erb_at_position_context_T context = { .position = position, .found_node = NULL };
+
+  herb_visit_node((AST_NODE_T*) document, find_erb_at_position_visitor, &context);
+
+  return context.found_node;
+}

src/extract.c

@@ -7,7 +7,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-void herb_extract_ruby_to_buffer_with_semicolons(const char* source, hb_buffer_T* output) {
+void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
   hb_array_T* tokens = herb_lex(source);
   bool skip_erb_content = false;
   bool is_comment_tag = false;
@@ -76,75 +76,9 @@ void herb_extract_ruby_to_buffer_with_semicolons(const char* source, hb_buffer_T
           break;
         }
 
-        bool needs_semicolon = false;
-        uint32_t current_line = token->location.end.line;
-
-        for (size_t j = i + 1; j < hb_array_size(tokens); j++) {
-          const token_T* next_token = hb_array_get(tokens, j);
-
-          if (next_token->type == TOKEN_NEWLINE) { break; }
-
-          if (next_token->type == TOKEN_ERB_START && next_token->location.start.line == current_line) {
-            needs_semicolon = true;
-            break;
-          }
-        }
-
-        if (needs_semicolon) {
-          hb_buffer_append_char(output, ' ');
-          hb_buffer_append_char(output, ';');
-          hb_buffer_append_whitespace(output, range_length(token->range) - 2);
-        } else {
-          hb_buffer_append_whitespace(output, range_length(token->range));
-        }
-        break;
-      }
-
-      default: {
-        hb_buffer_append_whitespace(output, range_length(token->range));
-      }
-    }
-  }
-
-  herb_free_tokens(&tokens);
-}
-
-void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output) {
-  hb_array_T* tokens = herb_lex(source);
-  bool skip_erb_content = false;
-
-  for (size_t i = 0; i < hb_array_size(tokens); i++) {
-    const token_T* token = hb_array_get(tokens, i);
-
-    switch (token->type) {
-      case TOKEN_NEWLINE: {
-        hb_buffer_append(output, token->value);
-        break;
-      }
-
-      case TOKEN_ERB_START: {
-        if (strcmp(token->value, "<%#") == 0 || strcmp(token->value, "<%%") == 0 || strcmp(token->value, "<%%=") == 0) {
-          skip_erb_content = true;
-        }
-
-        hb_buffer_append_whitespace(output, range_length(token->range));
-        break;
-      }
-
-      case TOKEN_ERB_CONTENT: {
-        if (skip_erb_content == false) {
-          hb_buffer_append(output, token->value);
-        } else {
-          hb_buffer_append_whitespace(output, range_length(token->range));
-        }
-
-        break;
-      }
-
-      case TOKEN_ERB_END: {
-        skip_erb_content = false;
-
-        hb_buffer_append_whitespace(output, range_length(token->range));
+        hb_buffer_append_char(output, ' ');
+        hb_buffer_append_char(output, ';');
+        hb_buffer_append_whitespace(output, range_length(token->range) - 2);
         break;
       }
 
@@ -180,7 +114,7 @@ char* herb_extract_ruby_with_semicolons(const char* source) {
   hb_buffer_T output;
   hb_buffer_init(&output, strlen(source));
 
-  herb_extract_ruby_to_buffer_with_semicolons(source, &output);
+  herb_extract_ruby_to_buffer(source, &output);
 
   return output.value;
 }

src/include/ast_node.h

@@ -32,4 +32,6 @@ void ast_node_set_positions_from_token(AST_NODE_T* node, const token_T* token);
 
 bool ast_node_is(const AST_NODE_T* node, ast_node_type_T type);
 
+AST_NODE_T* find_erb_content_at_offset(AST_DOCUMENT_NODE_T* document, const char* source, size_t offset);
+
 #endif

src/include/extract.h

@@ -12,7 +12,6 @@ void herb_extract_ruby_to_buffer(const char* source, hb_buffer_T* output);
 void herb_extract_html_to_buffer(const char* source, hb_buffer_T* output);
 
 char* herb_extract_ruby_with_semicolons(const char* source);
-void herb_extract_ruby_to_buffer_with_semicolons(const char* source, hb_buffer_T* output);
 
 char* herb_extract(const char* source, herb_extract_language_T language);
 char* herb_extract_from_file(const char* path, herb_extract_language_T language);

src/include/position.h

@@ -1,11 +1,16 @@
 #ifndef HERB_POSITION_H
 #define HERB_POSITION_H
 
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 typedef struct POSITION_STRUCT {
   uint32_t line;
   uint32_t column;
 } position_T;
 
+position_T position_from_source_with_offset(const char* source, size_t offset);
+bool position_is_within_range(position_T position, position_T start, position_T end);
+
 #endif

src/include/prism_helpers.h

@@ -16,6 +16,10 @@ RUBY_PARSE_ERROR_T* ruby_parse_error_from_prism_error(
   pm_parser_t* parser
 );
 
-position_T position_from_source_with_offset(const char* source, size_t offset);
+RUBY_PARSE_ERROR_T* ruby_parse_error_from_prism_error_with_positions(
+  const pm_diagnostic_t* error,
+  position_T start,
+  position_T end
+);
 
 #endif

src/position.c

@@ -0,0 +1,27 @@
+#include "include/position.h"
+#include "include/util.h"
+
+position_T position_from_source_with_offset(const char* source, size_t offset) {
+  position_T position = { .line = 1, .column = 0 };
+
+  for (size_t i = 0; i < offset; i++) {
+    if (is_newline(source[i])) {
+      position.line++;
+      position.column = 0;
+    } else {
+      position.column++;
+    }
+  }
+
+  return position;
+}
+
+bool position_is_within_range(position_T position, position_T start, position_T end) {
+  if (position.line < start.line) { return false; }
+  if (position.line == start.line && position.column < start.column) { return false; }
+
+  if (position.line > end.line) { return false; }
+  if (position.line == end.line && position.column > end.column) { return false; }
+
+  return true;
+}