Engine: Add Support for ERB trimming

lib/herb/engine.rb

@@ -87,11 +87,9 @@ def initialize(input, properties = {})
       end
 
       @src << "__herb = ::Herb::Engine; " if @escape && @escapefunc == "__herb.h"
-
       @src << preamble
-      @src << "\n" unless preamble.end_with?("\n")
 
-      parse_result = ::Herb.parse(input)
+      parse_result = ::Herb.parse(input, track_whitespace: true)
       ast = parse_result.value
       parser_errors = parse_result.errors
 
@@ -190,6 +188,8 @@ def add_code(code)
       if code.include?("=begin") || code.include?("=end")
         @src << "\n" << code << "\n"
       else
+        @src.chomp! if @src.end_with?("\n") && code.start_with?(" ") && !code.end_with?("\n")
+
         @src << " " << code
 
         # TODO: rework and check for Prism::InlineComment as soon as we expose the Prism Nodes in the Herb AST

lib/herb/engine/compiler.rb

@@ -86,7 +86,7 @@ def visit_html_open_tag_node(node)
       end
 
       def visit_html_attribute_node(node)
-        add_text(" ")
+        add_whitespace(" ")
 
         visit(node.name)
 
@@ -134,7 +134,7 @@ def visit_literal_node(node)
       end
 
       def visit_whitespace_node(node)
-        add_text(node.value.value) if node.value
+        add_whitespace(node.value.value)
       end
 
       def visit_html_comment_node(node)
@@ -168,7 +168,7 @@ def visit_erb_content_node(node)
       end
 
       def visit_erb_control_node(node, &_block)
-        add_code(node.content.value.strip)
+        apply_trim(node, node.content.value.strip)
 
         yield if block_given?
       end
@@ -325,17 +325,15 @@ def process_erb_tag(node, skip_comment_check: false)
         if erb_output?(opening)
           process_erb_output(opening, code)
         else
-          add_code(code)
+          apply_trim(node, code)
         end
-
-        handle_whitespace_trimming(node)
       end
 
       def add_text(text)
         return if text.empty?
 
         if @trim_next_whitespace
-          text = text.lstrip
+          text = text.sub(/\A[ \t]*\r?\n/, "")
           @trim_next_whitespace = false
         end
 
@@ -344,6 +342,10 @@ def add_text(text)
         @tokens << [:text, text, current_context]
       end
 
+      def add_whitespace(whitespace)
+        @tokens << [:whitespace, whitespace, current_context]
+      end
+
       def add_code(code)
         @tokens << [:code, code, current_context]
       end
@@ -359,11 +361,13 @@ def add_expression_escaped(code)
       def optimize_tokens(tokens)
         return tokens if tokens.empty?
 
+        compacted = compact_whitespace_tokens(tokens)
+
         optimized = [] #: Array[untyped]
         current_text = ""
         current_context = nil
 
-        tokens.each do |type, value, context|
+        compacted.each do |type, value, context|
           if type == :text
             current_text += value
             current_context ||= context
@@ -384,6 +388,56 @@ def optimize_tokens(tokens)
         optimized
       end
 
+      def compact_whitespace_tokens(tokens)
+        return tokens if tokens.empty?
+
+        tokens.map.with_index { |token, index|
+          next token unless token[0] == :whitespace
+
+          next nil if adjacent_whitespace?(tokens, index)
+          next nil if whitespace_before_code_sequence?(tokens, index)
+
+          [:text, token[1], token[2]]
+        }.compact
+      end
+
+      def adjacent_whitespace?(tokens, index)
+        prev_token = index.positive? ? tokens[index - 1] : nil
+        next_token = index < tokens.length - 1 ? tokens[index + 1] : nil
+
+        trailing_whitespace?(prev_token) || leading_whitespace?(next_token)
+      end
+
+      def trailing_whitespace?(token)
+        return false unless token
+
+        token[0] == :whitespace || (token[0] == :text && token[1] =~ /\s\z/)
+      end
+
+      def leading_whitespace?(token)
+        token && token[0] == :text && token[1] =~ /\A\s/
+      end
+
+      def whitespace_before_code_sequence?(tokens, current_index)
+        previous_token = tokens[current_index - 1] if current_index.positive?
+
+        return false unless previous_token && previous_token[0] == :code
+
+        token_before_code = find_token_before_code_sequence(tokens, current_index)
+
+        return false unless token_before_code
+
+        trailing_whitespace?(token_before_code)
+      end
+
+      def find_token_before_code_sequence(tokens, whitespace_index)
+        search_index = whitespace_index - 1
+
+        search_index -= 1 while search_index >= 0 && tokens[search_index][0] == :code
+
+        search_index >= 0 ? tokens[search_index] : nil
+      end
+
       def process_erb_output(opening, code)
         should_escape = should_escape_output?(opening)
         add_expression_with_escaping(code, should_escape)
@@ -402,8 +456,69 @@ def add_expression_with_escaping(code, should_escape)
         end
       end
 
-      def handle_whitespace_trimming(node)
-        @trim_next_whitespace = true if node.tag_closing&.value == "-%>"
+      def at_line_start?
+        @tokens.empty? ||
+          @tokens.last[0] != :text ||
+          @tokens.last[1].empty? ||
+          @tokens.last[1].end_with?("\n") ||
+          @tokens.last[1] =~ /\A[ \t]+\z/ ||
+          @tokens.last[1] =~ /\n[ \t]+\z/
+      end
+
+      def extract_lspace
+        return "" unless @tokens.last && @tokens.last[0] == :text
+
+        text = @tokens.last[1]
+
+        return Regexp.last_match(1) if text =~ /\n([ \t]+)\z/ || text =~ /\A([ \t]+)\z/
+
+        ""
+      end
+
+      def extract_and_remove_lspace!
+        lspace = extract_lspace
+        return lspace if lspace.empty?
+
+        text = @tokens.last[1]
+        if text =~ /\n[ \t]+\z/
+          text.sub!(/[ \t]+\z/, "")
+        elsif text =~ /\A[ \t]+\z/
+          text.replace("")
+        end
+        @tokens.last[1] = text
+
+        lspace
+      end
+
+      def apply_trim(node, code)
+        has_left_trim = node.tag_opening.value.start_with?("<%-")
+        node.tag_closing&.value
+
+        remove_trailing_whitespace_from_last_token! if has_left_trim
+
+        if at_line_start?
+          lspace = extract_and_remove_lspace!
+          rspace = " \n"
+
+          @tokens << [:code, "#{lspace}#{code}#{rspace}", current_context]
+          @trim_next_whitespace = true
+        else
+          @tokens << [:code, code, current_context]
+        end
+      end
+
+      def remove_trailing_whitespace_from_last_token!
+        return unless @tokens.last && @tokens.last[0] == :text
+
+        text = @tokens.last[1]
+
+        if text =~ /\n[ \t]+\z/
+          text.sub!(/[ \t]+\z/, "")
+          @tokens.last[1] = text
+        elsif text =~ /\A[ \t]+\z/
+          text.replace("")
+          @tokens.last[1] = text
+        end
       end
     end
   end

lib/herb/engine/debug_visitor.rb

@@ -180,9 +180,7 @@ def add_debug_attributes_to_element(open_tag_node)
           debug_attributes << create_debug_attribute("data-herb-debug-attach-to-parent", "true")
         end
 
-        debug_attributes.each do |attr|
-          open_tag_node.children << attr
-        end
+        open_tag_node.children.concat(debug_attributes)
 
         @debug_attributes_applied = true
       end
@@ -233,9 +231,7 @@ def create_debug_span_for_erb(erb_node)
         ]
 
         debug_attributes << create_debug_attribute("data-herb-debug-line", line.to_s) if line
-
         debug_attributes << create_debug_attribute("data-herb-debug-column", (column + 1).to_s) if column
-
         debug_attributes << create_debug_attribute("style", "display: contents;")
 
         tag_name_token = create_token(:tag_name, "span")

test/engine/cli_test.rb

@@ -274,9 +274,9 @@ def with_temp_file(content)
 
         output = captured_output
 
-        assert_includes output, "'<ul>\n  '.freeze"
-        assert_includes output, "'\n    <li>'.freeze"
-        assert_includes output, "'\n</ul>\n'.freeze"
+        assert_includes output, "'<ul>\n'.freeze"
+        assert_includes output, "'    <li>'.freeze"
+        assert_includes output, "'</ul>\n'.freeze"
       end
     end
 

test/engine/rails_compatibility_test.rb

@@ -153,7 +153,7 @@ def flush_newline_if_pending(src)
 
       engine = RailsHerb.new(template, escape: true)
 
-      assert_equal "\n @output_buffer.safe_append='<h1>'.freeze; @output_buffer.append=(@title); @output_buffer.safe_append='</h1>'.freeze;\n@output_buffer", engine.src
+      assert_equal " @output_buffer.safe_append='<h1>'.freeze; @output_buffer.append=(@title); @output_buffer.safe_append='</h1>'.freeze;\n@output_buffer", engine.src
     end
 
     test "drop-in replacement compatibility" do