marcoroth · marcoroth · Mar 6, 2026 · Oct 17, 2025 · Oct 17, 2025 · Oct 17, 2025
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@ docs/docs/linter/rules/
 # Herb output
 /herb
 /run_herb_tests
+/bench_allocs
 
 # Herb Ruby extension
 /ext/herb/extconf.h

diff --git a/Makefile b/Makefile
@@ -20,6 +20,9 @@ test_sources = $(wildcard test/**/*.c)
 test_objects = $(test_sources:.c=.o)
 non_main_objects = $(filter-out src/main.o, $(objects))
 
+bench_allocs_exec = bench_allocs
+bench_allocs_source = bench/bench_allocs.c
+
 soext ?= $(shell ruby -e 'puts RbConfig::CONFIG["DLEXT"]')
 lib_name = $(build_dir)/lib$(exec).$(soext)
 static_lib_name = $(build_dir)/lib$(exec).a
@@ -102,9 +105,14 @@ test/%.o: test/%.c templates prism
 test: $(test_objects) $(non_main_objects)
 	$(cc) $(test_objects) $(non_main_objects) $(test_cflags) $(test_ldflags) -o $(test_exec)
 
+.PHONY: bench_allocs
+bench_allocs: $(non_main_objects)
+	$(cc) $(bench_allocs_source) $(non_main_objects) $(flags) $(prism_ldflags) -o $(bench_allocs_exec)
+	./$(bench_allocs_exec)
+
 .PHONY: clean
 clean:
-	rm -f $(exec) $(test_exec) $(lib_name) $(shared_lib_name) $(ruby_extension)
+	rm -f $(exec) $(test_exec) $(bench_allocs_exec) $(lib_name) $(shared_lib_name) $(ruby_extension)
 	rm -rf $(objects) $(test_objects) $(extension_objects) lib/herb/*.bundle tmp
 	rm -rf $(prism_path)
 	rake prism:clean

diff --git a/bench/bench_allocs.c b/bench/bench_allocs.c
@@ -0,0 +1,178 @@
+#include "../src/include/herb.h"
+#include "../src/include/util/hb_allocator.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static const char* SMALL_INPUT = "<div class=\"hello\"><%= foo %></div>";
+
+static const char* MEDIUM_INPUT =
+  "<!DOCTYPE html>\n"
+  "<html lang=\"en\">\n"
+  "<head>\n"
+  "  <meta charset=\"UTF-8\">\n"
+  "  <title><%= @title %></title>\n"
+  "</head>\n"
+  "<body>\n"
+  "  <div id=\"app\" class=\"container\">\n"
+  "    <h1><%= @heading %></h1>\n"
+  "    <% @items.each do |item| %>\n"
+  "      <div class=\"item\" data-id=\"<%= item.id %>\">\n"
+  "        <span class=\"name\"><%= item.name %></span>\n"
+  "        <p><%= item.description %></p>\n"
+  "        <% if item.active? %>\n"
+  "          <span class=\"badge\">Active</span>\n"
+  "        <% else %>\n"
+  "          <span class=\"badge inactive\">Inactive</span>\n"
+  "        <% end %>\n"
+  "      </div>\n"
+  "    <% end %>\n"
+  "  </div>\n"
+  "  <%# This is a comment %>\n"
+  "  <%= render partial: 'footer', locals: { year: 2024 } %>\n"
+  "</body>\n"
+  "</html>\n";
+
+static const char* LARGE_INPUT =
+  "<!DOCTYPE html>\n"
+  "<html lang=\"<%= I18n.locale %>\">\n"
+  "<head>\n"
+  "  <meta charset=\"UTF-8\">\n"
+  "  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">\n"
+  "  <title><%= @page_title || 'Default' %></title>\n"
+  "  <%= csrf_meta_tags %>\n"
+  "  <%= csp_meta_tag %>\n"
+  "  <%= stylesheet_link_tag 'application' %>\n"
+  "  <%= javascript_include_tag 'application' %>\n"
+  "</head>\n"
+  "<body class=\"<%= body_class %>\">\n"
+  "  <nav class=\"navbar\">\n"
+  "    <div class=\"nav-brand\">\n"
+  "      <%= link_to root_path do %>\n"
+  "        <img src=\"<%= asset_path('logo.png') %>\" alt=\"Logo\">\n"
+  "      <% end %>\n"
+  "    </div>\n"
+  "    <ul class=\"nav-links\">\n"
+  "      <% @nav_items.each do |nav| %>\n"
+  "        <li class=\"<%= 'active' if current_page?(nav.path) %>\">\n"
+  "          <%= link_to nav.label, nav.path, class: 'nav-link' %>\n"
+  "        </li>\n"
+  "      <% end %>\n"
+  "    </ul>\n"
+  "    <div class=\"nav-user\">\n"
+  "      <% if current_user %>\n"
+  "        <span><%= current_user.name %></span>\n"
+  "        <%= link_to 'Logout', logout_path, method: :delete %>\n"
+  "      <% else %>\n"
+  "        <%= link_to 'Login', login_path %>\n"
+  "      <% end %>\n"
+  "    </div>\n"
+  "  </nav>\n"
+  "  <main id=\"content\">\n"
+  "    <% if flash[:notice] %>\n"
+  "      <div class=\"alert alert-info\"><%= flash[:notice] %></div>\n"
+  "    <% end %>\n"
+  "    <% if flash[:alert] %>\n"
+  "      <div class=\"alert alert-danger\"><%= flash[:alert] %></div>\n"
+  "    <% end %>\n"
+  "    <div class=\"container\">\n"
+  "      <h1><%= @heading %></h1>\n"
+  "      <table class=\"table\">\n"
+  "        <thead>\n"
+  "          <tr>\n"
+  "            <th>Name</th>\n"
+  "            <th>Email</th>\n"
+  "            <th>Role</th>\n"
+  "            <th>Status</th>\n"
+  "            <th>Actions</th>\n"
+  "          </tr>\n"
+  "        </thead>\n"
+  "        <tbody>\n"
+  "          <% @users.each do |user| %>\n"
+  "            <tr id=\"user-<%= user.id %>\" class=\"<%= cycle('odd', 'even') %>\">\n"
+  "              <td><%= user.name %></td>\n"
+  "              <td><%= mail_to user.email %></td>\n"
+  "              <td><%= user.role.titleize %></td>\n"
+  "              <td>\n"
+  "                <% if user.active? %>\n"
+  "                  <span class=\"badge badge-success\">Active</span>\n"
+  "                <% elsif user.suspended? %>\n"
+  "                  <span class=\"badge badge-warning\">Suspended</span>\n"
+  "                <% else %>\n"
+  "                  <span class=\"badge badge-danger\">Inactive</span>\n"
+  "                <% end %>\n"
+  "              </td>\n"
+  "              <td>\n"
+  "                <%= link_to 'View', user_path(user), class: 'btn btn-sm' %>\n"
+  "                <%= link_to 'Edit', edit_user_path(user), class: 'btn btn-sm' %>\n"
+  "                <% if can?(:destroy, user) %>\n"
+  "                  <%= link_to 'Delete', user_path(user), method: :delete, class: 'btn btn-sm btn-danger', data: { confirm: 'Are you sure?' } %>\n"
+  "                <% end %>\n"
+  "              </td>\n"
+  "            </tr>\n"
+  "          <% end %>\n"
+  "        </tbody>\n"
+  "      </table>\n"
+  "      <%= render 'shared/pagination', collection: @users %>\n"
+  "    </div>\n"
+  "  </main>\n"
+  "  <footer class=\"footer\">\n"
+  "    <p>&copy; <%= Time.current.year %> Company</p>\n"
+  "  </footer>\n"
+  "</body>\n"
+  "</html>\n";
+
+typedef struct {
+  const char* name;
+  const char* source;
+} test_case_T;
+
+static void run_lex_benchmark(const char* name, const char* source) {
+  hb_allocator_T allocator = hb_allocator_with_tracking();
+
+  hb_array_T* tokens = herb_lex(source, &allocator);
+
+  hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+
+  printf("  lex  %-10s  allocs: %-6zu  deallocs: %-6zu  bytes_alloc: %-8zu  tokens: %zu\n",
+    name, stats->allocation_count, stats->deallocation_count, stats->bytes_allocated, tokens->size);
+
+  herb_free_tokens(&tokens, &allocator);
+  hb_allocator_destroy(&allocator);
+}
+
+static void run_parse_benchmark(const char* name, const char* source) {
+  hb_allocator_T allocator = hb_allocator_with_tracking();
+
+  AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL, &allocator);
+
+  hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+
+  printf("  parse %-9s  allocs: %-6zu  deallocs: %-6zu  bytes_alloc: %-8zu\n",
+    name, stats->allocation_count, stats->deallocation_count, stats->bytes_allocated);
+
+  ast_node_free((AST_NODE_T*) root, &allocator);
+  hb_allocator_destroy(&allocator);
+}
+
+int main(void) {
+  test_case_T cases[] = {
+    { "small",  SMALL_INPUT },
+    { "medium", MEDIUM_INPUT },
+    { "large",  LARGE_INPUT },
+  };
+
+  size_t num_cases = sizeof(cases) / sizeof(cases[0]);
+
+  printf("=== Allocation Benchmark ===\n\n");
+
+  for (size_t i = 0; i < num_cases; i++) {
+    printf("[%s] (%zu bytes input)\n", cases[i].name, strlen(cases[i].source));
+    run_lex_benchmark(cases[i].name, cases[i].source);
+    run_parse_benchmark(cases[i].name, cases[i].source);
+    printf("\n");
+  }
+
+  return 0;
+}
diff --git a/config.yml b/config.yml
@@ -32,8 +32,8 @@ errors:
       message:
         template: "Found %s when expecting %s at (%u:%u)."
         arguments:
-          - token_type_to_friendly_string(found->type)
-          - token_type_to_friendly_string(expected_type)
+          - hb_string(token_type_to_friendly_string(found->type))
+          - hb_string(token_type_to_friendly_string(expected_type))
           - found->location.start.line
           - found->location.start.column
 

diff --git a/ext/herb/extension.c b/ext/herb/extension.c
@@ -121,10 +121,11 @@ static VALUE Herb_lex_file(int argc, VALUE* argv, VALUE self) {
 
   lex_args_T args = { 0 };
   args.source = read_file_to_ruby_string(file_path);
+  char* string = (char*) check_string(args.source);
 
   if (!hb_allocator_init(&args.allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
 
-  args.tokens = herb_lex_file(file_path, &args.allocator);
+  args.tokens = herb_lex(string, &args.allocator);
 
   if (print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args.allocator.context); }
 

diff --git a/ext/herb/extension_helpers.c b/ext/herb/extension_helpers.c
@@ -9,6 +9,7 @@
 #include "../../src/include/location.h"
 #include "../../src/include/position.h"
 #include "../../src/include/token.h"
+#include "../../src/include/util/hb_string.h"
 
 const char* check_string(VALUE value) {
   if (NIL_P(value)) { return NULL; }
@@ -44,14 +45,19 @@ VALUE rb_range_from_c_struct(range_T range) {
   return rb_class_new_instance(2, args, cRange);
 }
 
+VALUE rb_string_from_hb_string(hb_string_T string) {
+  if (hb_string_is_null(string)) { return Qnil; }
+
+  return rb_utf8_str_new(string.data, string.length);
+}
+
 VALUE rb_token_from_c_struct(token_T* token) {
   if (!token) { return Qnil; }
 
-  VALUE value = token->value ? rb_utf8_str_new_cstr(token->value) : Qnil;
-
+  VALUE value = rb_string_from_hb_string(token->value);
   VALUE range = rb_range_from_c_struct(token->range);
   VALUE location = rb_location_from_c_struct(token->location);
-  VALUE type = rb_utf8_str_new_cstr(token_type_to_string(token->type));
+  VALUE type = rb_string_from_hb_string(token_type_to_string(token->type));
 
   VALUE args[4] = { value, range, location, type };
 

diff --git a/ext/herb/extension_helpers.h b/ext/herb/extension_helpers.h
@@ -11,6 +11,7 @@
 
 const char* check_string(VALUE value);
 VALUE read_file_to_ruby_string(const char* file_path);
+VALUE rb_string_from_hb_string(hb_string_T string);
 
 VALUE rb_position_from_c_struct(position_T position);
 VALUE rb_location_from_c_struct(location_T location);

diff --git a/java/extension_helpers.c b/java/extension_helpers.c
@@ -10,11 +10,24 @@
 #include "../../src/include/range.h"
 #include "../../src/include/token.h"
 #include "../../src/include/util/hb_array.h"
+#include "../../src/include/util/hb_string.h"
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+jstring CreateStringFromHbString(JNIEnv* env, hb_string_T string) {
+  if (hb_string_is_null(string)) { return NULL; }
+
+  char* c_string = hb_string_to_c_string_using_malloc(string);
+  if (!c_string) { return NULL; }
+
+  jstring result = (*env)->NewStringUTF(env, c_string);
+  free(c_string);
+
+  return result;
+}
+
 jobject CreatePosition(JNIEnv* env, position_T position) {
   jclass positionClass = (*env)->FindClass(env, "org/herb/Position");
   jmethodID constructor = (*env)->GetMethodID(env, positionClass, "<init>", "(II)V");
@@ -47,8 +60,8 @@ jobject CreateToken(JNIEnv* env, token_T* token) {
   jmethodID constructor = (*env)->GetMethodID(
       env, tokenClass, "<init>", "(Ljava/lang/String;Ljava/lang/String;Lorg/herb/Location;Lorg/herb/Range;)V");
 
-  jstring type = (*env)->NewStringUTF(env, token_type_to_string(token->type));
-  jstring value = (*env)->NewStringUTF(env, token->value);
+  jstring type = CreateStringFromHbString(env, token_type_to_string(token->type));
+  jstring value = CreateStringFromHbString(env, token->value);
   jobject location = CreateLocation(env, token->location);
   jobject range = CreateRange(env, token->range);
 

diff --git a/java/extension_helpers.h b/java/extension_helpers.h
@@ -9,11 +9,13 @@
 #include "../../src/include/range.h"
 #include "../../src/include/token.h"
 #include "../../src/include/util/hb_array.h"
+#include "../../src/include/util/hb_string.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+jstring CreateStringFromHbString(JNIEnv* env, hb_string_T string);
 jobject CreatePosition(JNIEnv* env, position_T position);
 jobject CreateLocation(JNIEnv* env, location_T location);
 jobject CreateRange(JNIEnv* env, range_T range);

diff --git a/javascript/packages/core/src/util.ts b/javascript/packages/core/src/util.ts
@@ -5,15 +5,3 @@ export function ensureString(object: any): string {
 
   throw new TypeError("Argument must be a string")
 }
-
-export function convertToUTF8(string: string) {
-  const bytes = []
-
-  for (let i = 0; i < string.length; i++) {
-    bytes.push(string.charCodeAt(i))
-  }
-
-  const decoder = new TextDecoder("utf-8")
-
-  return decoder.decode(new Uint8Array(bytes))
-}