diff --git a/.gitignore b/.gitignore index 0fae38c52..d320f5db4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ docs/docs/linter/rules/ # Herb output /herb /run_herb_tests +/bench_allocs # Herb Ruby extension /ext/herb/extconf.h diff --git a/Makefile b/Makefile index 4638055ed..a5d67dd00 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,9 @@ test_sources = $(wildcard test/**/*.c) test_objects = $(test_sources:.c=.o) non_main_objects = $(filter-out src/main.o, $(objects)) +bench_allocs_exec = bench_allocs +bench_allocs_source = bench/bench_allocs.c + soext ?= $(shell ruby -e 'puts RbConfig::CONFIG["DLEXT"]') lib_name = $(build_dir)/lib$(exec).$(soext) static_lib_name = $(build_dir)/lib$(exec).a @@ -102,9 +105,14 @@ test/%.o: test/%.c templates prism test: $(test_objects) $(non_main_objects) $(cc) $(test_objects) $(non_main_objects) $(test_cflags) $(test_ldflags) -o $(test_exec) +.PHONY: bench_allocs +bench_allocs: $(non_main_objects) + $(cc) $(bench_allocs_source) $(non_main_objects) $(flags) $(prism_ldflags) -o $(bench_allocs_exec) + ./$(bench_allocs_exec) + .PHONY: clean clean: - rm -f $(exec) $(test_exec) $(lib_name) $(shared_lib_name) $(ruby_extension) + rm -f $(exec) $(test_exec) $(bench_allocs_exec) $(lib_name) $(shared_lib_name) $(ruby_extension) rm -rf $(objects) $(test_objects) $(extension_objects) lib/herb/*.bundle tmp rm -rf $(prism_path) rake prism:clean diff --git a/bench/bench_allocs.c b/bench/bench_allocs.c new file mode 100644 index 000000000..3bc3e94b2 --- /dev/null +++ b/bench/bench_allocs.c @@ -0,0 +1,178 @@ +#include "../src/include/herb.h" +#include "../src/include/util/hb_allocator.h" + +#include +#include +#include + +static const char* SMALL_INPUT = "
<%= foo %>
"; + +static const char* MEDIUM_INPUT = + "\n" + "\n" + "\n" + " \n" + " <%= @title %>\n" + "\n" + "\n" + "
\n" + "

<%= @heading %>

\n" + " <% @items.each do |item| %>\n" + "
\">\n" + " <%= item.name %>\n" + "

<%= item.description %>

\n" + " <% if item.active? %>\n" + " Active\n" + " <% else %>\n" + " Inactive\n" + " <% end %>\n" + "
\n" + " <% end %>\n" + "
\n" + " <%# This is a comment %>\n" + " <%= render partial: 'footer', locals: { year: 2024 } %>\n" + "\n" + "\n"; + +static const char* LARGE_INPUT = + "\n" + "\">\n" + "\n" + " \n" + " \n" + " <%= @page_title || 'Default' %>\n" + " <%= csrf_meta_tags %>\n" + " <%= csp_meta_tag %>\n" + " <%= stylesheet_link_tag 'application' %>\n" + " <%= javascript_include_tag 'application' %>\n" + "\n" + "\">\n" + " \n" + "
\n" + " <% if flash[:notice] %>\n" + "
<%= flash[:notice] %>
\n" + " <% end %>\n" + " <% if flash[:alert] %>\n" + "
<%= flash[:alert] %>
\n" + " <% end %>\n" + "
\n" + "

<%= @heading %>

\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " <% @users.each do |user| %>\n" + " \" class=\"<%= cycle('odd', 'even') %>\">\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " <% end %>\n" + " \n" + "
NameEmailRoleStatusActions
<%= user.name %><%= mail_to user.email %><%= user.role.titleize %>\n" + " <% if user.active? %>\n" + " Active\n" + " <% elsif user.suspended? %>\n" + " Suspended\n" + " <% else %>\n" + " Inactive\n" + " <% end %>\n" + " \n" + " <%= link_to 'View', user_path(user), class: 'btn btn-sm' %>\n" + " <%= link_to 'Edit', edit_user_path(user), class: 'btn btn-sm' %>\n" + " <% if can?(:destroy, user) %>\n" + " <%= link_to 'Delete', user_path(user), method: :delete, class: 'btn btn-sm btn-danger', data: { confirm: 'Are you sure?' } %>\n" + " <% end %>\n" + "
\n" + " <%= render 'shared/pagination', collection: @users %>\n" + "
\n" + "
\n" + "
\n" + "

© <%= Time.current.year %> Company

\n" + "
\n" + "\n" + "\n"; + +typedef struct { + const char* name; + const char* source; +} test_case_T; + +static void run_lex_benchmark(const char* name, const char* source) { + hb_allocator_T allocator = hb_allocator_with_tracking(); + + hb_array_T* tokens = herb_lex(source, &allocator); + + hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator); + + printf(" lex %-10s allocs: %-6zu deallocs: %-6zu bytes_alloc: %-8zu tokens: %zu\n", + name, stats->allocation_count, stats->deallocation_count, stats->bytes_allocated, tokens->size); + + herb_free_tokens(&tokens, &allocator); + hb_allocator_destroy(&allocator); +} + +static void run_parse_benchmark(const char* name, const char* source) { + hb_allocator_T allocator = hb_allocator_with_tracking(); + + AST_DOCUMENT_NODE_T* root = herb_parse(source, NULL, &allocator); + + hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator); + + printf(" parse %-9s allocs: %-6zu deallocs: %-6zu bytes_alloc: %-8zu\n", + name, stats->allocation_count, stats->deallocation_count, stats->bytes_allocated); + + ast_node_free((AST_NODE_T*) root, &allocator); + hb_allocator_destroy(&allocator); +} + +int main(void) { + test_case_T cases[] = { + { "small", SMALL_INPUT }, + { "medium", MEDIUM_INPUT }, + { "large", LARGE_INPUT }, + }; + + size_t num_cases = sizeof(cases) / sizeof(cases[0]); + + printf("=== Allocation Benchmark ===\n\n"); + + for (size_t i = 0; i < num_cases; i++) { + printf("[%s] (%zu bytes input)\n", cases[i].name, strlen(cases[i].source)); + run_lex_benchmark(cases[i].name, cases[i].source); + run_parse_benchmark(cases[i].name, cases[i].source); + printf("\n"); + } + + return 0; +} diff --git a/config.yml b/config.yml index 2363d9a77..421b8ee46 100644 --- a/config.yml +++ b/config.yml @@ -32,8 +32,8 @@ errors: message: template: "Found %s when expecting %s at (%u:%u)." arguments: - - token_type_to_friendly_string(found->type) - - token_type_to_friendly_string(expected_type) + - hb_string(token_type_to_friendly_string(found->type)) + - hb_string(token_type_to_friendly_string(expected_type)) - found->location.start.line - found->location.start.column diff --git a/ext/herb/extension.c b/ext/herb/extension.c index 697c9e487..3531ffb33 100644 --- a/ext/herb/extension.c +++ b/ext/herb/extension.c @@ -121,10 +121,11 @@ static VALUE Herb_lex_file(int argc, VALUE* argv, VALUE self) { lex_args_T args = { 0 }; args.source = read_file_to_ruby_string(file_path); + char* string = (char*) check_string(args.source); if (!hb_allocator_init(&args.allocator, HB_ALLOCATOR_ARENA)) { return Qnil; } - args.tokens = herb_lex_file(file_path, &args.allocator); + args.tokens = herb_lex(string, &args.allocator); if (print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args.allocator.context); } diff --git a/ext/herb/extension_helpers.c b/ext/herb/extension_helpers.c index f2295302e..545c1e0fb 100644 --- a/ext/herb/extension_helpers.c +++ b/ext/herb/extension_helpers.c @@ -9,6 +9,7 @@ #include "../../src/include/location.h" #include "../../src/include/position.h" #include "../../src/include/token.h" +#include "../../src/include/util/hb_string.h" const char* check_string(VALUE value) { if (NIL_P(value)) { return NULL; } @@ -44,14 +45,19 @@ VALUE rb_range_from_c_struct(range_T range) { return rb_class_new_instance(2, args, cRange); } +VALUE rb_string_from_hb_string(hb_string_T string) { + if (hb_string_is_null(string)) { return Qnil; } + + return rb_utf8_str_new(string.data, string.length); +} + VALUE rb_token_from_c_struct(token_T* token) { if (!token) { return Qnil; } - VALUE value = token->value ? rb_utf8_str_new_cstr(token->value) : Qnil; - + VALUE value = rb_string_from_hb_string(token->value); VALUE range = rb_range_from_c_struct(token->range); VALUE location = rb_location_from_c_struct(token->location); - VALUE type = rb_utf8_str_new_cstr(token_type_to_string(token->type)); + VALUE type = rb_string_from_hb_string(token_type_to_string(token->type)); VALUE args[4] = { value, range, location, type }; diff --git a/ext/herb/extension_helpers.h b/ext/herb/extension_helpers.h index b0a74f212..d824812f7 100644 --- a/ext/herb/extension_helpers.h +++ b/ext/herb/extension_helpers.h @@ -11,6 +11,7 @@ const char* check_string(VALUE value); VALUE read_file_to_ruby_string(const char* file_path); +VALUE rb_string_from_hb_string(hb_string_T string); VALUE rb_position_from_c_struct(position_T position); VALUE rb_location_from_c_struct(location_T location); diff --git a/java/extension_helpers.c b/java/extension_helpers.c index 7ad963410..023ca5cff 100644 --- a/java/extension_helpers.c +++ b/java/extension_helpers.c @@ -10,11 +10,24 @@ #include "../../src/include/range.h" #include "../../src/include/token.h" #include "../../src/include/util/hb_array.h" +#include "../../src/include/util/hb_string.h" #include #include #include +jstring CreateStringFromHbString(JNIEnv* env, hb_string_T string) { + if (hb_string_is_null(string)) { return NULL; } + + char* c_string = hb_string_to_c_string_using_malloc(string); + if (!c_string) { return NULL; } + + jstring result = (*env)->NewStringUTF(env, c_string); + free(c_string); + + return result; +} + jobject CreatePosition(JNIEnv* env, position_T position) { jclass positionClass = (*env)->FindClass(env, "org/herb/Position"); jmethodID constructor = (*env)->GetMethodID(env, positionClass, "", "(II)V"); @@ -47,8 +60,8 @@ jobject CreateToken(JNIEnv* env, token_T* token) { jmethodID constructor = (*env)->GetMethodID( env, tokenClass, "", "(Ljava/lang/String;Ljava/lang/String;Lorg/herb/Location;Lorg/herb/Range;)V"); - jstring type = (*env)->NewStringUTF(env, token_type_to_string(token->type)); - jstring value = (*env)->NewStringUTF(env, token->value); + jstring type = CreateStringFromHbString(env, token_type_to_string(token->type)); + jstring value = CreateStringFromHbString(env, token->value); jobject location = CreateLocation(env, token->location); jobject range = CreateRange(env, token->range); diff --git a/java/extension_helpers.h b/java/extension_helpers.h index 536535040..4c7359cc2 100644 --- a/java/extension_helpers.h +++ b/java/extension_helpers.h @@ -9,11 +9,13 @@ #include "../../src/include/range.h" #include "../../src/include/token.h" #include "../../src/include/util/hb_array.h" +#include "../../src/include/util/hb_string.h" #ifdef __cplusplus extern "C" { #endif +jstring CreateStringFromHbString(JNIEnv* env, hb_string_T string); jobject CreatePosition(JNIEnv* env, position_T position); jobject CreateLocation(JNIEnv* env, location_T location); jobject CreateRange(JNIEnv* env, range_T range); diff --git a/javascript/packages/core/src/util.ts b/javascript/packages/core/src/util.ts index 890aa13f8..8e000b36e 100644 --- a/javascript/packages/core/src/util.ts +++ b/javascript/packages/core/src/util.ts @@ -5,15 +5,3 @@ export function ensureString(object: any): string { throw new TypeError("Argument must be a string") } - -export function convertToUTF8(string: string) { - const bytes = [] - - for (let i = 0; i < string.length; i++) { - bytes.push(string.charCodeAt(i)) - } - - const decoder = new TextDecoder("utf-8") - - return decoder.decode(new Uint8Array(bytes)) -} diff --git a/javascript/packages/node-wasm/test/__snapshots__/utf8-strings.test.ts.snap b/javascript/packages/node-wasm/test/__snapshots__/utf8-strings.test.ts.snap new file mode 100644 index 000000000..0c3133006 --- /dev/null +++ b/javascript/packages/node-wasm/test/__snapshots__/utf8-strings.test.ts.snap @@ -0,0 +1,432 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`UTF-8 string handling > extractHTML preserves UTF-8 > preserves UTF-8 in attributes 1`] = `"
πŸš€
"`; + +exports[`UTF-8 string handling > extractHTML preserves UTF-8 > preserves multi-byte characters 1`] = `"

CafΓ© β€” πŸš€ ВСст

"`; + +exports[`UTF-8 string handling > extractRuby preserves UTF-8 > preserves multi-byte characters 1`] = `" "CafΓ© πŸš€" ;"`; + +exports[`UTF-8 string handling > handles edge cases > incomplete comment has empty token value not null 1`] = ` +"@ DocumentNode (location: (1:0)-(1:4)) +β”œβ”€β”€ errors: [] +└── children: (1 item) + └── @ HTMLCommentNode (location: (1:0)-(1:4)) + β”œβ”€β”€ errors: (1 item) + β”‚ └── @ UnexpectedTokenError (location: (1:4)-(1:4)) + β”‚ β”œβ”€β”€ message: "Found end of file when expecting \`-->\` at (1:4)." + β”‚ β”œβ”€β”€ expected_type: "TOKEN_HTML_COMMENT_END" + β”‚ └── found: "" (location: (1:4)-(1:4)) + β”‚ + β”‚ + β”œβ”€β”€ comment_start: "\` at (1:13)." + β”‚ β”œβ”€β”€ expected_type: "TOKEN_HTML_COMMENT_END" + β”‚ └── found: "" (location: (1:13)-(1:13)) + β”‚ + β”‚ + β”œβ”€β”€ comment_start: "\` at (1:4)." + β”‚ β”œβ”€β”€ expected_type: "TOKEN_HTML_COMMENT_END" + β”‚ └── found: "" (location: (1:4)-(1:4)) + β”‚ + β”‚ + β”œβ”€β”€ comment_start: "\` at (1:13)." + β”‚ β”œβ”€β”€ expected_type: "TOKEN_HTML_COMMENT_END" + β”‚ └── found: "" (location: (1:13)-(1:13)) + β”‚ + β”‚ + β”œβ”€β”€ comment_start: "