diff --git a/src/include/token.h b/src/include/token.h index cba31e65f..9680bbf6f 100644 --- a/src/include/token.h +++ b/src/include/token.h @@ -6,7 +6,7 @@ #include "token_struct.h" #include "util/hb_string.h" -token_T* token_init(const char* value, token_type_T type, lexer_T* lexer); +token_T* token_init(hb_string_T value, token_type_T type, lexer_T* lexer); hb_string_T token_to_string(const token_T* token); const char* token_type_to_string(token_type_T type); diff --git a/src/include/util/hb_string.h b/src/include/util/hb_string.h index 6d59e2d02..7b5d83c3c 100644 --- a/src/include/util/hb_string.h +++ b/src/include/util/hb_string.h @@ -20,6 +20,10 @@ bool hb_string_starts_with(hb_string_T string, hb_string_T expected_prefix); bool hb_string_is_empty(hb_string_T string); hb_string_T hb_string_truncate(hb_string_T string, uint32_t max_length); +hb_string_T hb_string_range(hb_string_T string, uint32_t from, uint32_t to); + +char* hb_string_to_c_string_using_malloc(hb_string_T string); + char* hb_string_to_c_string(hb_arena_T* allocator, hb_string_T string); #endif diff --git a/src/lexer.c b/src/lexer.c index 45995751c..a2832b44c 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -67,7 +67,7 @@ token_T* lexer_error(lexer_T* lexer, const char* message) { lexer->current_column ); - return token_init(error_message, TOKEN_ERROR, lexer); + return token_init(hb_string(error_message), TOKEN_ERROR, lexer); } static void lexer_advance(lexer_T* lexer) { @@ -104,22 +104,17 @@ static void lexer_advance_by(lexer_T* lexer, const size_t count) { static token_T* lexer_advance_with(lexer_T* lexer, const char* value, const token_type_T type) { lexer_advance_by(lexer, strlen(value)); - return token_init(value, type, lexer); + return token_init(hb_string(value), type, lexer); } static token_T* lexer_advance_with_next(lexer_T* lexer, size_t count, token_type_T type) { - char* collected = malloc(count + 1); - if (!collected) { return NULL; } + uint32_t start_position = lexer->current_position; for (size_t i = 0; i < count; i++) { - collected[i] = lexer->current_character; lexer_advance(lexer); } - collected[count] = '\0'; - - token_T* token = token_init(collected, type, lexer); - free(collected); + token_T* token = token_init(hb_string_range(lexer->source, start_position, lexer->current_position), type, lexer); return token; } @@ -130,29 +125,16 @@ static token_T* lexer_advance_current(lexer_T* lexer, const token_type_T type) { static token_T* lexer_advance_utf8_character(lexer_T* lexer, const token_type_T type) { int char_byte_length = utf8_sequence_length(lexer->source.data, lexer->current_position, lexer->source.length); - if (char_byte_length <= 1) { return lexer_advance_current(lexer, type); } - - char* utf8_char = malloc(char_byte_length + 1); - - if (!utf8_char) { return lexer_advance_current(lexer, type); } + uint32_t start_position = lexer->current_position; for (int i = 0; i < char_byte_length; i++) { - if (lexer->current_position + i >= lexer->source.length) { - free(utf8_char); - return lexer_advance_current(lexer, type); - } - - utf8_char[i] = lexer->source.data[lexer->current_position + i]; + if (lexer->current_position + i >= lexer->source.length) { return lexer_advance_current(lexer, type); } } - utf8_char[char_byte_length] = '\0'; - lexer_advance_utf8_bytes(lexer, char_byte_length); - token_T* token = token_init(utf8_char, type, lexer); - - free(utf8_char); + token_T* token = token_init(hb_string_range(lexer->source, start_position, lexer->current_position), type, lexer); return token; } @@ -168,37 +150,31 @@ static token_T* lexer_match_and_advance(lexer_T* lexer, const char* value, const // ===== Specialized Parsers static token_T* lexer_parse_whitespace(lexer_T* lexer) { - hb_buffer_T buffer; - hb_buffer_init(&buffer, 128); + uint32_t start_position = lexer->current_position; while (isspace(lexer->current_character) && lexer->current_character != '\n' && lexer->current_character != '\r' && !lexer_eof(lexer)) { - hb_buffer_append_char(&buffer, lexer->current_character); lexer_advance(lexer); } - token_T* token = token_init(buffer.value, TOKEN_WHITESPACE, lexer); - - free(buffer.value); + token_T* token = + token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_WHITESPACE, lexer); return token; } static token_T* lexer_parse_identifier(lexer_T* lexer) { - hb_buffer_T buffer; - hb_buffer_init(&buffer, 128); + uint32_t start_position = lexer->current_position; while ((isalnum(lexer->current_character) || lexer->current_character == '-' || lexer->current_character == '_' || lexer->current_character == ':') && !lexer_peek_for_html_comment_end(lexer, 0) && !lexer_eof(lexer)) { - hb_buffer_append_char(&buffer, lexer->current_character); lexer_advance(lexer); } - token_T* token = token_init(buffer.value, TOKEN_IDENTIFIER, lexer); - - free(buffer.value); + token_T* token = + token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_IDENTIFIER, lexer); return token; } @@ -219,20 +195,19 @@ static token_T* lexer_parse_erb_open(lexer_T* lexer) { } static token_T* lexer_parse_erb_content(lexer_T* lexer) { - hb_buffer_T buffer; - hb_buffer_init(&buffer, 1024); + uint32_t start_position = lexer->current_position; while (!lexer_peek_erb_end(lexer, 0)) { if (lexer_eof(lexer)) { - token_T* token = token_init(buffer.value, TOKEN_ERROR, lexer); // Handle unexpected EOF - - free(buffer.value); + token_T* token = token_init( + hb_string_range(lexer->source, start_position, lexer->current_position), + TOKEN_ERROR, + lexer + ); // Handle unexpected EOF return token; } - hb_buffer_append_char(&buffer, lexer->current_character); - if (is_newline(lexer->current_character)) { lexer->current_line++; lexer->current_column = 0; @@ -246,9 +221,8 @@ static token_T* lexer_parse_erb_content(lexer_T* lexer) { lexer->state = STATE_ERB_CLOSE; - token_T* token = token_init(buffer.value, TOKEN_ERB_CONTENT, lexer); - - free(buffer.value); + token_T* token = + token_init(hb_string_range(lexer->source, start_position, lexer->current_position), TOKEN_ERB_CONTENT, lexer); return token; } @@ -266,7 +240,7 @@ static token_T* lexer_parse_erb_close(lexer_T* lexer) { // ===== Tokenizing Function token_T* lexer_next_token(lexer_T* lexer) { - if (lexer_eof(lexer)) { return token_init("", TOKEN_EOF, lexer); } + if (lexer_eof(lexer)) { return token_init(hb_string(""), TOKEN_EOF, lexer); } if (lexer_stalled(lexer)) { return lexer_error(lexer, "Lexer stalled after 5 iterations"); } if (lexer->state == STATE_ERB_CONTENT) { return lexer_parse_erb_content(lexer); } diff --git a/src/token.c b/src/token.c index 42c62bcac..e91837e5f 100644 --- a/src/token.c +++ b/src/token.c @@ -10,7 +10,7 @@ #include #include -token_T* token_init(const char* value, const token_type_T type, lexer_T* lexer) { +token_T* token_init(hb_string_T value, const token_type_T type, lexer_T* lexer) { token_T* token = calloc(1, sizeof(token_T)); if (type == TOKEN_NEWLINE) { @@ -18,11 +18,7 @@ token_T* token_init(const char* value, const token_type_T type, lexer_T* lexer) lexer->current_column = 0; } - if (value) { - token->value = herb_strdup(value); - } else { - token->value = NULL; - } + token->value = hb_string_to_c_string_using_malloc(value); token->type = type; token->range = (range_T) { .from = lexer->previous_position, .to = lexer->current_position }; diff --git a/src/util/hb_string.c b/src/util/hb_string.c index 358bc53fd..e8a3e83a3 100644 --- a/src/util/hb_string.c +++ b/src/util/hb_string.c @@ -58,6 +58,21 @@ hb_string_T hb_string_truncate(hb_string_T string, uint32_t max_length) { return truncated_string; } +hb_string_T hb_string_range(hb_string_T string, uint32_t from, uint32_t to) { + return hb_string_truncate(hb_string_slice(string, from), to - from); +} + +char* hb_string_to_c_string_using_malloc(hb_string_T string) { + size_t string_length_in_bytes = sizeof(char) * (string.length); + char* buffer = malloc(string_length_in_bytes + sizeof(char) * 1); + + if (!hb_string_is_empty(string)) { memcpy(buffer, string.data, string_length_in_bytes); } + + buffer[string_length_in_bytes] = '\0'; + + return buffer; +} + char* hb_string_to_c_string(hb_arena_T* allocator, hb_string_T string) { size_t string_length_in_bytes = sizeof(char) * (string.length); char* buffer = hb_arena_alloc(allocator, string_length_in_bytes + sizeof(char) * 1); diff --git a/test/c/test_hb_string.c b/test/c/test_hb_string.c index 2d47baca2..cf0d7363f 100644 --- a/test/c/test_hb_string.c +++ b/test/c/test_hb_string.c @@ -195,6 +195,41 @@ TEST(hb_string_truncate_tests) } END +TEST(hb_string_range_tests) + + { + hb_string_T string = hb_string("Test String"); + hb_string_T range = hb_string_range(string, 0, 4); + + ck_assert(hb_string_equals(hb_string("Test"), range)); + } + + { + hb_string_T string = hb_string("Test String"); + hb_string_T range = hb_string_range(string, 0, 1); + + ck_assert(hb_string_equals(hb_string("T"), range)); + } + + { + hb_string_T string = hb_string("Test String"); + hb_string_T range = hb_string_range(string, 5, 11); + ck_assert(hb_string_equals(hb_string("String"), range)); + } + + { + hb_string_T string = hb_string("Test String"); + hb_string_T range = hb_string_range(string, 5, 11); + ck_assert(hb_string_equals(hb_string("String"), range)); + } + + { + hb_string_T string = hb_string(" Test "); + hb_string_T range = hb_string_range(string, 1, 5); + ck_assert(hb_string_equals(hb_string("Test"), range)); + } +END + TCase *hb_string_tests(void) { TCase *tags = tcase_create("Herb String"); @@ -204,6 +239,7 @@ TCase *hb_string_tests(void) { tcase_add_test(tags, hb_string_is_empty_tests); tcase_add_test(tags, hb_string_starts_with_tests); tcase_add_test(tags, hb_string_truncate_tests); + tcase_add_test(tags, hb_string_range_tests); return tags; }