From dfca529319e1f7ff68962b9c677949d4f7f69e47 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 16 Oct 2025 12:55:17 -0700 Subject: [PATCH 01/58] Explicitly set bdwgc debug flags --- compiler+runtime/cmake/dependency/bdwgc.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler+runtime/cmake/dependency/bdwgc.cmake b/compiler+runtime/cmake/dependency/bdwgc.cmake index fff5b9cba..26706a9f1 100644 --- a/compiler+runtime/cmake/dependency/bdwgc.cmake +++ b/compiler+runtime/cmake/dependency/bdwgc.cmake @@ -11,11 +11,15 @@ set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY}) set(enable_docs OFF CACHE BOOL "Enable docs") set(enable_large_config ON CACHE BOOL "Optimize for large heap or root set") set(enable_throw_bad_alloc_library ON CACHE BOOL "Enable C++ gctba library build") + set(enable_gc_debug OFF CACHE BOOL "Support for pointer back-tracing") add_subdirectory(third-party/bdwgc EXCLUDE_FROM_ALL) unset(enable_cplusplus) unset(build_cord) unset(enable_docs) + unset(enable_large_config) + unset(enable_throw_bad_alloc_library) + unset(enable_gc_debug) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_OLD}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_OLD}") set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_OLD}) From bf31860d7106b98ed02c4bb27c7983e73fb77387 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 16 Oct 2025 12:56:30 -0700 Subject: [PATCH 02/58] Move reusable_context to jtl::ref --- .../cpp/jank/codegen/llvm_processor.hpp | 2 +- .../src/cpp/jank/codegen/llvm_processor.cpp | 39 ++++++------------- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/codegen/llvm_processor.hpp b/compiler+runtime/include/cpp/jank/codegen/llvm_processor.hpp index c2bb669c6..123969dc1 100644 --- a/compiler+runtime/include/cpp/jank/codegen/llvm_processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/llvm_processor.hpp @@ -94,7 +94,7 @@ namespace jank::codegen compilation_target target); /* For this ctor, we're inheriting the context from another function, which means * we're building a nested function. */ - llvm_processor(analyze::expr::function_ref expr, std::unique_ptr ctx); + llvm_processor(analyze::expr::function_ref expr, jtl::ref ctx); llvm_processor(llvm_processor const &) = delete; llvm_processor(llvm_processor &&) noexcept = default; diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index 05eef930f..c55e2f48e 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -88,7 +88,7 @@ namespace jank::codegen compilation_target target); /* For this ctor, we're inheriting the context from another function, which means * we're building a nested function. */ - impl(analyze::expr::function_ref expr, std::unique_ptr ctx); + impl(analyze::expr::function_ref expr, jtl::ref ctx); jtl::string_result gen(); llvm::Value *gen(analyze::expression_ref, analyze::expr::function_arity const &); @@ -181,10 +181,9 @@ namespace jank::codegen compilation_target target{}; analyze::expr::function_ref root_fn; jtl::ptr fn{}; - std::unique_ptr ctx; + jtl::ref ctx; native_unordered_map> locals; - /* TODO: Use gc allocator to avoid leaks. */ - std::list deferred_inits{}; + native_list deferred_inits{}; jtl::ref llvm_ctx; jtl::ref llvm_module; jtl::ptr current_loop; @@ -479,9 +478,8 @@ namespace jank::codegen /* Whenever we have an object in an `alloca`, we need to load it before using. This fn only * makes sense to use with jank objects, as opposed to native values. */ - static llvm::Value *load_if_needed(std::unique_ptr const &ctx, - llvm::Value *arg, - jtl::ptr const type) + static llvm::Value * + load_if_needed(jtl::ref const ctx, llvm::Value *arg, jtl::ptr const type) { if(!arg) { @@ -495,8 +493,7 @@ namespace jank::codegen return arg; } - static llvm::Value * - load_if_needed(std::unique_ptr const &ctx, llvm::Value * const arg) + static llvm::Value *load_if_needed(jtl::ref const ctx, llvm::Value * const arg) { return load_if_needed(ctx, arg, cpp_util::untyped_object_ptr_type()); } @@ -576,8 +573,8 @@ namespace jank::codegen } llvm_processor::llvm_processor(expr::function_ref const expr, - std::unique_ptr ctx) - : _impl{ make_ref(expr, jtl::move(ctx)) } + jtl::ref const ctx) + : _impl{ make_ref(expr, ctx) } { } @@ -586,13 +583,13 @@ namespace jank::codegen compilation_target const target) : target{ target } , root_fn{ expr } - , ctx{ std::make_unique(module_name, std::make_unique()) } + , ctx{ make_ref(module_name, std::make_unique()) } , llvm_ctx{ extract_context(ctx->module) } , llvm_module{ ctx->module.getModuleUnlocked() } { } - llvm_processor::impl::impl(expr::function_ref const expr, std::unique_ptr ctx) + llvm_processor::impl::impl(expr::function_ref const expr, jtl::ref ctx) : target{ compilation_target::function } , root_fn{ expr } , ctx{ std::move(ctx) } @@ -1147,27 +1144,13 @@ namespace jank::codegen { llvm::IRBuilder<>::InsertPointGuard const guard{ *ctx->builder }; - llvm_processor nested{ expr, std::move(ctx) }; - - /* We need to make sure to transfer ownership of the context back, even if an exception - * is thrown. */ - util::scope_exit const finally{ [&]() { - if(nested._impl->ctx) - { - ctx = std::move(nested._impl->ctx); - } - } }; - + llvm_processor nested{ expr, ctx }; auto const res{ nested.gen() }; if(res.is_err()) { /* TODO: Return error. */ res.expect_ok(); } - - /* This is covered by finally, but clang-tidy can't figure that out, so we have - * to make this more clear. */ - ctx = std::move(nested._impl->ctx); } auto const fn_obj(gen_function_instance(expr, fn_arity)); From b8f24b53e4e955692fddbd8f73c3ff6f4b3ddcdc Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 16 Oct 2025 12:57:06 -0700 Subject: [PATCH 03/58] Use gc allocator and remove thread_local usage --- .../include/cpp/jank/read/parse.hpp | 2 +- .../include/cpp/jank/runtime/context.hpp | 5 ++--- .../src/cpp/jank/analyze/processor.cpp | 2 +- .../src/cpp/jank/runtime/context.cpp | 17 ++++++----------- compiler+runtime/src/cpp/jank/runtime/var.cpp | 2 +- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/read/parse.hpp b/compiler+runtime/include/cpp/jank/read/parse.hpp index 8de1c5e14..e42d364d4 100644 --- a/compiler+runtime/include/cpp/jank/read/parse.hpp +++ b/compiler+runtime/include/cpp/jank/read/parse.hpp @@ -117,7 +117,7 @@ namespace jank::read::parse * token, we should check this list to see if there's already a form we should pull out. * This is needed because parse iteration works one form at a time and splicing potentially * turns one form into many. */ - std::list pending_forms; + native_list pending_forms; lex::token latest_token; jtl::option shorthand; /* Whether or not the next form is considered quoted. */ diff --git a/compiler+runtime/include/cpp/jank/runtime/context.hpp b/compiler+runtime/include/cpp/jank/runtime/context.hpp index 353b17fcf..d6661e10e 100644 --- a/compiler+runtime/include/cpp/jank/runtime/context.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/context.hpp @@ -43,7 +43,6 @@ namespace jank::runtime context(); context(context const &) = delete; context(context &&) noexcept = delete; - ~context(); ns_ref intern_ns(jtl::immutable_string const &); ns_ref intern_ns(obj::symbol_ref const &); @@ -162,8 +161,8 @@ namespace jank::runtime /* Hold onto the CLI Options for use at runtime */ util::cli::options opts; - /* TODO: Remove this map. Just use the list. */ - static thread_local native_unordered_map> + /* XXX: We can't use thread_local here, due to bdwgc not supporting it. */ + static native_unordered_map> thread_binding_frames; /* This must go last, since it'll try to access other bits in the runtime context during diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 1ece62ee9..e0408cf1a 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -1632,7 +1632,7 @@ namespace jank::analyze native_vector param_symbols; param_symbols.reserve(params->data.size()); - std::set unique_param_symbols; + native_set unique_param_symbols; bool is_variadic{}; for(auto it(params->data.begin()); it != params->data.end(); ++it) diff --git a/compiler+runtime/src/cpp/jank/runtime/context.cpp b/compiler+runtime/src/cpp/jank/runtime/context.cpp index 4eea03e64..4af42365a 100644 --- a/compiler+runtime/src/cpp/jank/runtime/context.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/context.cpp @@ -34,7 +34,7 @@ namespace jank::runtime { /* NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) */ - thread_local decltype(context::thread_binding_frames) context::thread_binding_frames{}; + decltype(context::thread_binding_frames) context::thread_binding_frames{}; /* NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) */ context *__rt_ctx{}; @@ -93,11 +93,6 @@ namespace jank::runtime .expect_ok(); } - context::~context() - { - thread_binding_frames.erase(this); - } - obj::symbol_ref context::qualify_symbol(obj::symbol_ref const &sym) const { obj::symbol_ref qualified_sym{ sym }; @@ -708,7 +703,7 @@ namespace jank::runtime jtl::string_result context::push_thread_bindings() { auto bindings(obj::persistent_hash_map::empty()); - auto &tbfs(thread_binding_frames[this]); + auto &tbfs(thread_binding_frames[std::this_thread::get_id()]); if(!tbfs.empty()) { bindings = tbfs.front().bindings; @@ -737,7 +732,7 @@ namespace jank::runtime context::push_thread_bindings(obj::persistent_hash_map_ref const bindings) { thread_binding_frame frame{ obj::persistent_hash_map::empty() }; - auto &tbfs(thread_binding_frames[this]); + auto &tbfs(thread_binding_frames[std::this_thread::get_id()]); if(!tbfs.empty()) { frame.bindings = tbfs.front().bindings; @@ -780,7 +775,7 @@ namespace jank::runtime jtl::string_result context::pop_thread_bindings() { - auto &tbfs(thread_binding_frames[this]); + auto &tbfs(thread_binding_frames[std::this_thread::get_id()]); if(tbfs.empty()) { return err("Mismatched thread binding pop"); @@ -793,7 +788,7 @@ namespace jank::runtime obj::persistent_hash_map_ref context::get_thread_bindings() const { - auto const &tbfs(thread_binding_frames[this]); + auto const &tbfs(thread_binding_frames[std::this_thread::get_id()]); if(tbfs.empty()) { return obj::persistent_hash_map::empty(); @@ -803,7 +798,7 @@ namespace jank::runtime jtl::option context::current_thread_binding_frame() { - auto &tbfs(thread_binding_frames[this]); + auto &tbfs(thread_binding_frames[std::this_thread::get_id()]); if(tbfs.empty()) { return none; diff --git a/compiler+runtime/src/cpp/jank/runtime/var.cpp b/compiler+runtime/src/cpp/jank/runtime/var.cpp index 44f739671..aea909fc9 100644 --- a/compiler+runtime/src/cpp/jank/runtime/var.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/var.cpp @@ -150,7 +150,7 @@ namespace jank::runtime return {}; } - auto &tbfs(__rt_ctx->thread_binding_frames[__rt_ctx]); + auto &tbfs(__rt_ctx->thread_binding_frames[std::this_thread::get_id()]); if(tbfs.empty()) { return {}; From c538cd5582f124e2b39aa2053046afa43e4c9b8e Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 16 Oct 2025 16:58:02 -0700 Subject: [PATCH 04/58] Upgrade bdwgc and use malloc redirection This currently doesn't work with ASan, but I have an open issue for this here: https://github.com/bdwgc/bdwgc/issues/772 --- compiler+runtime/CMakeLists.txt | 1 + compiler+runtime/cmake/dependency/bdwgc.cmake | 11 +++++++++-- compiler+runtime/src/cpp/jank/c_api.cpp | 1 - compiler+runtime/src/cpp/jank/read/lex.cpp | 2 +- compiler+runtime/src/cpp/jank/read/parse.cpp | 12 ++++++------ compiler+runtime/src/cpp/jank/runtime/context.cpp | 2 +- .../cpp/jank/runtime/obj/native_vector_sequence.cpp | 6 +++--- compiler+runtime/third-party/bdwgc | 2 +- 8 files changed, 22 insertions(+), 15 deletions(-) diff --git a/compiler+runtime/CMakeLists.txt b/compiler+runtime/CMakeLists.txt index 4eefdd793..1272533a2 100644 --- a/compiler+runtime/CMakeLists.txt +++ b/compiler+runtime/CMakeLists.txt @@ -42,6 +42,7 @@ option(jank_coverage "Enable code coverage measurement" OFF) option(jank_analyze "Enable static analysis" OFF) option(jank_test "Enable jank's test suite" OFF) option(jank_unity_build "Optimize translation unit compilation for the number of cores" OFF) +option(jank_debug_gc "Enable GC debug assertions" OFF) set(jank_sanitize "none" CACHE STRING "The type of Clang sanitization to use (or none)") set(jank_resource_dir "../lib/jank/${CMAKE_PROJECT_VERSION}" diff --git a/compiler+runtime/cmake/dependency/bdwgc.cmake b/compiler+runtime/cmake/dependency/bdwgc.cmake index 26706a9f1..a0b897c78 100644 --- a/compiler+runtime/cmake/dependency/bdwgc.cmake +++ b/compiler+runtime/cmake/dependency/bdwgc.cmake @@ -2,8 +2,8 @@ set(CMAKE_C_FLAGS_OLD "${CMAKE_C_FLAGS}") set(CMAKE_CXX_FLAGS_OLD "${CMAKE_CXX_FLAGS}") set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS}) set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY}) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable") set(BUILD_SHARED_LIBS OFF) set(CMAKE_CXX_CLANG_TIDY "") set(enable_cplusplus ON CACHE BOOL "Enable C++") @@ -12,6 +12,13 @@ set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY}) set(enable_large_config ON CACHE BOOL "Optimize for large heap or root set") set(enable_throw_bad_alloc_library ON CACHE BOOL "Enable C++ gctba library build") set(enable_gc_debug OFF CACHE BOOL "Support for pointer back-tracing") + + if(jank_debug_gc) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DGC_ASSERTIONS") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGC_ASSERTIONS") + set(enable_gc_debug ON CACHE BOOL "Support for pointer back-tracing") + endif() + add_subdirectory(third-party/bdwgc EXCLUDE_FROM_ALL) unset(enable_cplusplus) diff --git a/compiler+runtime/src/cpp/jank/c_api.cpp b/compiler+runtime/src/cpp/jank/c_api.cpp index b6f317a33..ac766ee0c 100644 --- a/compiler+runtime/src/cpp/jank/c_api.cpp +++ b/compiler+runtime/src/cpp/jank/c_api.cpp @@ -1017,7 +1017,6 @@ extern "C" /* The GC needs to enabled even before arg parsing, since our native types, * like strings, use the GC for allocations. It can still be configured later. */ GC_set_all_interior_pointers(1); - GC_enable(); GC_init(); llvm::llvm_shutdown_obj const Y{}; diff --git a/compiler+runtime/src/cpp/jank/read/lex.cpp b/compiler+runtime/src/cpp/jank/read/lex.cpp index dd71dab54..1e8339cce 100644 --- a/compiler+runtime/src/cpp/jank/read/lex.cpp +++ b/compiler+runtime/src/cpp/jank/read/lex.cpp @@ -1081,7 +1081,7 @@ namespace jank::read::lex return error::lex_invalid_number( util::format( "Characters '{}' are invalid for a base {} number.", - jtl::immutable_string_view{ invalid_digits.begin(), invalid_digits.end() }, + jtl::immutable_string_view{ invalid_digits.data(), invalid_digits.size() }, radix), { token_start, pos }); } diff --git a/compiler+runtime/src/cpp/jank/read/parse.cpp b/compiler+runtime/src/cpp/jank/read/parse.cpp index 4ca0535b8..40f2cdc20 100644 --- a/compiler+runtime/src/cpp/jank/read/parse.cpp +++ b/compiler+runtime/src/cpp/jank/read/parse.cpp @@ -409,7 +409,7 @@ namespace jank::read::parse parsed_keys.insert({ key.ptr, key }); - if constexpr(std::same_as) + if constexpr(jtl::is_same) { map.insert_or_assign(key.ptr, value.unwrap().ptr); } @@ -435,7 +435,7 @@ namespace jank::read::parse return object_source_info{ make_box( source_to_meta(start_token.start, latest_token.end), - std::move(map)), + jtl::move(map)), start_token, latest_token }; } @@ -453,7 +453,7 @@ namespace jank::read::parse return object_source_info{ make_box( source_to_meta(start_token.start, latest_token.end), - std::move(map)), + jtl::move(map)), start_token, latest_token }; } @@ -538,7 +538,7 @@ namespace jank::read::parse auto meta_result(visit_object( [&](auto const typed_val) -> processor::object_result { using T = typename decltype(typed_val)::value_type; - if constexpr(std::same_as) + if constexpr(jtl::is_same) { return object_source_info{ obj::persistent_array_map::create_unique(typed_val, jank_true), start_token, @@ -683,7 +683,7 @@ namespace jank::read::parse expected_closer = prev_expected_closer; return object_source_info{ make_box( source_to_meta(start_token.start, latest_token.end), - std::move(ret).persistent()), + jtl::move(ret).persistent()), start_token, latest_token }; } @@ -1285,7 +1285,7 @@ namespace jank::read::parse [&](auto const typed_form) -> jtl::result { using T = typename decltype(typed_form)::value_type; - if constexpr(std::same_as) + if constexpr(jtl::is_same) { auto const seq(typed_form->seq()); if(seq.is_nil()) diff --git a/compiler+runtime/src/cpp/jank/runtime/context.cpp b/compiler+runtime/src/cpp/jank/runtime/context.cpp index 4af42365a..24bb7c1dc 100644 --- a/compiler+runtime/src/cpp/jank/runtime/context.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/context.cpp @@ -186,7 +186,7 @@ namespace jank::runtime auto const name{ module::module_to_load_function(module) }; auto const form{ runtime::conj( - runtime::conj(runtime::conj(make_box(std::move(forms)), + runtime::conj(runtime::conj(make_box(jtl::move(forms)), obj::persistent_vector::empty()), make_box(name)), make_box("fn*")) }; diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/native_vector_sequence.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/native_vector_sequence.cpp index 7603e8e3e..cea169687 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/native_vector_sequence.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/native_vector_sequence.cpp @@ -12,13 +12,13 @@ namespace jank::runtime::obj } native_vector_sequence::native_vector_sequence(native_vector &&data) - : data{ std::move(data) } + : data{ jtl::move(data) } { jank_debug_assert(!this->data.empty()); } native_vector_sequence::native_vector_sequence(native_vector &&data, usize index) - : data{ std::move(data) } + : data{ jtl::move(data) } , index{ index } { jank_debug_assert(!this->data.empty()); @@ -26,7 +26,7 @@ namespace jank::runtime::obj native_vector_sequence::native_vector_sequence(jtl::option const &meta, native_vector &&data) - : data{ std::move(data) } + : data{ jtl::move(data) } , meta{ meta } { } diff --git a/compiler+runtime/third-party/bdwgc b/compiler+runtime/third-party/bdwgc index 88a61fc6f..19a7f495c 160000 --- a/compiler+runtime/third-party/bdwgc +++ b/compiler+runtime/third-party/bdwgc @@ -1 +1 @@ -Subproject commit 88a61fc6f2688d8e1ea94f3f810c23db89a3f30b +Subproject commit 19a7f495cd7c48cdf6f56c593c6dd57187afa079 From 238cfdbee3ca9161e6d0f4fcd9021955fdc650e0 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 16 Oct 2025 17:56:16 -0700 Subject: [PATCH 05/58] Fix clang-tidy issues --- compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp | 2 +- compiler+runtime/src/cpp/jank/runtime/var.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index c55e2f48e..1f4666a6d 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -1144,7 +1144,7 @@ namespace jank::codegen { llvm::IRBuilder<>::InsertPointGuard const guard{ *ctx->builder }; - llvm_processor nested{ expr, ctx }; + llvm_processor const nested{ expr, ctx }; auto const res{ nested.gen() }; if(res.is_err()) { diff --git a/compiler+runtime/src/cpp/jank/runtime/var.cpp b/compiler+runtime/src/cpp/jank/runtime/var.cpp index aea909fc9..8894d37e8 100644 --- a/compiler+runtime/src/cpp/jank/runtime/var.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/var.cpp @@ -150,7 +150,7 @@ namespace jank::runtime return {}; } - auto &tbfs(__rt_ctx->thread_binding_frames[std::this_thread::get_id()]); + auto &tbfs(runtime::context::thread_binding_frames[std::this_thread::get_id()]); if(tbfs.empty()) { return {}; From 0c6bfe16d242cda64e5f50358be3fcf3837ad2d6 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 5 Nov 2025 11:41:01 -0800 Subject: [PATCH 06/58] Remove BppTree It leaks, leads to premature GC, takes too long to compile, and doesn't support runtime sorting functions. We'll need a new solution. For now, we're deep copying standard containers, which isn't going to last us very long. --- .../include/cpp/jank/runtime/detail/type.hpp | 27 ++++++------------- .../jank/runtime/obj/transient_sorted_map.hpp | 3 +-- .../jank/runtime/obj/transient_sorted_set.hpp | 3 +-- compiler+runtime/include/cpp/jank/type.hpp | 2 +- .../src/cpp/jank/analyze/processor.cpp | 17 +----------- .../src/cpp/jank/runtime/detail/type.cpp | 8 ------ .../runtime/obj/persistent_sorted_map.cpp | 12 +++++---- .../runtime/obj/persistent_sorted_set.cpp | 18 +++++++------ .../jank/runtime/obj/transient_sorted_map.cpp | 15 ++++------- .../jank/runtime/obj/transient_sorted_set.cpp | 26 +++++++----------- compiler+runtime/third-party/bpptree | 1 - 11 files changed, 44 insertions(+), 88 deletions(-) delete mode 160000 compiler+runtime/third-party/bpptree diff --git a/compiler+runtime/include/cpp/jank/runtime/detail/type.hpp b/compiler+runtime/include/cpp/jank/runtime/detail/type.hpp index 8eab57d00..a115e6602 100644 --- a/compiler+runtime/include/cpp/jank/runtime/detail/type.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/detail/type.hpp @@ -8,9 +8,6 @@ #include #include -#include -#include - #include #include @@ -36,15 +33,6 @@ namespace immer jank::memory_policy>; } -namespace bpptree::detail -{ - extern template struct BppTreeSet; - extern template struct BppTreeMap; -} - namespace jank::runtime::detail { using native_persistent_vector = immer::vector; @@ -54,11 +42,10 @@ namespace jank::runtime::detail set, std::equal_to, memory_policy>; using native_transient_hash_set = native_persistent_hash_set::transient_type; - /* TODO: These BppTree types will leak until we get them GC allocated. */ + /* TODO: Bring in proper immutable sorted maps/sets. */ using native_persistent_sorted_set - = bpptree::BppTreeSet::Persistent; - using native_transient_sorted_set - = bpptree::BppTreeSet::Transient; + = std::set>; + using native_transient_sorted_set = native_persistent_sorted_set; using native_persistent_hash_map = immer::map::Persistent; - using native_transient_sorted_map - = bpptree::BppTreeMap::Transient; + = std::map>>; + using native_transient_sorted_map = native_persistent_sorted_map; /* If an object requires this in its constructor, use your runtime context to intern * it instead. */ diff --git a/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_map.hpp b/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_map.hpp index 97e3fdcc1..5b77e1145 100644 --- a/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_map.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_map.hpp @@ -18,8 +18,7 @@ namespace jank::runtime::obj transient_sorted_map() = default; transient_sorted_map(transient_sorted_map &&) noexcept = default; transient_sorted_map(transient_sorted_map const &) = default; - transient_sorted_map(runtime::detail::native_persistent_sorted_map const &d); - transient_sorted_map(runtime::detail::native_persistent_sorted_map &&d); + transient_sorted_map(value_type const &d); transient_sorted_map(value_type &&d); static transient_sorted_map_ref empty(); diff --git a/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_set.hpp b/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_set.hpp index e2d679eaf..1f05fab0b 100644 --- a/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_set.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/obj/transient_sorted_set.hpp @@ -18,8 +18,7 @@ namespace jank::runtime::obj transient_sorted_set() = default; transient_sorted_set(transient_sorted_set &&) noexcept = default; transient_sorted_set(transient_sorted_set const &) = default; - transient_sorted_set(runtime::detail::native_persistent_sorted_set const &d); - transient_sorted_set(runtime::detail::native_persistent_sorted_set &&d); + transient_sorted_set(value_type const &d); transient_sorted_set(value_type &&d); static transient_sorted_set_ref empty(); diff --git a/compiler+runtime/include/cpp/jank/type.hpp b/compiler+runtime/include/cpp/jank/type.hpp index dbd3eef3a..0fb35c802 100644 --- a/compiler+runtime/include/cpp/jank/type.hpp +++ b/compiler+runtime/include/cpp/jank/type.hpp @@ -43,7 +43,7 @@ namespace jank template using native_list = std::list>; template - using native_map = std::map>>; + using native_map = std::map, native_allocator>>; template using native_set = std::set, native_allocator>; diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index aba6ec9af..95ddfe3ff 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -2884,27 +2884,12 @@ namespace jank::analyze /* TODO: Detect literal and act accordingly. */ return visit_map_like( [&](auto const typed_o) -> processor::expression_result { - using T = typename decltype(typed_o)::value_type; - native_vector> exprs; exprs.reserve(typed_o->data.size()); for(auto const &kv : typed_o->data) { - /* The two maps (hash and sorted) have slightly different iterators, so we need to - * pull out the entries differently. */ - object_ref first{}, second{}; - if constexpr(std::same_as) - { - auto const &entry(kv.get()); - first = entry.first; - second = entry.second; - } - else - { - first = kv.first; - second = kv.second; - } + object_ref const first{ kv.first }, second{ kv.second }; auto k_expr(analyze(first, current_frame, expression_position::value, fn_ctx, true)); if(k_expr.is_err()) diff --git a/compiler+runtime/src/cpp/jank/runtime/detail/type.cpp b/compiler+runtime/src/cpp/jank/runtime/detail/type.cpp index cd8a24c82..b8c4abafd 100644 --- a/compiler+runtime/src/cpp/jank/runtime/detail/type.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/detail/type.cpp @@ -22,11 +22,3 @@ namespace immer std::equal_to, jank::memory_policy>; } - -namespace bpptree::detail -{ - template struct BppTreeSet; - template struct BppTreeMap; -} diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_map.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_map.cpp index 23fbbea2d..212a26131 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_map.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_map.cpp @@ -56,9 +56,9 @@ namespace jank::runtime::obj typed_seq->to_string()) }; } auto const val(*it); - transient.insert_or_assign(key, val); + transient[key] = val; } - return transient.persistent(); + return transient; } else { @@ -100,19 +100,21 @@ namespace jank::runtime::obj bool persistent_sorted_map::contains(object_ref const key) const { - return data.find(key) != data.end(); + return data.contains(key); } persistent_sorted_map_ref persistent_sorted_map::assoc(object_ref const key, object_ref const val) const { - auto copy(data.insert_or_assign(key, val)); + auto copy(data); + copy[key] = val; return make_box(meta, std::move(copy)); } persistent_sorted_map_ref persistent_sorted_map::dissoc(object_ref const key) const { - auto copy(data.erase_key(key)); + auto copy(data); + copy.erase(key); return make_box(meta, std::move(copy)); } diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_set.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_set.cpp index 2516fc267..283d1bef3 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_set.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/persistent_sorted_set.cpp @@ -41,9 +41,9 @@ namespace jank::runtime::obj runtime::detail::native_transient_sorted_set transient; for(auto const e : make_sequence_range(typed_seq)) { - transient.insert_v(e); + transient.insert(e); } - return transient.persistent(); + return transient; }, seq)); } @@ -130,8 +130,9 @@ namespace jank::runtime::obj persistent_sorted_set_ref persistent_sorted_set::conj(object_ref const head) const { - auto set(data.insert_v(head)); - auto ret(make_box(meta, std::move(set))); + auto copy(data); + copy.insert(head); + auto ret(make_box(meta, std::move(copy))); return ret; } @@ -140,7 +141,7 @@ namespace jank::runtime::obj auto const found(data.find(o)); if(found != data.end()) { - return found.get(); + return *found; } return jank_nil; } @@ -152,13 +153,14 @@ namespace jank::runtime::obj bool persistent_sorted_set::contains(object_ref const o) const { - return data.find(o) != data.end(); + return data.contains(o); } persistent_sorted_set_ref persistent_sorted_set::disj(object_ref const o) const { - auto set(data.erase_key(o)); - auto ret(make_box(meta, std::move(set))); + auto copy(data); + copy.erase(o); + auto ret(make_box(meta, std::move(copy))); return ret; } } diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp index 7fbb68553..0629d1c22 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp @@ -9,17 +9,12 @@ namespace jank::runtime::obj { - transient_sorted_map::transient_sorted_map(runtime::detail::native_persistent_sorted_map &&d) - : data{ std::move(d).transient() } - { - } - transient_sorted_map::transient_sorted_map(runtime::detail::native_persistent_sorted_map const &d) - : data{ d.transient() } + : data{ d } { } - transient_sorted_map::transient_sorted_map(runtime::detail::native_transient_sorted_map &&d) + transient_sorted_map::transient_sorted_map(runtime::detail::native_persistent_sorted_map &&d) : data{ std::move(d) } { } @@ -107,14 +102,14 @@ namespace jank::runtime::obj transient_sorted_map::assoc_in_place(object_ref const key, object_ref const val) { assert_active(); - data.insert_or_assign(key, val); + data[key] = val; return this; } transient_sorted_map_ref transient_sorted_map::dissoc_in_place(object_ref const key) { assert_active(); - data.erase_key(key); + data.erase(key); return this; } @@ -151,7 +146,7 @@ namespace jank::runtime::obj { assert_active(); active = false; - return make_box(std::move(data).persistent()); + return make_box(std::move(data)); } object_ref transient_sorted_map::call(object_ref const o) const diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp index 0e92a615a..1aa04f45b 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp @@ -6,17 +6,12 @@ namespace jank::runtime::obj { - transient_sorted_set::transient_sorted_set(runtime::detail::native_persistent_sorted_set &&d) - : data{ std::move(d).transient() } - { - } - transient_sorted_set::transient_sorted_set(runtime::detail::native_persistent_sorted_set const &d) - : data{ d.transient() } + : data{ d } { } - transient_sorted_set::transient_sorted_set(runtime::detail::native_transient_sorted_set &&d) + transient_sorted_set::transient_sorted_set(runtime::detail::native_persistent_sorted_set &&d) : data{ std::move(d) } { } @@ -64,7 +59,7 @@ namespace jank::runtime::obj transient_sorted_set_ref transient_sorted_set::conj_in_place(object_ref const elem) { assert_active(); - data.insert_v(elem); + data.insert(elem); return this; } @@ -72,7 +67,7 @@ namespace jank::runtime::obj { assert_active(); active = false; - return make_box(data.persistent()); + return make_box(data); } object_ref transient_sorted_set::call(object_ref const elem) @@ -81,7 +76,7 @@ namespace jank::runtime::obj auto const found(data.find(elem)); if(found != data.end()) { - return found.get(); + return *found; } return jank_nil; } @@ -92,7 +87,7 @@ namespace jank::runtime::obj auto const found(data.find(elem)); if(found != data.end()) { - return found.get(); + return *found; } return fallback; } @@ -109,11 +104,10 @@ namespace jank::runtime::obj object_ref transient_sorted_set::get_entry(object_ref const elem) { - auto const found = call(elem); - auto const nil(jank_nil); - if(found == nil) + auto const found{ call(elem) }; + if(found == jank_nil) { - return nil; + return found; } return make_box(std::in_place, found, found); @@ -128,7 +122,7 @@ namespace jank::runtime::obj transient_sorted_set_ref transient_sorted_set::disjoin_in_place(object_ref const elem) { assert_active(); - data.erase_key(elem); + data.erase(elem); return this; } diff --git a/compiler+runtime/third-party/bpptree b/compiler+runtime/third-party/bpptree deleted file mode 160000 index addf92b02..000000000 --- a/compiler+runtime/third-party/bpptree +++ /dev/null @@ -1 +0,0 @@ -Subproject commit addf92b029bef74ae07f4ebb7e82312a6950ae8c From c561a8533307ff51d4828ccc529064a0e8207180 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 5 Nov 2025 14:39:31 -0800 Subject: [PATCH 07/58] Remove direct std::set usages --- compiler+runtime/src/cpp/jank/ui/highlight.cpp | 2 +- compiler+runtime/src/cpp/jank/util/try.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/ui/highlight.cpp b/compiler+runtime/src/cpp/jank/ui/highlight.cpp index 7c70d3812..b22afcf11 100644 --- a/compiler+runtime/src/cpp/jank/ui/highlight.cpp +++ b/compiler+runtime/src/cpp/jank/ui/highlight.cpp @@ -10,7 +10,7 @@ namespace jank::ui using namespace ftxui; /* TODO: Also support core fns? */ - static std::set const specials{ + static native_set const specials{ "def", "fn*", "fn", "let*", "let", "loop*", "loop", "do", "if", "quote", "var", "try", "catch", "finally", "throw", "letfn*", }; diff --git a/compiler+runtime/src/cpp/jank/util/try.cpp b/compiler+runtime/src/cpp/jank/util/try.cpp index 0a4430917..c50e77d98 100644 --- a/compiler+runtime/src/cpp/jank/util/try.cpp +++ b/compiler+runtime/src/cpp/jank/util/try.cpp @@ -47,7 +47,7 @@ namespace jank::util * at the start/end of each stack trace. */ static bool filter_frame(cpptrace::stacktrace_frame const &frame) { - static std::set const symbols_to_ignore{ + static native_set const symbols_to_ignore{ /* (Top) Linux exception pipework. */ "get_adjusted_ptr", "__gxx_personality_v0", From c7e6d39d7a4900def1fc5597bc29fdf9dc9ad7f7 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 6 Nov 2025 12:46:49 -0800 Subject: [PATCH 08/58] Hack in some cpp gen fixes --- .../jank/runtime/obj/persistent_hash_map.hpp | 6 +- .../include/cpp/jank/runtime/oref.hpp | 9 +- .../src/cpp/jank/analyze/processor.cpp | 2 +- .../src/cpp/jank/codegen/processor.cpp | 116 +++++++++++++----- .../jank/runtime/obj/persistent_hash_map.cpp | 6 + 5 files changed, 103 insertions(+), 36 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/runtime/obj/persistent_hash_map.hpp b/compiler+runtime/include/cpp/jank/runtime/obj/persistent_hash_map.hpp index 770ac4542..b511a8ecd 100644 --- a/compiler+runtime/include/cpp/jank/runtime/obj/persistent_hash_map.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/obj/persistent_hash_map.hpp @@ -48,11 +48,7 @@ namespace jank::runtime::obj this->meta = meta; } - static persistent_hash_map_ref empty() - { - static auto const ret(make_box()); - return ret; - } + static persistent_hash_map_ref empty(); using base_persistent_map::base_persistent_map; diff --git a/compiler+runtime/include/cpp/jank/runtime/oref.hpp b/compiler+runtime/include/cpp/jank/runtime/oref.hpp index 5c1f24e9e..9b09341c5 100644 --- a/compiler+runtime/include/cpp/jank/runtime/oref.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/oref.hpp @@ -100,6 +100,14 @@ namespace jank::runtime constexpr oref &operator=(oref const &rhs) noexcept = default; constexpr oref &operator=(oref &&rhs) noexcept = default; + template + requires behavior::object_like + constexpr oref &operator=(oref const &rhs) noexcept + { + data = &rhs->base; + return *this; + } + constexpr bool operator==(oref const &rhs) const noexcept { return data == rhs.data; @@ -499,5 +507,4 @@ namespace jank::runtime { return static_cast(ptr.data); } - } diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index b219a91ee..d47c5484b 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -3597,7 +3597,7 @@ namespace jank::analyze for(usize i{}; i < arg_count; ++i, it = it.rest()) { auto arg_expr{ - analyze(it.first().unwrap(), current_frame, expression_position::value, fn_ctx, needs_box) + analyze(it.first().unwrap(), current_frame, expression_position::value, fn_ctx, true) }; if(arg_expr.is_err()) { diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 6fba00c93..c01fd29e1 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -347,7 +347,7 @@ namespace jank::codegen static jtl::immutable_string boxed_local_name(jtl::immutable_string const &local_name) { - return local_name + "__boxed"; + return local_name; // + "__boxed"; } } @@ -399,20 +399,21 @@ namespace jank::codegen } } - jtl::immutable_string handle::str(bool const needs_box) const + jtl::immutable_string handle::str([[maybe_unused]] bool const needs_box) const { - if(needs_box) - { - if(boxed_name.empty()) - { - throw std::runtime_error{ util::format("Missing boxed name for handle {}", unboxed_name) }; - } - return boxed_name; - } - else - { - return unboxed_name; - } + return boxed_name; + //if(needs_box) + //{ + // if(boxed_name.empty()) + // { + // throw std::runtime_error{ util::format("Missing boxed name for handle {}", unboxed_name) }; + // } + // return boxed_name; + //} + //else + //{ + // return unboxed_name; + //} } processor::processor(analyze::expr::function_ref const expr, @@ -1296,12 +1297,40 @@ namespace jank::codegen } auto arg_tmp_it(arg_tmps.begin()); - for(auto const ¶m : fn_arity.params) + if(expr->loop_target.is_some()) { - util::format_to(body_buffer, "{} = {};", runtime::munge(param->name), arg_tmp_it->str(true)); - ++arg_tmp_it; + auto const let{ expr->loop_target.unwrap() }; + for(usize i{}; i < expr->arg_exprs.size(); ++i) + { + auto const &pair{ let->pairs[i] }; + auto const local(expr->frame->find_local_or_capture(pair.first)); + if(local.is_none()) + { + throw std::runtime_error{ util::format("ICE: unable to find local: {}", + pair.first->to_string()) }; + } + + auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); + + util::format_to(body_buffer, "{} = {};", munged_name, arg_tmp_it->str(true)); + ++arg_tmp_it; + } + + util::format_to(body_buffer, "continue;"); + } + else + { + for(auto const ¶m : fn_arity.params) + { + util::format_to(body_buffer, + "{} = {};", + runtime::munge(param->name), + arg_tmp_it->str(true)); + ++arg_tmp_it; + } + util::format_to(body_buffer, "continue;"); } - util::format_to(body_buffer, "continue;"); + return none; } @@ -1398,21 +1427,45 @@ namespace jank::codegen auto const &val_tmp(gen(pair.second, fn_arity, pair.second->needs_box)); auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); + /* Every binding is wrapped in its own scope, to allow shadowing. * * Also, bindings are references to their value expression, rather than a copy. * This is important for C++ interop, since the we don't want to, and we may not - * be able to, just copy stack-allocated C++ objects around willy nillly. */ - util::format_to(body_buffer, "{ auto &&{}({}); ", munged_name, val_tmp.unwrap().str(false)); - - auto const binding(local.unwrap().binding); - if(!binding->needs_box && binding->has_boxed_usage) + * be able to, just copy stack-allocated C++ objects around willy nilly. */ + if(expr->is_loop) { - util::format_to(body_buffer, - "auto const {}({});", - detail::boxed_local_name(munged_name), - val_tmp.unwrap().str(true)); + auto const local_type{ cpp_util::expression_type(pair.second) }; + if(cpp_util::is_any_object(local_type)) + { + util::format_to(body_buffer, + "{ object_ref {}({}); ", + munged_name, + val_tmp.unwrap().str(true)); + } + else + { + util::format_to(body_buffer, "{ auto {}({}); ", munged_name, val_tmp.unwrap().str(true)); + } } + else + { + util::format_to(body_buffer, "{ auto &&{}({}); ", munged_name, val_tmp.unwrap().str(false)); + } + + //auto const binding(local.unwrap().binding); + //if(!binding->needs_box && binding->has_boxed_usage) + //{ + // util::format_to(body_buffer, + // "auto const {}({});", + // detail::boxed_local_name(munged_name), + // val_tmp.unwrap().str(true)); + //} + } + + if(expr->is_loop) + { + util::format_to(body_buffer, "while(true){"); } for(auto it(expr->body->values.begin()); it != expr->body->values.end();) @@ -1442,6 +1495,11 @@ namespace jank::codegen util::format_to(body_buffer, "}"); } + if(expr->is_loop) + { + util::format_to(body_buffer, "}"); + } + if(expr->needs_box) { util::format_to(body_buffer, "}"); @@ -1727,7 +1785,7 @@ namespace jank::codegen arg_tmps.reserve(expr->arg_exprs.size()); for(auto const &arg_expr : expr->arg_exprs) { - arg_tmps.emplace_back(gen(arg_expr, arity, false).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, arity, true).unwrap()); } util::format_to(body_buffer, @@ -1742,7 +1800,7 @@ namespace jank::codegen { util::format_to(body_buffer, ", "); } - util::format_to(body_buffer, "{}", arg_tmp.str(false)); + util::format_to(body_buffer, "{}", arg_tmp.str(true)); need_comma = true; } diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/persistent_hash_map.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/persistent_hash_map.cpp index 406a1deec..793e95923 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/persistent_hash_map.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/persistent_hash_map.cpp @@ -40,6 +40,12 @@ namespace jank::runtime::obj { } + persistent_hash_map_ref persistent_hash_map::empty() + { + static auto const ret(make_box()); + return ret; + } + persistent_hash_map_ref persistent_hash_map::create_from_seq(object_ref const seq) { return make_box(visit_seqable( From 5679cf84761cf8f7562db7c6d0381b815c481db4 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 11 Nov 2025 09:54:50 -0800 Subject: [PATCH 09/58] Just adding everything so I can have it on my mac --- .gitignore | 1 - compiler+runtime/cmake/dependency/bdwgc.cmake | 6 +- compiler+runtime/cmake/summary.cmake | 1 + .../include/cpp/jank/analyze/cpp_util.hpp | 1 + .../include/cpp/jank/codegen/processor.hpp | 103 +-- .../include/cpp/jank/util/cli.hpp | 2 +- .../src/cpp/jank/analyze/cpp_util.cpp | 14 + .../src/cpp/jank/codegen/llvm_processor.cpp | 1 + .../src/cpp/jank/codegen/processor.cpp | 803 +++++------------- compiler+runtime/src/cpp/jank/evaluate.cpp | 2 +- compiler+runtime/src/cpp/jank/util/cli.cpp | 6 +- .../src/cpp/jtl/string_builder.cpp | 9 +- 12 files changed, 271 insertions(+), 678 deletions(-) diff --git a/.gitignore b/.gitignore index 66bce157b..454712591 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ a.out .jank-repl-history .envrc .direnv -/notes # Vim files /.ycm_extra_conf.py* diff --git a/compiler+runtime/cmake/dependency/bdwgc.cmake b/compiler+runtime/cmake/dependency/bdwgc.cmake index a0b897c78..034bab9c6 100644 --- a/compiler+runtime/cmake/dependency/bdwgc.cmake +++ b/compiler+runtime/cmake/dependency/bdwgc.cmake @@ -2,13 +2,14 @@ set(CMAKE_C_FLAGS_OLD "${CMAKE_C_FLAGS}") set(CMAKE_CXX_FLAGS_OLD "${CMAKE_CXX_FLAGS}") set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS}) set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY}) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS") set(BUILD_SHARED_LIBS OFF) set(CMAKE_CXX_CLANG_TIDY "") set(enable_cplusplus ON CACHE BOOL "Enable C++") set(build_cord OFF CACHE BOOL "Build cord") set(enable_docs OFF CACHE BOOL "Enable docs") + set(enable_threads ON CACHE BOOL "Enable multi-threading support") set(enable_large_config ON CACHE BOOL "Optimize for large heap or root set") set(enable_throw_bad_alloc_library ON CACHE BOOL "Enable C++ gctba library build") set(enable_gc_debug OFF CACHE BOOL "Support for pointer back-tracing") @@ -24,6 +25,7 @@ set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY}) unset(enable_cplusplus) unset(build_cord) unset(enable_docs) + unset(enable_threads) unset(enable_large_config) unset(enable_throw_bad_alloc_library) unset(enable_gc_debug) diff --git a/compiler+runtime/cmake/summary.cmake b/compiler+runtime/cmake/summary.cmake index d5000b845..8477b6e01 100644 --- a/compiler+runtime/cmake/summary.cmake +++ b/compiler+runtime/cmake/summary.cmake @@ -11,6 +11,7 @@ jank_message("│ jank analyze : ${jank_analyze}") jank_message("│ jank sanitize : ${jank_sanitize}") jank_message("│ jank unity build : ${jank_unity_build}") jank_message("│ jank resource dir : ${jank_resource_dir}") +jank_message("│ jank debug gc : ${jank_debug_gc}") jank_message("│ clang version : ${LLVM_PACKAGE_VERSION}") jank_message("│ clang prefix : ${CLANG_INSTALL_PREFIX}") jank_message("│ clang resource dir : ${clang_resource_dir}") diff --git a/compiler+runtime/include/cpp/jank/analyze/cpp_util.hpp b/compiler+runtime/include/cpp/jank/analyze/cpp_util.hpp index 8cd03e655..9d8ec82ac 100644 --- a/compiler+runtime/include/cpp/jank/analyze/cpp_util.hpp +++ b/compiler+runtime/include/cpp/jank/analyze/cpp_util.hpp @@ -28,6 +28,7 @@ namespace jank::analyze::cpp_util native_vector> find_adl_scopes(native_vector> const &starters); jtl::immutable_string get_qualified_name(jtl::ptr scope); + jtl::immutable_string get_qualified_type_name(jtl::ptr type); void register_rtti(jtl::ptr type); jtl::ptr expression_type(expression_ref expr); diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index 048215b59..f3f73d4eb 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -92,104 +92,69 @@ namespace jank::codegen processor(processor const &) = delete; processor(processor &&) noexcept = delete; + jtl::option gen(analyze::expression_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::def_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expression_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::var_deref_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::def_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::var_ref_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::call_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::var_deref_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::primitive_literal_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::vector_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::map_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::set_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::var_ref_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::local_reference_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::call_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option gen(analyze::expr::primitive_literal_ref const, - analyze::expr::function_arity const &, - bool box_needed); + gen(analyze::expr::function_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::recur_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::vector_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::recursion_reference_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::map_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::named_recursion_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::let_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::letfn_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::do_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::if_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::throw_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::try_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::case_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::set_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option gen(analyze::expr::local_reference_ref const, - analyze::expr::function_arity const &, - bool box_needed); + gen(analyze::expr::cpp_raw_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::function_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_type_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::recur_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option gen(analyze::expr::recursion_reference_ref const, - analyze::expr::function_arity const &, - bool box_needed); - jtl::option gen(analyze::expr::named_recursion_ref const, - analyze::expr::function_arity const &, - bool box_needed); + gen(analyze::expr::cpp_value_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::let_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_cast_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::letfn_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_call_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::do_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_constructor_call_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::if_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_member_call_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::throw_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_member_access_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::try_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_builtin_operator_call_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::case_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_box_ref const, analyze::expr::function_arity const &); jtl::option - gen(analyze::expr::cpp_raw_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option - gen(analyze::expr::cpp_type_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option - gen(analyze::expr::cpp_value_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option - gen(analyze::expr::cpp_cast_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option - gen(analyze::expr::cpp_call_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option gen(analyze::expr::cpp_constructor_call_ref const, - analyze::expr::function_arity const &, - bool box_needed); - jtl::option gen(analyze::expr::cpp_member_call_ref const, - analyze::expr::function_arity const &, - bool box_needed); - jtl::option gen(analyze::expr::cpp_member_access_ref const, - analyze::expr::function_arity const &, - bool box_needed); - jtl::option gen(analyze::expr::cpp_builtin_operator_call_ref const, - analyze::expr::function_arity const &, - bool box_needed); - jtl::option - gen(analyze::expr::cpp_box_ref const, analyze::expr::function_arity const &, bool box_needed); - jtl::option - gen(analyze::expr::cpp_unbox_ref const, analyze::expr::function_arity const &, bool box_needed); + gen(analyze::expr::cpp_unbox_ref const, analyze::expr::function_arity const &); jtl::immutable_string declaration_str(); void build_header(); void build_body(); void build_footer(); - jtl::immutable_string expression_str(bool box_needed); + jtl::immutable_string expression_str(); jtl::immutable_string module_init_str(jtl::immutable_string const &module); - void format_elided_var(jtl::immutable_string const &start, - jtl::immutable_string const &end, - jtl::immutable_string const &ret_tmp, - native_vector const &arg_exprs, - analyze::expr::function_arity const &fn_arity, - bool arg_box_needed, - bool ret_box_needed); - void format_direct_call(jtl::immutable_string const &source_tmp, - jtl::immutable_string const &ret_tmp, - native_vector const &arg_exprs, - analyze::expr::function_arity const &fn_arity, - bool arg_box_needed); void format_dynamic_call(jtl::immutable_string const &source_tmp, jtl::immutable_string const &ret_tmp, native_vector const &arg_exprs, - analyze::expr::function_arity const &fn_arity, - bool arg_box_needed); + analyze::expr::function_arity const &fn_arity); analyze::expr::function_ref root_fn; jtl::immutable_string module; diff --git a/compiler+runtime/include/cpp/jank/util/cli.hpp b/compiler+runtime/include/cpp/jank/util/cli.hpp index 8e792b4eb..defa3e09a 100644 --- a/compiler+runtime/include/cpp/jank/util/cli.hpp +++ b/compiler+runtime/include/cpp/jank/util/cli.hpp @@ -29,7 +29,7 @@ namespace jank::util::cli bool profiler_enabled{}; bool perf_profiling_enabled{}; bool gc_incremental{}; - codegen_type codegen{ codegen_type::llvm_ir }; + codegen_type codegen{ codegen_type::cpp }; /* Native dependencies. */ native_vector include_dirs; diff --git a/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp b/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp index aac12ebaa..fc08303c9 100644 --- a/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp @@ -292,6 +292,20 @@ namespace jank::analyze::cpp_util return res; } + jtl::immutable_string get_qualified_type_name(jtl::ptr const type) + { + if(auto const scope{ Cpp::GetScopeFromType(type) }; scope) + { + auto name{ get_qualified_name(scope) }; + if(Cpp::IsPointerType(type)) + { + name = name + "*"; + } + return name; + } + return Cpp::GetTypeAsString(type); + } + /* This is a quick and dirty helper to get the RTTI for a given QualType. We need * this for exception catching. */ void register_rtti(jtl::ptr const type) diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index d60e82064..4266f59c2 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -729,6 +729,7 @@ namespace jank::codegen jtl::string_result llvm_processor::impl::gen() { + throw "no IR gen allowed!"; profile::timer const timer{ "ir gen" }; if(target != compilation_target::function) { diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index c01fd29e1..d7244e8a7 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -216,6 +216,12 @@ namespace jank::codegen typed_o->sym->ns, typed_o->sym->name); } + else if constexpr(std::same_as) + { + util::format_to(buffer, + R"(jank::runtime::make_box({}))", + typed_o->to_code_string().substr(1)); + } else if constexpr(std::same_as) { util::format_to(buffer, @@ -427,18 +433,16 @@ namespace jank::codegen assert(root_fn->frame.data); } - jtl::option processor::gen(analyze::expression_ref const ex, - analyze::expr::function_arity const &fn_arity, - bool const box_needed) + jtl::option + processor::gen(analyze::expression_ref const ex, analyze::expr::function_arity const &fn_arity) { jtl::option ret; - visit_expr([&, this](auto const typed_ex) { ret = gen(typed_ex, fn_arity, box_needed); }, ex); + visit_expr([&, this](auto const typed_ex) { ret = gen(typed_ex, fn_arity); }, ex); return ret; } - jtl::option processor::gen(analyze::expr::def_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::def_ref const expr, analyze::expr::function_arity const &fn_arity) { auto const &var(expr->frame->find_lifted_var(expr->name).unwrap().get()); auto const &munged_name(runtime::munge(var.native_name)); @@ -469,7 +473,7 @@ namespace jank::codegen } } - auto const val(gen(expr->value.unwrap(), fn_arity, true).unwrap()); + auto const val(gen(expr->value.unwrap(), fn_arity).unwrap()); switch(expr->position) { case analyze::expression_position::value: @@ -521,9 +525,8 @@ namespace jank::codegen } } - jtl::option processor::gen(analyze::expr::var_deref_ref const expr, - analyze::expr::function_arity const &, - bool const) + jtl::option + processor::gen(analyze::expr::var_deref_ref const expr, analyze::expr::function_arity const &) { auto const &var(expr->frame->find_lifted_var(expr->qualified_name).unwrap().get()); switch(expr->position) @@ -541,9 +544,8 @@ namespace jank::codegen } } - jtl::option processor::gen(analyze::expr::var_ref_ref const expr, - analyze::expr::function_arity const &, - bool const) + jtl::option + processor::gen(analyze::expr::var_ref_ref const expr, analyze::expr::function_arity const &) { auto const &var(expr->frame->find_lifted_var(expr->qualified_name).unwrap().get()); switch(expr->position) @@ -561,74 +563,10 @@ namespace jank::codegen } } - void processor::format_elided_var(jtl::immutable_string const &start, - jtl::immutable_string const &end, - jtl::immutable_string const &ret_tmp, - native_vector const &arg_exprs, - analyze::expr::function_arity const &fn_arity, - bool const arg_box_needed, - bool const ret_box_needed) - { - /* TODO: Assert arg count when we know it. */ - native_vector arg_tmps; - arg_tmps.reserve(arg_exprs.size()); - for(auto const &arg_expr : arg_exprs) - { - arg_tmps.emplace_back(gen(arg_expr, fn_arity, arg_box_needed).unwrap()); - } - - jtl::immutable_string ret_box; - if(ret_box_needed) - { - ret_box = "jank::runtime::make_box("; - } - util::format_to(body_buffer, "auto const {}({}{}", ret_tmp, ret_box, start); - bool need_comma{}; - for(size_t i{}; i < runtime::max_params && i < arg_tmps.size(); ++i) - { - if(need_comma) - { - util::format_to(body_buffer, ", "); - } - util::format_to(body_buffer, "{}", arg_tmps[i].str(arg_box_needed)); - need_comma = true; - } - util::format_to(body_buffer, "{}{});", end, (ret_box_needed ? ")" : "")); - } - - void processor::format_direct_call(jtl::immutable_string const &source_tmp, - jtl::immutable_string const &ret_tmp, - native_vector const &arg_exprs, - analyze::expr::function_arity const &fn_arity, - bool const arg_box_needed) - { - native_vector arg_tmps; - arg_tmps.reserve(arg_exprs.size()); - for(auto const &arg_expr : arg_exprs) - { - arg_tmps.emplace_back(gen(arg_expr, fn_arity, arg_box_needed).unwrap()); - } - - util::format_to(body_buffer, "auto const {}({}.call(", ret_tmp, source_tmp); - - bool need_comma{}; - for(size_t i{}; i < runtime::max_params && i < arg_tmps.size(); ++i) - { - if(need_comma) - { - util::format_to(body_buffer, ", "); - } - util::format_to(body_buffer, "{}", arg_tmps[i].str(true)); - need_comma = true; - } - util::format_to(body_buffer, "));"); - } - void processor::format_dynamic_call(jtl::immutable_string const &source_tmp, jtl::immutable_string const &ret_tmp, native_vector const &arg_exprs, - analyze::expr::function_arity const &fn_arity, - bool const arg_box_needed) + analyze::expr::function_arity const &fn_arity) { //util::println("format_dynamic_call source {}", source_tmp); native_vector arg_tmps; @@ -637,7 +575,7 @@ namespace jank::codegen { //util::println("\tformat_dynamic_call arg {}", // runtime::to_code_string(arg_expr->to_runtime_data())); - arg_tmps.emplace_back(gen(arg_expr, fn_arity, arg_box_needed).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, fn_arity).unwrap()); } util::format_to(body_buffer, @@ -663,374 +601,18 @@ namespace jank::codegen util::format_to(body_buffer, "));"); } - jtl::option processor::gen(analyze::expr::call_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const box_needed) + jtl::option + processor::gen(analyze::expr::call_ref const expr, analyze::expr::function_arity const &fn_arity) { - /* TODO: Doesn't take into account boxing. */ handle ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("call")) }; - /* Clojure's codegen actually skips vars for certain calls to clojure.core - * fns; this is not the same as direct linking, which uses `invokeStatic` - * instead. Rather, this makes calls to `get` become `RT.get`, calls to `+` become - * `Numbers.add`, and so on. We do the same thing here. */ - bool elided{}; - /* TODO: Use the actual var meta to do this, not a hard-coded set of if checks. */ - if(auto const * const ref = dynamic_cast(expr->source_expr.data)) - { - auto const &name{ ref->var->name->name }; - if(ref->var->n->name->name != "clojure.core") - { - } - else if(name == "get") - { - format_elided_var("jank::runtime::get(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - else if(expr->arg_exprs.empty()) - { - if(name == "rand") - { - format_elided_var("jank::runtime::rand(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - } - else if(expr->arg_exprs.size() == 1) - { - //if(name == "print") - //{ - // format_elided_var("jank::runtime::print(", - // ")", - // ret_tmp.str(false), - // expr->arg_exprs, - // fn_arity, - // true, - // false); - // elided = true; - //} - if(name == "abs") - { - format_elided_var("jank::runtime::abs(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "sqrt") - { - format_elided_var("jank::runtime::sqrt(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "int") - { - format_elided_var("jank::runtime::to_int(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "seq") - { - format_elided_var("jank::runtime::seq(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - else if(name == "fresh-seq") - { - format_elided_var("jank::runtime::fresh_seq(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - else if(name == "first") - { - format_elided_var("jank::runtime::first(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - else if(name == "next") - { - format_elided_var("jank::runtime::next(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - else if(name == "next-in-place") - { - format_elided_var("jank::runtime::next_in_place(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - else if(name == "nil?") - { - format_elided_var("jank::runtime::is_nil(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - box_needed); - elided = true; - } - else if(name == "some?") - { - format_elided_var("jank::runtime::is_some(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - box_needed); - elided = true; - } - } - else if(expr->arg_exprs.size() == 2) - { - if(name == "+") - { - format_elided_var("jank::runtime::add(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "-") - { - format_elided_var("jank::runtime::sub(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "*") - { - format_elided_var("jank::runtime::mul(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "/") - { - format_elided_var("jank::runtime::div(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "<") - { - format_elided_var("jank::runtime::lt(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "<=") - { - format_elided_var("jank::runtime::lte(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == ">") - { - format_elided_var("jank::runtime::lt(", - ")", - ret_tmp.str(false), - { expr->arg_exprs.rbegin(), expr->arg_exprs.rend() }, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == ">=") - { - format_elided_var("jank::runtime::lte(", - ")", - ret_tmp.str(false), - { expr->arg_exprs.rbegin(), expr->arg_exprs.rend() }, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "min") - { - format_elided_var("jank::runtime::min(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "max") - { - format_elided_var("jank::runtime::max(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "pow") - { - format_elided_var("jank::runtime::pow(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - false, - box_needed); - elided = true; - ret_tmp = { ret_tmp.unboxed_name, box_needed }; - } - else if(name == "conj") - { - format_elided_var("jank::runtime::conj(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - } - else if(expr->arg_exprs.size() == 3) - { - if(name == "assoc") - { - format_elided_var("jank::runtime::assoc(", - ")", - ret_tmp.str(false), - expr->arg_exprs, - fn_arity, - true, - false); - elided = true; - } - } - } - else if(auto const * const fn = dynamic_cast(expr->source_expr.data)) - { - bool variadic{}; - for(auto const &arity : fn->arities) - { - if(arity.fn_ctx->is_variadic) - { - variadic = true; - } - } - if(!variadic) - { - auto const &source_tmp(gen(expr->source_expr, fn_arity, false)); - format_direct_call(source_tmp.unwrap().str(false), - ret_tmp.str(true), - expr->arg_exprs, - fn_arity, - true); - elided = true; - } - } - - if(!elided) - { - auto const &source_tmp(gen(expr->source_expr, fn_arity, false)); - format_dynamic_call(source_tmp.unwrap().str(true), - ret_tmp.str(true), - expr->arg_exprs, - fn_arity, - true); - } + auto const &source_tmp(gen(expr->source_expr, fn_arity)); + format_dynamic_call(source_tmp.unwrap().str(true), + ret_tmp.str(true), + expr->arg_exprs, + fn_arity); if(expr->position == analyze::expression_position::tail) { - /* TODO: Box here, not in the calls above. Using false when we mean true is not good. */ - /* No need for extra boxing on this, since the boxing was done on the call above. */ util::format_to(body_buffer, "return {};", ret_tmp.str(false)); return none; } @@ -1039,8 +621,7 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::primitive_literal_ref const expr, - analyze::expr::function_arity const &, - bool const) + analyze::expr::function_arity const &) { auto const &constant(expr->frame->find_lifted_constant(expr->data).unwrap().get()); @@ -1067,14 +648,13 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::vector_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + analyze::expr::function_arity const &fn_arity) { native_vector data_tmps; data_tmps.reserve(expr->data_exprs.size()); for(auto const &data_expr : expr->data_exprs) { - data_tmps.emplace_back(gen(data_expr, fn_arity, true).unwrap()); + data_tmps.emplace_back(gen(data_expr, fn_arity).unwrap()); } auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("vec"))); @@ -1103,16 +683,15 @@ namespace jank::codegen return ret_tmp; } - jtl::option processor::gen(analyze::expr::map_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::map_ref const expr, analyze::expr::function_arity const &fn_arity) { native_vector> data_tmps; data_tmps.reserve(expr->data_exprs.size()); for(auto const &data_expr : expr->data_exprs) { - data_tmps.emplace_back(gen(data_expr.first, fn_arity, true).unwrap(), - gen(data_expr.second, fn_arity, true).unwrap()); + data_tmps.emplace_back(gen(data_expr.first, fn_arity).unwrap(), + gen(data_expr.second, fn_arity).unwrap()); } auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("map"))); @@ -1183,15 +762,14 @@ namespace jank::codegen return ret_tmp; } - jtl::option processor::gen(analyze::expr::set_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::set_ref const expr, analyze::expr::function_arity const &fn_arity) { native_vector data_tmps; data_tmps.reserve(expr->data_exprs.size()); for(auto const &data_expr : expr->data_exprs) { - data_tmps.emplace_back(gen(data_expr, fn_arity, true).unwrap()); + data_tmps.emplace_back(gen(data_expr, fn_arity).unwrap()); } auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("set"))); @@ -1222,8 +800,7 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::local_reference_ref const expr, - analyze::expr::function_arity const &, - bool const) + analyze::expr::function_arity const &) { auto const munged_name(runtime::munge(expr->binding->native_name)); @@ -1252,22 +829,16 @@ namespace jank::codegen } } - jtl::option processor::gen(analyze::expr::function_ref const expr, - analyze::expr::function_arity const &, - bool const box_needed) + jtl::option + processor::gen(analyze::expr::function_ref const expr, analyze::expr::function_arity const &) { auto const compiling(truthy(__rt_ctx->compile_files_var->deref())); /* Since each codegen proc handles one callable struct, we create a new one for this fn. */ processor prc{ expr, module, - //runtime::module::nest_module(module, runtime::munge(expr->unique_name)), compiling ? compilation_target::function : compilation_target::eval }; - /* If we're compiling, we'll create a separate file for this. */ - //if(target != compilation_target::module) - { - util::format_to(deps_buffer, "{}", prc.declaration_str()); - } + util::format_to(deps_buffer, "{}", prc.declaration_str()); switch(expr->position) { @@ -1275,25 +846,24 @@ namespace jank::codegen case analyze::expression_position::value: /* TODO: Return a handle. */ { - return prc.expression_str(box_needed); + return prc.expression_str(); } case analyze::expression_position::tail: { - util::format_to(body_buffer, "return {};", prc.expression_str(box_needed)); + util::format_to(body_buffer, "return {};", prc.expression_str()); return none; } } } - jtl::option processor::gen(analyze::expr::recur_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::recur_ref const expr, analyze::expr::function_arity const &fn_arity) { native_vector arg_tmps; arg_tmps.reserve(expr->arg_exprs.size()); for(auto const &arg_expr : expr->arg_exprs) { - arg_tmps.emplace_back(gen(arg_expr, fn_arity, true).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, fn_arity).unwrap()); } auto arg_tmp_it(arg_tmps.begin()); @@ -1310,9 +880,13 @@ namespace jank::codegen pair.first->to_string()) }; } - auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); + auto const &local_name(runtime::munge(local.unwrap().binding->native_name)); + auto const &val_name(arg_tmp_it->str(true)); - util::format_to(body_buffer, "{} = {};", munged_name, arg_tmp_it->str(true)); + if(local_name != val_name) + { + util::format_to(body_buffer, "{} = {};", local_name, val_name); + } ++arg_tmp_it; } @@ -1336,8 +910,7 @@ namespace jank::codegen /* NOLINTNEXTLINE(readability-make-member-function-const): Can't be const, due to overload resolution. */ jtl::option processor::gen(analyze::expr::recursion_reference_ref const expr, - analyze::expr::function_arity const &, - bool const) + analyze::expr::function_arity const &) { if(expr->position == analyze::expression_position::tail) { @@ -1348,18 +921,16 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::named_recursion_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + analyze::expr::function_arity const &fn_arity) { handle ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("named_recursion")) }; auto const &source_tmp( - gen(jtl::ref{ &expr->recursion_ref }, fn_arity, false)); + gen(jtl::ref{ &expr->recursion_ref }, fn_arity)); format_dynamic_call(source_tmp.unwrap().str(true), ret_tmp.str(true), expr->arg_exprs, - fn_arity, - true); + fn_arity); if(expr->position == analyze::expression_position::tail) { @@ -1370,9 +941,8 @@ namespace jank::codegen return ret_tmp; } - jtl::option processor::gen(analyze::expr::let_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::let_ref const expr, analyze::expr::function_arity const &fn_arity) { handle const ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")), expr->needs_box }; @@ -1425,7 +995,7 @@ namespace jank::codegen pair.first->to_string()) }; } - auto const &val_tmp(gen(pair.second, fn_arity, pair.second->needs_box)); + auto const &val_tmp(gen(pair.second, fn_arity)); auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); /* Every binding is wrapped in its own scope, to allow shadowing. @@ -1470,7 +1040,7 @@ namespace jank::codegen for(auto it(expr->body->values.begin()); it != expr->body->values.end();) { - auto const &val_tmp(gen(*it, fn_arity, true)); + auto const &val_tmp(gen(*it, fn_arity)); /* We ignore all values but the last. */ if(++it == expr->body->values.end() && val_tmp.is_some()) @@ -1482,6 +1052,11 @@ namespace jank::codegen "{} = std::move({});", ret_tmp.str(true), val_tmp.unwrap().str(expr->needs_box)); + + if(expr->is_loop) + { + util::format_to(body_buffer, " break;"); + } } else { @@ -1522,19 +1097,18 @@ namespace jank::codegen } jtl::option - processor::gen(analyze::expr::letfn_ref const, analyze::expr::function_arity const &, bool const) + processor::gen(analyze::expr::letfn_ref const, analyze::expr::function_arity const &) { return none; } - jtl::option processor::gen(analyze::expr::do_ref const expr, - analyze::expr::function_arity const &arity, - bool const) + jtl::option + processor::gen(analyze::expr::do_ref const expr, analyze::expr::function_arity const &arity) { jtl::option last; for(auto const &form : expr->values) { - last = gen(form, arity, true); + last = gen(form, arity); } switch(expr->position) @@ -1559,18 +1133,17 @@ namespace jank::codegen } } - jtl::option processor::gen(analyze::expr::if_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::if_ref const expr, analyze::expr::function_arity const &fn_arity) { /* TODO: Handle unboxed results! */ auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("if"))); util::format_to(body_buffer, "object_ref {}{ };", ret_tmp); - auto const &condition_tmp(gen(expr->condition, fn_arity, false)); + auto const &condition_tmp(gen(expr->condition, fn_arity)); util::format_to(body_buffer, "if(jank::runtime::truthy({})) {", condition_tmp.unwrap().str(false)); - auto const &then_tmp(gen(expr->then, fn_arity, true)); + auto const &then_tmp(gen(expr->then, fn_arity)); if(then_tmp.is_some()) { util::format_to(body_buffer, "{} = {}; }", ret_tmp, then_tmp.unwrap().str(expr->needs_box)); @@ -1583,7 +1156,7 @@ namespace jank::codegen if(expr->else_.is_some()) { util::format_to(body_buffer, "else {"); - auto const &else_tmp(gen(expr->else_.unwrap(), fn_arity, true)); + auto const &else_tmp(gen(expr->else_.unwrap(), fn_arity)); if(else_tmp.is_some()) { util::format_to(body_buffer, "{} = {}; }", ret_tmp, else_tmp.unwrap().str(expr->needs_box)); @@ -1603,11 +1176,10 @@ namespace jank::codegen return ret_tmp; } - jtl::option processor::gen(analyze::expr::throw_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const) + jtl::option + processor::gen(analyze::expr::throw_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &value_tmp(gen(expr->value, fn_arity, true)); + auto const &value_tmp(gen(expr->value, fn_arity)); /* We static_cast to object_ref here, since we'll be trying to catch an object_ref in any * try/catch forms. This loses us our type info, but C++ doesn't do implicit conversions * when catching and we're not using inheritance. */ @@ -1617,9 +1189,8 @@ namespace jank::codegen return none; } - jtl::option processor::gen(analyze::expr::try_ref const expr, - analyze::expr::function_arity const &fn_arity, - bool const box_needed) + jtl::option + processor::gen(analyze::expr::try_ref const expr, analyze::expr::function_arity const &fn_arity) { auto const has_catch{ expr->catch_body.is_some() }; auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("try"))); @@ -1629,17 +1200,17 @@ namespace jank::codegen if(expr->finally_body.is_some()) { util::format_to(body_buffer, "jank::util::scope_exit const finally{ [&](){ "); - gen(expr->finally_body.unwrap(), fn_arity, box_needed); + gen(expr->finally_body.unwrap(), fn_arity); util::format_to(body_buffer, "} };"); } if(has_catch) { util::format_to(body_buffer, "try {"); - auto const &body_tmp(gen(expr->body, fn_arity, box_needed)); + auto const &body_tmp(gen(expr->body, fn_arity)); if(body_tmp.is_some()) { - util::format_to(body_buffer, "{} = {};", ret_tmp, body_tmp.unwrap().str(box_needed)); + util::format_to(body_buffer, "{} = {};", ret_tmp, body_tmp.unwrap().str(true)); } if(expr->position == analyze::expression_position::tail) { @@ -1648,20 +1219,20 @@ namespace jank::codegen util::format_to(body_buffer, "}"); /* There's a gotcha here, tied to how we throw exceptions. We're catching an object_ref, which - * means we need to be throwing an object_ref. Since we're not using inheritance, we can't - * rely on a catch-all and C++ doesn't do implicit conversions into catch types. So, if we - * throw a persistent_string_ref, for example, it will not be caught as an object_ref. - * - * We mitigate this by ensuring during the codegen for throw that we type-erase to - * an object_ref. - */ + * means we need to be throwing an object_ref. Since we're not using inheritance, we can't + * rely on a catch-all and C++ doesn't do implicit conversions into catch types. So, if we + * throw a persistent_string_ref, for example, it will not be caught as an object_ref. + * + * We mitigate this by ensuring during the codegen for throw that we type-erase to + * an object_ref. + */ util::format_to(body_buffer, "catch(jank::runtime::object_ref const {}) {", runtime::munge(expr->catch_body.unwrap().sym->name)); - auto const &catch_tmp(gen(expr->catch_body.unwrap().body, fn_arity, box_needed)); + auto const &catch_tmp(gen(expr->catch_body.unwrap().body, fn_arity)); if(catch_tmp.is_some()) { - util::format_to(body_buffer, "{} = {};", ret_tmp, catch_tmp.unwrap().str(box_needed)); + util::format_to(body_buffer, "{} = {};", ret_tmp, catch_tmp.unwrap().str(true)); } if(expr->position == analyze::expression_position::tail) { @@ -1671,10 +1242,10 @@ namespace jank::codegen } else { - auto const &body_tmp(gen(expr->body, fn_arity, box_needed)); + auto const &body_tmp(gen(expr->body, fn_arity)); if(body_tmp.is_some()) { - util::format_to(body_buffer, "{} = {};", ret_tmp, body_tmp.unwrap().str(box_needed)); + util::format_to(body_buffer, "{} = {};", ret_tmp, body_tmp.unwrap().str(true)); } if(expr->position == analyze::expression_position::tail) { @@ -1688,15 +1259,14 @@ namespace jank::codegen } jtl::option - processor::gen(analyze::expr::case_ref const, analyze::expr::function_arity const &, bool) + processor::gen(analyze::expr::case_ref const, analyze::expr::function_arity const &) { return none; } - jtl::option - processor::gen(expr::cpp_raw_ref const expr, expr::function_arity const &, bool) + jtl::option processor::gen(expr::cpp_raw_ref const expr, expr::function_arity const &) { - util::format_to(deps_buffer, "{}", expr->code); + util::format_to(deps_buffer, "{}\n", expr->code); if(expr->position == analyze::expression_position::tail) { @@ -1707,14 +1277,13 @@ namespace jank::codegen } jtl::option - processor::gen(analyze::expr::cpp_type_ref const, analyze::expr::function_arity const &, bool) + processor::gen(analyze::expr::cpp_type_ref const, analyze::expr::function_arity const &) { throw std::runtime_error{ "cpp_type has no codegen" }; } - jtl::option processor::gen(analyze::expr::cpp_value_ref const expr, - analyze::expr::function_arity const &, - bool) + jtl::option + processor::gen(analyze::expr::cpp_value_ref const expr, analyze::expr::function_arity const &) { if(expr->val_kind == expr::cpp_value::value_kind::null) { @@ -1748,20 +1317,25 @@ namespace jank::codegen return tmp; } - jtl::option processor::gen(analyze::expr::cpp_cast_ref const expr, - analyze::expr::function_arity const &arity, - bool const box_needed) + jtl::option + processor::gen(analyze::expr::cpp_cast_ref const expr, analyze::expr::function_arity const &arity) { auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_cast"))); - auto const value_tmp{ gen(expr->value_expr, arity, box_needed) }; + auto const value_tmp{ gen(expr->value_expr, arity) }; + + if(Cpp::IsVoid(expr->conversion_type)) + { + return "jank::runtime::jank_nil"; + } util::format_to( body_buffer, - "auto const {}{ jank::runtime::convert<{}>::{}({}) };", + "auto const {}{ jank::runtime::convert<{}>::{}({}{}) };", ret_tmp, - Cpp::GetTypeAsString(expr->conversion_type), + cpp_util::get_qualified_type_name(expr->conversion_type), (expr->policy == conversion_policy::into_object ? "into_object" : "from_object"), - value_tmp.unwrap().str(true)); + value_tmp.unwrap().str(true), + (expr->policy == conversion_policy::into_object ? "" : ".data")); if(expr->position == expression_position::tail) { @@ -1772,9 +1346,8 @@ namespace jank::codegen return ret_tmp; } - jtl::option processor::gen(analyze::expr::cpp_call_ref const expr, - analyze::expr::function_arity const &arity, - bool const) + jtl::option + processor::gen(analyze::expr::cpp_call_ref const expr, analyze::expr::function_arity const &arity) { if(expr->source_expr->kind == expression_kind::cpp_value) { @@ -1785,13 +1358,21 @@ namespace jank::codegen arg_tmps.reserve(expr->arg_exprs.size()); for(auto const &arg_expr : expr->arg_exprs) { - arg_tmps.emplace_back(gen(arg_expr, arity, true).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, arity).unwrap()); } - util::format_to(body_buffer, - "auto const {}{ {}(", - ret_tmp, - Cpp::GetQualifiedCompleteName(source->scope)); + auto const is_void{ Cpp::IsVoid(Cpp::GetFunctionReturnType(source->scope)) }; + + if(is_void) + { + util::format_to(body_buffer, "object_ref const {};", ret_tmp); + } + else + { + util::format_to(body_buffer, "auto const &{}{ ", ret_tmp); + } + + util::format_to(body_buffer, "{}(", Cpp::GetQualifiedCompleteName(source->scope)); bool need_comma{}; for(auto const &arg_tmp : arg_tmps) @@ -1804,7 +1385,16 @@ namespace jank::codegen need_comma = true; } - util::format_to(body_buffer, ") };"); + util::format_to(body_buffer, ")"); + + if(!is_void) + { + util::format_to(body_buffer, "};"); + } + else + { + util::format_to(body_buffer, ";"); + } if(expr->position == expression_position::tail) { @@ -1822,8 +1412,7 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::cpp_constructor_call_ref const expr, - analyze::expr::function_arity const &arity, - bool const) + analyze::expr::function_arity const &arity) { auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_ctor"))); @@ -1831,26 +1420,43 @@ namespace jank::codegen arg_tmps.reserve(expr->arg_exprs.size()); for(auto const &arg_expr : expr->arg_exprs) { - arg_tmps.emplace_back(gen(arg_expr, arity, false).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, arity).unwrap()); } if(expr->arg_exprs.empty()) { - util::format_to(body_buffer, "{} {}{ };", Cpp::GetTypeAsString(expr->type), ret_tmp); + util::format_to(body_buffer, + "{} {}{ };", + cpp_util::get_qualified_type_name(expr->type), + ret_tmp); return ret_tmp; } - util::format_to(body_buffer, "{} {}( ", Cpp::GetTypeAsString(expr->type), ret_tmp); + util::format_to(body_buffer, "{} {}( ", cpp_util::get_qualified_type_name(expr->type), ret_tmp); - bool need_comma{}; - for(auto const &arg_tmp : arg_tmps) + auto const is_primitive{ cpp_util::is_primitive(expr->type) }; + auto const arg_type{ cpp_util::expression_type(expr->arg_exprs[0]) }; + auto const needs_conversion{ !Cpp::IsConstructible(expr->type, arg_type) }; + if(is_primitive && needs_conversion) { - if(need_comma) + util::format_to(body_buffer, + "jank::runtime::convert<{}>::{}({}.get())", + cpp_util::get_qualified_type_name(expr->type), + "from_object", + arg_tmps[0].str(false)); + } + else + { + bool need_comma{}; + for(auto const &arg_tmp : arg_tmps) { - util::format_to(body_buffer, ", "); + if(need_comma) + { + util::format_to(body_buffer, ", "); + } + util::format_to(body_buffer, "{}", arg_tmp.str(false)); + need_comma = true; } - util::format_to(body_buffer, "{}", arg_tmp.str(false)); - need_comma = true; } util::format_to(body_buffer, " );"); @@ -1864,8 +1470,7 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::cpp_member_call_ref const expr, - analyze::expr::function_arity const &arity, - bool) + analyze::expr::function_arity const &arity) { auto const fn_name{ Cpp::GetName(expr->fn) }; auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(fn_name))); @@ -1874,16 +1479,31 @@ namespace jank::codegen arg_tmps.reserve(expr->arg_exprs.size()); for(auto const &arg_expr : expr->arg_exprs) { - arg_tmps.emplace_back(gen(arg_expr, arity, false).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, arity).unwrap()); } - util::format_to( - body_buffer, - "auto &&{}{ {}{}{}(", - ret_tmp, - arg_tmps[0].str(false), - (Cpp::IsPointerType(cpp_util::expression_type(expr->arg_exprs[0])) ? "->" : "."), - fn_name); + auto const is_void{ Cpp::IsVoid(Cpp::GetFunctionReturnType(expr->fn)) }; + + if(is_void) + { + util::format_to(body_buffer, "jank::runtime::object_ref {}{ };", ret_tmp); + util::format_to( + body_buffer, + "{}{}{}(", + arg_tmps[0].str(false), + (Cpp::IsPointerType(cpp_util::expression_type(expr->arg_exprs[0])) ? "->" : "."), + fn_name); + } + else + { + util::format_to( + body_buffer, + "auto &&{}{ {}{}{}(", + ret_tmp, + arg_tmps[0].str(false), + (Cpp::IsPointerType(cpp_util::expression_type(expr->arg_exprs[0])) ? "->" : "."), + fn_name); + } bool need_comma{}; for(auto it{ arg_tmps.begin() + 1 }; it != arg_tmps.end(); ++it) @@ -1896,7 +1516,14 @@ namespace jank::codegen need_comma = true; } - util::format_to(body_buffer, ") };"); + if(is_void) + { + util::format_to(body_buffer, ");"); + } + else + { + util::format_to(body_buffer, ") };"); + } if(expr->position == expression_position::tail) { @@ -1908,11 +1535,10 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::cpp_member_access_ref const expr, - analyze::expr::function_arity const &arity, - bool) + analyze::expr::function_arity const &arity) { auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(expr->name))); - auto obj_tmp(gen(expr->obj_expr, arity, false)); + auto obj_tmp(gen(expr->obj_expr, arity)); util::format_to(body_buffer, "auto &&{}{ {}{}{} };", @@ -1931,8 +1557,7 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::cpp_builtin_operator_call_ref const expr, - analyze::expr::function_arity const &arity, - bool) + analyze::expr::function_arity const &arity) { auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_operator"))); @@ -1940,7 +1565,7 @@ namespace jank::codegen arg_tmps.reserve(expr->arg_exprs.size()); for(auto const &arg_expr : expr->arg_exprs) { - arg_tmps.emplace_back(gen(arg_expr, arity, false).unwrap()); + arg_tmps.emplace_back(gen(arg_expr, arity).unwrap()); } if(expr->arg_exprs.size() == 1) @@ -1970,12 +1595,11 @@ namespace jank::codegen return ret_tmp; } - jtl::option processor::gen(analyze::expr::cpp_box_ref const expr, - analyze::expr::function_arity const &arity, - bool) + jtl::option + processor::gen(analyze::expr::cpp_box_ref const expr, analyze::expr::function_arity const &arity) { auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_box")) }; - auto value_tmp{ gen(expr->value_expr, arity, false) }; + auto value_tmp{ gen(expr->value_expr, arity) }; util::format_to(body_buffer, "auto {}{ jank::runtime::make_box({}) };", @@ -1992,11 +1616,10 @@ namespace jank::codegen } jtl::option processor::gen(analyze::expr::cpp_unbox_ref const expr, - analyze::expr::function_arity const &arity, - bool) + analyze::expr::function_arity const &arity) { auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_unbox")) }; - auto value_tmp{ gen(expr->value_expr, arity, false) }; + auto value_tmp{ gen(expr->value_expr, arity) }; util::format_to(body_buffer, "auto {}{ " @@ -2267,7 +1890,7 @@ namespace jank::codegen for(auto const &form : arity.body->values) { - gen(form, arity, true); + gen(form, arity); } if(arity.body->values.empty()) @@ -2324,28 +1947,16 @@ namespace jank::codegen } } - jtl::immutable_string processor::expression_str(bool const box_needed) + jtl::immutable_string processor::expression_str() { auto const module_ns(runtime::module::module_to_native_ns(module)); if(!generated_expression) { jtl::immutable_string close = ")"; - if(box_needed) - { - util::format_to( - expression_buffer, - "jank::runtime::make_box<{}>(", - runtime::module::nest_native_ns(module_ns, runtime::munge(struct_name.name))); - } - else - { - util::format_to( - expression_buffer, - "{}{ ", - runtime::module::nest_native_ns(module_ns, runtime::munge(struct_name.name))); - close = "}"; - } + util::format_to(expression_buffer, + "jank::runtime::make_box<{}>(", + runtime::module::nest_native_ns(module_ns, runtime::munge(struct_name.name))); native_set used_captures; bool need_comma{}; diff --git a/compiler+runtime/src/cpp/jank/evaluate.cpp b/compiler+runtime/src/cpp/jank/evaluate.cpp index cfe9c7b9c..909749420 100644 --- a/compiler+runtime/src/cpp/jank/evaluate.cpp +++ b/compiler+runtime/src/cpp/jank/evaluate.cpp @@ -607,7 +607,7 @@ namespace jank::evaluate codegen::processor cg_prc{ expr, module, codegen::compilation_target::eval }; util::println("{}\n", util::format_cpp_source(cg_prc.declaration_str()).expect_ok()); __rt_ctx->jit_prc.eval_string(cg_prc.declaration_str()); - auto const expr_str{ cg_prc.expression_str(true) + ".erase()" }; + auto const expr_str{ cg_prc.expression_str() + ".erase()" }; clang::Value v; auto res( __rt_ctx->jit_prc.interpreter->ParseAndExecute({ expr_str.data(), expr_str.size() }, &v)); diff --git a/compiler+runtime/src/cpp/jank/util/cli.cpp b/compiler+runtime/src/cpp/jank/util/cli.cpp index e48405128..c68cfd118 100644 --- a/compiler+runtime/src/cpp/jank/util/cli.cpp +++ b/compiler+runtime/src/cpp/jank/util/cli.cpp @@ -47,12 +47,12 @@ namespace jank::util::cli ->check(CLI::Range(0, 3)); std::map const codegen_types{ - { "llvm_ir", codegen_type::llvm_ir }, + { "llvm-ir", codegen_type::llvm_ir }, { "cpp", codegen_type::cpp } }; cli.add_option("--codegen", opts.codegen, "The type of code generation to use.") - ->transform(CLI::CheckedTransformer(codegen_types).description("{llvm_ir,cpp}")) - ->default_str(make_default("llvm_ir")); + ->transform(CLI::CheckedTransformer(codegen_types).description("{llvm-ir,cpp}")) + ->default_str(make_default("cpp")); /* Native dependencies. */ cli.add_option("-I,--include-dir", diff --git a/compiler+runtime/src/cpp/jtl/string_builder.cpp b/compiler+runtime/src/cpp/jtl/string_builder.cpp index ef7de4a32..369ec23b4 100644 --- a/compiler+runtime/src/cpp/jtl/string_builder.cpp +++ b/compiler+runtime/src/cpp/jtl/string_builder.cpp @@ -13,15 +13,14 @@ namespace jtl using allocator_traits = std::allocator_traits; /* NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) */ - static allocator_type allocator; + //static allocator_type allocator; static void realloc(string_builder &sb, usize const required) { auto const new_capacity{ std::bit_ceil(required) }; - /* TODO: Pointer-free GC alloc. */ - auto const new_data{ allocator_traits::allocate(allocator, new_capacity) }; + auto const new_data{ reinterpret_cast(GC_malloc(new_capacity)) }; string_builder::traits_type::copy(new_data, sb.buffer, sb.pos); - allocator_traits::deallocate(allocator, sb.buffer, sb.pos); + GC_free(sb.buffer); sb.buffer = new_data; sb.capacity = new_capacity; } @@ -103,7 +102,7 @@ namespace jtl string_builder::~string_builder() { - allocator_traits::deallocate(allocator, buffer, pos); + GC_free(buffer); } string_builder &string_builder::operator()(bool const d) & From 278f7cb12b35b046282553d32b96961981e29df0 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 18 Nov 2025 10:58:05 -0800 Subject: [PATCH 10/58] Address some tidy issues --- .../src/cpp/jank/runtime/obj/transient_sorted_map.cpp | 2 +- .../src/cpp/jank/runtime/obj/transient_sorted_set.cpp | 2 +- compiler+runtime/src/cpp/jtl/string_builder.cpp | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp index 0629d1c22..530c8e4cb 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_map.cpp @@ -95,7 +95,7 @@ namespace jank::runtime::obj bool transient_sorted_map::contains(object_ref const key) const { assert_active(); - return data.find(key) != data.end(); + return data.contains(key); } transient_sorted_map_ref diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp index 1aa04f45b..cba4ce9af 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/transient_sorted_set.cpp @@ -116,7 +116,7 @@ namespace jank::runtime::obj bool transient_sorted_set::contains(object_ref const elem) const { assert_active(); - return data.find(elem) != data.end(); + return data.contains(elem); } transient_sorted_set_ref transient_sorted_set::disjoin_in_place(object_ref const elem) diff --git a/compiler+runtime/src/cpp/jtl/string_builder.cpp b/compiler+runtime/src/cpp/jtl/string_builder.cpp index 369ec23b4..3c683c534 100644 --- a/compiler+runtime/src/cpp/jtl/string_builder.cpp +++ b/compiler+runtime/src/cpp/jtl/string_builder.cpp @@ -12,13 +12,10 @@ namespace jtl using allocator_type = jank::native_allocator; using allocator_traits = std::allocator_traits; - /* NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) */ - //static allocator_type allocator; - static void realloc(string_builder &sb, usize const required) { auto const new_capacity{ std::bit_ceil(required) }; - auto const new_data{ reinterpret_cast(GC_malloc(new_capacity)) }; + auto const new_data{ reinterpret_cast(GC_malloc_atomic(new_capacity)) }; string_builder::traits_type::copy(new_data, sb.buffer, sb.pos); GC_free(sb.buffer); sb.buffer = new_data; From 69418a2eab87901218e09e673da2d3f7201ad774 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 18 Nov 2025 14:33:26 -0800 Subject: [PATCH 11/58] Generate stack save/restore during loop IR gen This addresses a GC crash we were seeing, as documented here: https://github.com/bdwgc/bdwgc/issues/800#issuecomment-3495453933 --- .../include/cpp/jank/util/cli.hpp | 15 +- .../src/cpp/jank/codegen/llvm_processor.cpp | 128 +++++++++++++----- compiler+runtime/src/cpp/jank/util/cli.cpp | 2 +- 3 files changed, 112 insertions(+), 33 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/util/cli.hpp b/compiler+runtime/include/cpp/jank/util/cli.hpp index defa3e09a..0d63837e3 100644 --- a/compiler+runtime/include/cpp/jank/util/cli.hpp +++ b/compiler+runtime/include/cpp/jank/util/cli.hpp @@ -21,6 +21,19 @@ namespace jank::util::cli cpp }; + constexpr char const *codegen_type_str(codegen_type const type) + { + switch(type) + { + case codegen_type::llvm_ir: + return "llvm-ir"; + case codegen_type::cpp: + return "cpp"; + default: + return "unknown"; + } + } + struct options { /* Runtime. */ @@ -29,7 +42,7 @@ namespace jank::util::cli bool profiler_enabled{}; bool perf_profiling_enabled{}; bool gc_incremental{}; - codegen_type codegen{ codegen_type::cpp }; + codegen_type codegen{ codegen_type::llvm_ir }; /* Native dependencies. */ native_vector include_dirs; diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index 4266f59c2..e39af6c3b 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -173,6 +173,11 @@ namespace jank::codegen llvm::StructType *get_or_insert_struct_type(std::string const &name, std::vector const &fields) const; + util::scope_exit gen_stack_save(); + void gen_stack_restore(); + jtl::ptr gen_ret(jtl::ptr const value); + jtl::ptr gen_ret(); + compilation_target target{}; analyze::expr::function_ref root_fn; jtl::ptr llvm_fn{}; @@ -195,6 +200,7 @@ namespace jank::codegen * We don't use this within the current fn, but it's passed upward to * the fn gen which is above us, all the way up to the module level. */ native_unordered_map global_rtti; + native_vector> stack_saves; }; struct llvm_type_info @@ -729,7 +735,6 @@ namespace jank::codegen jtl::string_result llvm_processor::impl::gen() { - throw "no IR gen allowed!"; profile::timer const timer{ "ir gen" }; if(target != compilation_target::function) { @@ -757,7 +762,7 @@ namespace jank::codegen continue; } - ctx->builder->CreateRet(gen_global(jank_nil)); + gen_ret(gen_global(jank_nil)); } if(target != compilation_target::function) @@ -775,7 +780,7 @@ namespace jank::codegen { gen_c_string(util::format("global ctor for {}", root_fn->name)) }); } - ctx->builder->CreateRetVoid(); + gen_ret(); } /* For modules, we need to make sure to define RTTI symbols manually. Since @@ -857,7 +862,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(ref); + return gen_ret(ref); } return ref; @@ -898,7 +903,7 @@ namespace jank::codegen } if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -911,7 +916,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(var); + return gen_ret(var); } return var; @@ -988,7 +993,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1035,7 +1040,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(ret); + return gen_ret(ret); } return ret; @@ -1062,7 +1067,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1089,7 +1094,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1117,7 +1122,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1144,7 +1149,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1161,7 +1166,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(load_if_needed(ctx, ret)); + return gen_ret(load_if_needed(ctx, ret)); } return ret; @@ -1188,7 +1193,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(fn_obj); + return gen_ret(fn_obj); } return fn_obj; @@ -1230,6 +1235,7 @@ namespace jank::codegen ctx->builder->CreateStore(store.first, store.second); } + gen_stack_restore(); return ctx->builder->CreateBr(current_loop.data); } else @@ -1279,7 +1285,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1299,7 +1305,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(fn_obj); + return gen_ret(fn_obj); } return fn_obj; @@ -1377,7 +1383,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -1431,6 +1437,8 @@ namespace jank::codegen ctx->builder->CreateBr(loop_block); ctx->builder->SetInsertPoint(loop_block); + auto const stack_save{ gen_stack_save() }; + auto const ret(gen(expr->body, arity)); locals = std::move(old_locals); @@ -1440,6 +1448,7 @@ namespace jank::codegen auto const postloop_block(llvm::BasicBlock::Create(*llvm_ctx, "postloop", current_fn)); if(!ctx->builder->GetInsertBlock()->getTerminator()) { + gen_stack_restore(); ctx->builder->CreateBr(postloop_block); } ctx->builder->SetInsertPoint(postloop_block); @@ -1574,7 +1583,7 @@ namespace jank::codegen else_ = gen_global(jank_nil); if(expr->position == expression_position::tail) { - else_ = ctx->builder->CreateRet(else_); + else_ = gen_ret(else_); } } @@ -1648,7 +1657,7 @@ namespace jank::codegen auto const ret{ gen_global(jank_nil) }; if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(ret); + return gen_ret(ret); } return ret; } @@ -2055,7 +2064,7 @@ namespace jank::codegen if(is_return) { - ctx->builder->CreateRet(final_val); + gen_ret(final_val); } return final_val; } @@ -2142,7 +2151,7 @@ namespace jank::codegen auto const ret{ gen_global(jank_nil) }; if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(ret); + return gen_ret(ret); } return ret; } @@ -2166,7 +2175,7 @@ namespace jank::codegen ctx->builder->CreateStore(null, alloc); if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(alloc); + return gen_ret(alloc); } return alloc; } @@ -2181,7 +2190,7 @@ namespace jank::codegen ctx->builder->CreateStore(ir_val, alloc); if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(alloc); + return gen_ret(alloc); } return alloc; } @@ -2199,7 +2208,7 @@ namespace jank::codegen ctx->builder->CreateStore(ir_val, alloc); if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(alloc); + return gen_ret(alloc); } return alloc; } @@ -2227,7 +2236,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(alloc); + return gen_ret(alloc); } return alloc; @@ -2248,7 +2257,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(load_if_needed(ctx, converted)); + return gen_ret(load_if_needed(ctx, converted)); } return converted; @@ -2424,11 +2433,11 @@ namespace jank::codegen { if(is_void) { - return ctx->builder->CreateRet(gen_global(jank_nil)); + return gen_ret(gen_global(jank_nil)); } auto const ret_load{ ctx->builder->CreateLoad(ctx->builder->getPtrTy(), ret_alloc, "ret") }; - return ctx->builder->CreateRet(ret_load); + return gen_ret(ret_load); } if(is_void) @@ -2693,7 +2702,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return call; @@ -2720,7 +2729,7 @@ namespace jank::codegen if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(call); + return gen_ret(call); } return alloc; @@ -2818,7 +2827,7 @@ namespace jank::codegen auto const ret{ gen_global(jank_nil) }; if(expr->position == expression_position::tail) { - return ctx->builder->CreateRet(ret); + return gen_ret(ret); } return ret; @@ -3723,6 +3732,63 @@ namespace jank::codegen return struct_type; } + util::scope_exit llvm_processor::impl::gen_stack_save() + { + /* In some cases, such as loops, we use LLVM's stack preservation intrinsics. + * These will help us reset the stack on each iteration so that each new alloc + * doesn't actually keep grabbing more stack space. + * + * However, there is some tricky logic in balancing each save/restore, since we + * can have multiple terminators in an IR function and we need to make sure that + * each of them gets a restore. Also, we can have nested saves and we need to + * make sure they get restored in the correct order. */ + auto const stack_ptr{ ctx->builder->CreateStackSave() }; + stack_saves.emplace_back(stack_ptr); + + /* The main logic here is to pop the back of the stack, but we add some error handling + * as well, to detect cases where we're not restoring in the correct order. */ + return { [stack_ptr, this]() { + ssize found{ -1 }; + for(ssize i{}; i != static_cast(stack_saves.size()); ++i) + { + if(stack_saves[i].data == stack_ptr) + { + found = i; + } + } + + jank_debug_assert(found == -1 || found == static_cast(stack_saves.size()) - 1); + if(found != -1) + { + stack_saves.erase(stack_saves.begin() + found); + } + } }; + } + + void llvm_processor::impl::gen_stack_restore() + { + jank_debug_assert(!stack_saves.empty()); + ctx->builder->CreateStackRestore(stack_saves.back()); + } + + jtl::ptr llvm_processor::impl::gen_ret(jtl::ptr const value) + { + for(auto it{ stack_saves.rbegin() }; it != stack_saves.rend(); ++it) + { + ctx->builder->CreateStackRestore(*it); + } + return ctx->builder->CreateRet(value); + } + + jtl::ptr llvm_processor::impl::gen_ret() + { + for(auto it{ stack_saves.rbegin() }; it != stack_saves.rend(); ++it) + { + ctx->builder->CreateStackRestore(*it); + } + return ctx->builder->CreateRetVoid(); + } + void llvm_processor::optimize() const { jtl::immutable_string_view const print_settings{ getenv("JANK_PRINT_IR") ?: "" }; diff --git a/compiler+runtime/src/cpp/jank/util/cli.cpp b/compiler+runtime/src/cpp/jank/util/cli.cpp index c68cfd118..8752374d4 100644 --- a/compiler+runtime/src/cpp/jank/util/cli.cpp +++ b/compiler+runtime/src/cpp/jank/util/cli.cpp @@ -52,7 +52,7 @@ namespace jank::util::cli }; cli.add_option("--codegen", opts.codegen, "The type of code generation to use.") ->transform(CLI::CheckedTransformer(codegen_types).description("{llvm-ir,cpp}")) - ->default_str(make_default("cpp")); + ->default_str(make_default(codegen_type_str(opts.codegen))); /* Native dependencies. */ cli.add_option("-I,--include-dir", From 746e7560b7de94c427b2677e153d8dd08dbaa4cb Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 20 Nov 2025 12:19:23 -0800 Subject: [PATCH 12/58] Fix more cppgen issues --- .../include/cpp/jank/analyze/expr/if.hpp | 1 + .../include/cpp/jank/codegen/processor.hpp | 4 + .../src/cpp/jank/analyze/cpp_util.cpp | 6 + .../src/cpp/jank/analyze/local_frame.cpp | 8 + .../src/cpp/jank/codegen/processor.cpp | 183 ++++++++++++++++-- compiler+runtime/src/cpp/jank/evaluate.cpp | 9 +- .../src/cpp/jank/runtime/obj/uuid.cpp | 2 +- .../test/jank/cpp/auto-box/pass-closure.jank | 4 +- 8 files changed, 192 insertions(+), 25 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/analyze/expr/if.hpp b/compiler+runtime/include/cpp/jank/analyze/expr/if.hpp index a97780514..a8636d3c8 100644 --- a/compiler+runtime/include/cpp/jank/analyze/expr/if.hpp +++ b/compiler+runtime/include/cpp/jank/analyze/expr/if.hpp @@ -25,6 +25,7 @@ namespace jank::analyze::expr /* TODO: Rename to have _expr suffixes. */ expression_ref condition; + /* The then/else exprs are expected to have the same type. We handle this during analysis. */ expression_ref then; jtl::option else_; }; diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index f3f73d4eb..05cdb814f 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -142,6 +142,10 @@ namespace jank::codegen gen(analyze::expr::cpp_box_ref const, analyze::expr::function_arity const &); jtl::option gen(analyze::expr::cpp_unbox_ref const, analyze::expr::function_arity const &); + jtl::option + gen(analyze::expr::cpp_new_ref const, analyze::expr::function_arity const &); + jtl::option + gen(analyze::expr::cpp_delete_ref const, analyze::expr::function_arity const &); jtl::immutable_string declaration_str(); void build_header(); diff --git a/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp b/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp index fc08303c9..a16daf244 100644 --- a/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp @@ -294,6 +294,12 @@ namespace jank::analyze::cpp_util jtl::immutable_string get_qualified_type_name(jtl::ptr const type) { + if(type == untyped_object_ptr_type()) + { + return "jank::runtime::object_ref"; + } + /* TODO: Handle typed object refs, too. */ + if(auto const scope{ Cpp::GetScopeFromType(type) }; scope) { auto name{ get_qualified_name(scope) }; diff --git a/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp b/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp index d5dc397db..c3b5f4072 100644 --- a/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -103,6 +104,13 @@ namespace jank::analyze res.first->second.has_boxed_usage = true; /* To start with, we assume it's only boxed. */ res.first->second.has_unboxed_usage = false; + + /* Native values which are captured get auto-boxed, so we need to adjust the type + * of the binding. */ + if(!cpp_util::is_any_object(res.first->second.type)) + { + res.first->second.type = cpp_util::untyped_object_ptr_type(); + } } } diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index d7244e8a7..0f83ff3c2 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -34,13 +34,13 @@ * roughly this C++: * * ```c++ - * object_ref thing_result(thing->call()); - * object_ref if_result; + * object_ref thing_tmp(thing->call()); + * object_ref if_tmp; * if(foo) - * { if_result = bar; } + * { if_tmp = bar; } * else - * { if_result = spam; } - * println->call(thing_result, if_result); + * { if_tmp = spam; } + * println->call(thing_tmp, if_tmp); * ``` * * This is optimized by knowing what position every expression in, so trivial expressions used @@ -205,8 +205,8 @@ namespace jank::codegen else if constexpr(std::same_as) { util::format_to(buffer, - R"(jank::runtime::make_box({}))", - typed_o->to_code_string()); + R"(jank::runtime::make_box("{}"))", + util::escape(typed_o->to_string())); } else if constexpr(std::same_as) { @@ -220,8 +220,15 @@ namespace jank::codegen { util::format_to(buffer, R"(jank::runtime::make_box({}))", + /* We remove the # prefix here. */ typed_o->to_code_string().substr(1)); } + else if constexpr(std::same_as) + { + util::format_to(buffer, + R"(jank::runtime::make_box("{}"))", + typed_o->to_string()); + } else if constexpr(std::same_as) { util::format_to(buffer, @@ -1136,9 +1143,12 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::if_ref const expr, analyze::expr::function_arity const &fn_arity) { - /* TODO: Handle unboxed results! */ auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("if"))); - util::format_to(body_buffer, "object_ref {}{ };", ret_tmp); + auto const expr_type{ cpp_util::expression_type(expr->then) }; + util::format_to(body_buffer, + "{} {}{ };", + cpp_util::get_qualified_type_name(expr_type), + ret_tmp); auto const &condition_tmp(gen(expr->condition, fn_arity)); util::format_to(body_buffer, "if(jank::runtime::truthy({})) {", @@ -1330,12 +1340,12 @@ namespace jank::codegen util::format_to( body_buffer, - "auto const {}{ jank::runtime::convert<{}>::{}({}{}) };", + "auto const {}{ jank::runtime::convert<{}>::{}({}) };", ret_tmp, - cpp_util::get_qualified_type_name(expr->conversion_type), + cpp_util::get_qualified_type_name( + Cpp::GetTypeWithoutCv(Cpp::GetNonReferenceType(expr->conversion_type))), (expr->policy == conversion_policy::into_object ? "into_object" : "from_object"), - value_tmp.unwrap().str(true), - (expr->policy == conversion_policy::into_object ? "" : ".data")); + value_tmp.unwrap().str(true)); if(expr->position == expression_position::tail) { @@ -1406,8 +1416,59 @@ namespace jank::codegen } else { - jank_debug_assert(false); - return none; + auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_call"))); + + auto const source_tmp{ gen(expr->source_expr, arity).unwrap() }; + + native_vector arg_tmps; + arg_tmps.reserve(expr->arg_exprs.size()); + for(auto const &arg_expr : expr->arg_exprs) + { + arg_tmps.emplace_back(gen(arg_expr, arity).unwrap()); + } + + auto const is_void{ Cpp::IsVoid(expr->type) }; + + if(is_void) + { + util::format_to(body_buffer, "object_ref const {};", ret_tmp); + } + else + { + util::format_to(body_buffer, "auto const &{}{ ", ret_tmp); + } + + util::format_to(body_buffer, "{}(", source_tmp.str(true)); + + bool need_comma{}; + for(auto const &arg_tmp : arg_tmps) + { + if(need_comma) + { + util::format_to(body_buffer, ", "); + } + util::format_to(body_buffer, "{}", arg_tmp.str(true)); + need_comma = true; + } + + util::format_to(body_buffer, ")"); + + if(!is_void) + { + util::format_to(body_buffer, "};"); + } + else + { + util::format_to(body_buffer, ";"); + } + + if(expr->position == expression_position::tail) + { + util::format_to(body_buffer, "return {};", ret_tmp); + return none; + } + + return ret_tmp; } } @@ -1638,6 +1699,76 @@ namespace jank::codegen return ret_tmp; } + jtl::option + processor::gen(analyze::expr::cpp_new_ref const expr, analyze::expr::function_arity const &arity) + { + auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_new")) }; + auto finalizer_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("finalizer")) }; + auto value_tmp{ gen(expr->value_expr, arity) }; + + auto const type_name{ cpp_util::get_qualified_type_name(expr->type) }; + auto const needs_finalizer{ !Cpp::IsTriviallyDestructible(expr->type) }; + + if(needs_finalizer) + { + util::format_to(body_buffer, + "using T = {};\n" + "static auto const {}{ " + "[](void * const obj, void *){" + "reinterpret_cast(obj)->~T();" + "} };", + type_name, + finalizer_tmp); + } + + util::format_to(body_buffer, + "auto {}{ " + "new (GC{}) {}{ {} }" + " };", + ret_tmp, + (needs_finalizer ? ", " + finalizer_tmp : ""), + type_name, + value_tmp.unwrap().str(false)); + + if(expr->position == expression_position::tail) + { + util::format_to(body_buffer, "return {};", ret_tmp); + return none; + } + + return ret_tmp; + } + + jtl::option processor::gen(analyze::expr::cpp_delete_ref const expr, + analyze::expr::function_arity const &arity) + { + auto value_tmp{ gen(expr->value_expr, arity).unwrap() }; + auto const value_type{ Cpp::GetPointeeType(cpp_util::expression_type(expr->value_expr)) }; + auto const type_name{ cpp_util::get_qualified_type_name(value_type) }; + auto const needs_finalizer{ !Cpp::IsTriviallyDestructible(value_type) }; + + /* Calling GC_free won't trigger the finalizer. Not sure why, but it's explicitly + * documented in bdwgc. So, we'll invoke it manually if needed, prior to GC_free. */ + if(needs_finalizer) + { + util::format_to(body_buffer, + "using T = {};\n" + "{}->~T();", + type_name, + value_tmp.str(false)); + } + + util::format_to(body_buffer, "GC_free({});", value_tmp.str(false)); + + if(expr->position == expression_position::tail) + { + util::format_to(body_buffer, "return jank::runtime::jank_nil;"); + return none; + } + + return "jank::runtime::jank_nil"; + } + jtl::immutable_string processor::declaration_str() { if(!generated_declaration) @@ -1988,12 +2119,22 @@ namespace jank::codegen { auto const originating_local(root_fn->frame->find_local_or_capture(v.first)); handle const h{ originating_local.unwrap().binding }; - util::format_to(expression_buffer, - "{} {}", - (need_comma ? "," : ""), - h.str(true), - originating_local.unwrap().binding->name->to_code_string(), - originating_local.unwrap().binding->native_name); + auto const local_type{ originating_local.unwrap().binding->type }; + auto const needs_conversion{ !cpp_util::is_any_object(local_type) }; + + if(needs_conversion) + { + util::format_to(expression_buffer, + "{} jank::runtime::convert<{}>::{}({})", + (need_comma ? "," : ""), + cpp_util::get_qualified_type_name(local_type), + "into_object", + h.str(true)); + } + else + { + util::format_to(expression_buffer, "{} {}", (need_comma ? "," : ""), h.str(true)); + } } need_comma = true; } diff --git a/compiler+runtime/src/cpp/jank/evaluate.cpp b/compiler+runtime/src/cpp/jank/evaluate.cpp index 909749420..bf0abb321 100644 --- a/compiler+runtime/src/cpp/jank/evaluate.cpp +++ b/compiler+runtime/src/cpp/jank/evaluate.cpp @@ -605,7 +605,14 @@ namespace jank::evaluate else { codegen::processor cg_prc{ expr, module, codegen::compilation_target::eval }; - util::println("{}\n", util::format_cpp_source(cg_prc.declaration_str()).expect_ok()); + + /* TODO: Rename to something generic which makes sense for IR and C++ gen? */ + jtl::immutable_string_view const print_settings{ getenv("JANK_PRINT_IR") ?: "" }; + if(print_settings == "1") + { + util::println("{}\n", util::format_cpp_source(cg_prc.declaration_str()).expect_ok()); + } + __rt_ctx->jit_prc.eval_string(cg_prc.declaration_str()); auto const expr_str{ cg_prc.expression_str() + ".erase()" }; clang::Value v; diff --git a/compiler+runtime/src/cpp/jank/runtime/obj/uuid.cpp b/compiler+runtime/src/cpp/jank/runtime/obj/uuid.cpp index 38717e840..ef9634ebb 100644 --- a/compiler+runtime/src/cpp/jank/runtime/obj/uuid.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/obj/uuid.cpp @@ -17,7 +17,7 @@ namespace jank::runtime::obj static jtl::ref from_string(jtl::immutable_string const &s) { - auto const result = uuids::uuid::from_string(s.c_str()); + auto const result{ uuids::uuid::from_string(s.c_str()) }; if(result) { return jtl::make_ref(result.value()); diff --git a/compiler+runtime/test/jank/cpp/auto-box/pass-closure.jank b/compiler+runtime/test/jank/cpp/auto-box/pass-closure.jank index b0640ac19..a07fd8182 100644 --- a/compiler+runtime/test/jank/cpp/auto-box/pass-closure.jank +++ b/compiler+runtime/test/jank/cpp/auto-box/pass-closure.jank @@ -1,9 +1,9 @@ (let [i (cpp/int. 5) - ifn (fn* [] + ifn (fn* capture-i [] (assert (= 5 i))) _ (ifn) f (cpp/float. 5.0) - ffn (fn* [] + ffn (fn* capture-f [] (if (= 5.0 f) :success))] (ffn)) From 5025847ffad682ae0557d1cdd7552f78f5c05caa Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 20 Nov 2025 13:47:35 -0800 Subject: [PATCH 13/58] Fix some more cppgen tests --- .../src/cpp/jank/codegen/processor.cpp | 67 +++++++++++++------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 0f83ff3c2..d54874c8f 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1493,34 +1493,61 @@ namespace jank::codegen return ret_tmp; } - util::format_to(body_buffer, "{} {}( ", cpp_util::get_qualified_type_name(expr->type), ret_tmp); + util::format_to(body_buffer, "{} {}{ ", cpp_util::get_qualified_type_name(expr->type), ret_tmp); - auto const is_primitive{ cpp_util::is_primitive(expr->type) }; - auto const arg_type{ cpp_util::expression_type(expr->arg_exprs[0]) }; - auto const needs_conversion{ !Cpp::IsConstructible(expr->type, arg_type) }; - if(is_primitive && needs_conversion) + if(!expr->arg_exprs.empty()) { - util::format_to(body_buffer, - "jank::runtime::convert<{}>::{}({}.get())", - cpp_util::get_qualified_type_name(expr->type), - "from_object", - arg_tmps[0].str(false)); - } - else - { - bool need_comma{}; - for(auto const &arg_tmp : arg_tmps) + auto const arg_type{ cpp_util::expression_type(expr->arg_exprs[0]) }; + bool needs_conversion{}; + jtl::immutable_string conversion_type; + if(cpp_util::is_any_object(expr->type) && !cpp_util::is_any_object(arg_type)) { - if(need_comma) + needs_conversion = true; + conversion_type = "into_object"; + } + else if(!cpp_util::is_any_object(expr->type) && cpp_util::is_any_object(arg_type)) + { + needs_conversion = true; + conversion_type = "from_object"; + } + + if(needs_conversion) + { + util::format_to(body_buffer, + "jank::runtime::convert<{}>::{}({}.get())", + cpp_util::get_qualified_type_name(expr->type), + conversion_type, + arg_tmps[0].str(false)); + } + else + { + auto const needs_static_cast{ expr->type != arg_type && expr->arg_exprs.size() == 1 }; + if(needs_static_cast) { - util::format_to(body_buffer, ", "); + util::format_to(body_buffer, + "static_cast<{}>(", + cpp_util::get_qualified_type_name(expr->type)); + } + + bool need_comma{}; + for(auto const &arg_tmp : arg_tmps) + { + if(need_comma) + { + util::format_to(body_buffer, ", "); + } + util::format_to(body_buffer, "{}", arg_tmp.str(false)); + need_comma = true; + } + + if(needs_static_cast) + { + util::format_to(body_buffer, ")"); } - util::format_to(body_buffer, "{}", arg_tmp.str(false)); - need_comma = true; } } - util::format_to(body_buffer, " );"); + util::format_to(body_buffer, " };"); if(expr->position == expression_position::tail) { From 88623e52dc3de80abbadeb95692dd3c14f94fcaf Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 20 Nov 2025 14:06:46 -0800 Subject: [PATCH 14/58] More cppgen fixes --- .../src/cpp/jank/codegen/processor.cpp | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index d54874c8f..12d38e148 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1375,7 +1375,7 @@ namespace jank::codegen if(is_void) { - util::format_to(body_buffer, "object_ref const {};", ret_tmp); + util::format_to(body_buffer, "jank::runtime::object_ref const {};", ret_tmp); } else { @@ -1385,13 +1385,22 @@ namespace jank::codegen util::format_to(body_buffer, "{}(", Cpp::GetQualifiedCompleteName(source->scope)); bool need_comma{}; - for(auto const &arg_tmp : arg_tmps) + for(u8 arg_idx{}; arg_idx < expr->arg_exprs.size(); ++arg_idx) { + auto const arg_expr{ expr->arg_exprs[arg_idx] }; + auto const arg_type{ cpp_util::expression_type(arg_expr) }; + auto const param_type{ Cpp::GetFunctionArgType(source->scope, arg_idx) }; + auto const &arg_tmp{ arg_tmps[arg_idx] }; + if(need_comma) { util::format_to(body_buffer, ", "); } util::format_to(body_buffer, "{}", arg_tmp.str(true)); + if(Cpp::IsPointerType(param_type) && cpp_util::is_any_object(arg_type)) + { + util::format_to(body_buffer, ".erase()"); + } need_comma = true; } @@ -1431,7 +1440,7 @@ namespace jank::codegen if(is_void) { - util::format_to(body_buffer, "object_ref const {};", ret_tmp); + util::format_to(body_buffer, "jank::runtime::object_ref const {};", ret_tmp); } else { @@ -1867,20 +1876,21 @@ namespace jank::codegen } used_constants.emplace(v.second.native_name.to_hash()); + /* TODO: Typed lifted constants. */ util::format_to(header_buffer, "{} const {};", detail::gen_constant_type(v.second.data, true), runtime::munge(v.second.native_name)); - if(v.second.unboxed_native_name.is_some()) - { - util::format_to(header_buffer, - "static constexpr {} const {}{ ", - detail::gen_constant_type(v.second.data, false), - runtime::munge(v.second.unboxed_native_name.unwrap())); - detail::gen_constant(v.second.data, header_buffer, false); - util::format_to(header_buffer, "};"); - } + //if(v.second.unboxed_native_name.is_some()) + //{ + // util::format_to(header_buffer, + // "static constexpr {} const {}{ ", + // detail::gen_constant_type(v.second.data, false), + // runtime::munge(v.second.unboxed_native_name.unwrap())); + // detail::gen_constant(v.second.data, header_buffer, false); + // util::format_to(header_buffer, "};"); + //} } /* TODO: More useful types here. */ @@ -2026,7 +2036,9 @@ namespace jank::codegen if(!param_shadows_fn) { - util::format_to(body_buffer, "object_ref const {}{ this };", runtime::munge(root_fn->name)); + util::format_to(body_buffer, + "jank::runtime::object_ref const {}{ this };", + runtime::munge(root_fn->name)); } if(arity.fn_ctx->is_tail_recursive) From 05dda52db3600bd9d60ac5367545388631785d67 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 20 Nov 2025 14:12:38 -0800 Subject: [PATCH 15/58] More cppgen fixes --- compiler+runtime/src/cpp/jank/codegen/processor.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 12d38e148..f592b8656 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1697,11 +1697,16 @@ namespace jank::codegen { auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_box")) }; auto value_tmp{ gen(expr->value_expr, arity) }; + auto const value_expr_type{ cpp_util::expression_type(expr->value_expr) }; + auto const type_str{ Cpp::GetTypeAsString( + Cpp::GetCanonicalType(Cpp::GetNonReferenceType(value_expr_type))) }; - util::format_to(body_buffer, - "auto {}{ jank::runtime::make_box({}) };", - ret_tmp, - value_tmp.unwrap().str(false)); + util::format_to( + body_buffer, + "auto {}{ jank::runtime::make_box({}, \"{}\") };", + ret_tmp, + value_tmp.unwrap().str(false), + type_str); if(expr->position == expression_position::tail) { From a807c3dbad1441c9e8f2809c6fba4673afbb0190 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 20 Nov 2025 16:49:04 -0800 Subject: [PATCH 16/58] More cppgen fixes --- .../include/cpp/jank/util/cli.hpp | 2 +- .../src/cpp/jank/codegen/processor.cpp | 37 ++++++++++++++----- .../src/cpp/jank/runtime/core/munge.cpp | 4 ++ 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/util/cli.hpp b/compiler+runtime/include/cpp/jank/util/cli.hpp index 0d63837e3..56b44ba88 100644 --- a/compiler+runtime/include/cpp/jank/util/cli.hpp +++ b/compiler+runtime/include/cpp/jank/util/cli.hpp @@ -42,7 +42,7 @@ namespace jank::util::cli bool profiler_enabled{}; bool perf_profiling_enabled{}; bool gc_incremental{}; - codegen_type codegen{ codegen_type::llvm_ir }; + codegen_type codegen{ codegen_type::cpp }; /* Native dependencies. */ native_vector include_dirs; diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index f592b8656..e302475d3 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1335,6 +1335,11 @@ namespace jank::codegen if(Cpp::IsVoid(expr->conversion_type)) { + if(expr->position == expression_position::tail) + { + util::format_to(body_buffer, "return jank::runtime::jank_nil;"); + return none; + } return "jank::runtime::jank_nil"; } @@ -1379,7 +1384,7 @@ namespace jank::codegen } else { - util::format_to(body_buffer, "auto const &{}{ ", ret_tmp); + util::format_to(body_buffer, "auto &&{}{ ", ret_tmp); } util::format_to(body_buffer, "{}(", Cpp::GetQualifiedCompleteName(source->scope)); @@ -1397,7 +1402,7 @@ namespace jank::codegen util::format_to(body_buffer, ", "); } util::format_to(body_buffer, "{}", arg_tmp.str(true)); - if(Cpp::IsPointerType(param_type) && cpp_util::is_any_object(arg_type)) + if(param_type && Cpp::IsPointerType(param_type) && cpp_util::is_any_object(arg_type)) { util::format_to(body_buffer, ".erase()"); } @@ -1444,7 +1449,7 @@ namespace jank::codegen } else { - util::format_to(body_buffer, "auto const &{}{ ", ret_tmp); + util::format_to(body_buffer, "auto &&{}{ ", ret_tmp); } util::format_to(body_buffer, "{}(", source_tmp.str(true)); @@ -1667,16 +1672,30 @@ namespace jank::codegen if(expr->arg_exprs.size() == 1) { - util::format_to(body_buffer, - "auto {}( {}{} );", - ret_tmp, - cpp_util::operator_name(static_cast(expr->op)).unwrap(), - arg_tmps[0].str(false)); + auto const arg_type{ cpp_util::expression_type(expr->arg_exprs[0]) }; + if(Cpp::IsArrayType(Cpp::GetNonReferenceType(arg_type))) + { + util::format_to(body_buffer, + "auto &&{}( {} static_cast<{}>({}) );", + ret_tmp, + cpp_util::operator_name(static_cast(expr->op)).unwrap(), + cpp_util::get_qualified_type_name( + Cpp::GetPointerType(Cpp::GetArrayElementType(arg_type))), + arg_tmps[0].str(false)); + } + else + { + util::format_to(body_buffer, + "auto &&{}( {}{} );", + ret_tmp, + cpp_util::operator_name(static_cast(expr->op)).unwrap(), + arg_tmps[0].str(false)); + } } else { util::format_to(body_buffer, - "auto {}( {} {} {} );", + "auto &&{}( {} {} {} );", ret_tmp, arg_tmps[0].str(false), cpp_util::operator_name(static_cast(expr->op)).unwrap(), diff --git a/compiler+runtime/src/cpp/jank/runtime/core/munge.cpp b/compiler+runtime/src/cpp/jank/runtime/core/munge.cpp index 1c508e91e..449eaffa4 100644 --- a/compiler+runtime/src/cpp/jank/runtime/core/munge.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/core/munge.cpp @@ -28,6 +28,8 @@ namespace jank::runtime { '}', "_RBRACE_" }, { '[', "_LBRACK_" }, { ']', "_RBRACK_" }, + { '(', "_LPAREN_" }, + { ')', "_RPAREN_" }, { '/', "_SLASH_" }, { '\\', "_BSLASH_" }, { '?', "_QMARK_" } @@ -40,6 +42,8 @@ namespace jank::runtime { "_RBRACE_", '}' }, { "_LBRACK_", '[' }, { "_RBRACK_", ']' }, + { "_LPAREN_", '(' }, + { "_RPAREN_", ')' }, { "_BSLASH_", '\\' }, { "_SQUOTE_", '\'' }, { "_DQUOTE_", '"' }, From 39ed6d1a34f391dd5e084a15114bdb9a53fa68c2 Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 20 Nov 2025 16:58:15 -0800 Subject: [PATCH 17/58] More cppgen fixes --- .../src/cpp/jank/analyze/processor.cpp | 4 -- .../src/cpp/jank/codegen/processor.cpp | 63 +++++++------------ 2 files changed, 23 insertions(+), 44 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 95900b8a8..6826c40df 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -3353,10 +3353,6 @@ namespace jank::analyze /* TODO: Error if it's static and non-primitive. */ type = Cpp::GetLValueReferenceType(type); } - if(Cpp::IsArrayType(Cpp::GetNonReferenceType(type))) - { - type = Cpp::GetPointerType(Cpp::GetArrayElementType(Cpp::GetNonReferenceType(type))); - } } else if(Cpp::IsEnumConstant(scope)) { diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index e302475d3..0c7d9ebe7 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1002,6 +1002,7 @@ namespace jank::codegen pair.first->to_string()) }; } + auto const local_type{ cpp_util::expression_type(pair.second) }; auto const &val_tmp(gen(pair.second, fn_arity)); auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); @@ -1012,7 +1013,6 @@ namespace jank::codegen * be able to, just copy stack-allocated C++ objects around willy nilly. */ if(expr->is_loop) { - auto const local_type{ cpp_util::expression_type(pair.second) }; if(cpp_util::is_any_object(local_type)) { util::format_to(body_buffer, @@ -1027,17 +1027,24 @@ namespace jank::codegen } else { - util::format_to(body_buffer, "{ auto &&{}({}); ", munged_name, val_tmp.unwrap().str(false)); + /* Local array refs should be turned into pointers so we can work with them more easily. */ + if(Cpp::IsArrayType(Cpp::GetNonReferenceType(local_type))) + { + util::format_to(body_buffer, + "{ {} {}({}); ", + cpp_util::get_qualified_type_name(Cpp::GetPointerType( + Cpp::GetArrayElementType(Cpp::GetNonReferenceType(local_type)))), + munged_name, + val_tmp.unwrap().str(false)); + } + else + { + util::format_to(body_buffer, + "{ auto &&{}({}); ", + munged_name, + val_tmp.unwrap().str(false)); + } } - - //auto const binding(local.unwrap().binding); - //if(!binding->needs_box && binding->has_boxed_usage) - //{ - // util::format_to(body_buffer, - // "auto const {}({});", - // detail::boxed_local_name(munged_name), - // val_tmp.unwrap().str(true)); - //} } if(expr->is_loop) @@ -1672,25 +1679,11 @@ namespace jank::codegen if(expr->arg_exprs.size() == 1) { - auto const arg_type{ cpp_util::expression_type(expr->arg_exprs[0]) }; - if(Cpp::IsArrayType(Cpp::GetNonReferenceType(arg_type))) - { - util::format_to(body_buffer, - "auto &&{}( {} static_cast<{}>({}) );", - ret_tmp, - cpp_util::operator_name(static_cast(expr->op)).unwrap(), - cpp_util::get_qualified_type_name( - Cpp::GetPointerType(Cpp::GetArrayElementType(arg_type))), - arg_tmps[0].str(false)); - } - else - { - util::format_to(body_buffer, - "auto &&{}( {}{} );", - ret_tmp, - cpp_util::operator_name(static_cast(expr->op)).unwrap(), - arg_tmps[0].str(false)); - } + util::format_to(body_buffer, + "auto &&{}( {}{} );", + ret_tmp, + cpp_util::operator_name(static_cast(expr->op)).unwrap(), + arg_tmps[0].str(false)); } else { @@ -1905,16 +1898,6 @@ namespace jank::codegen "{} const {};", detail::gen_constant_type(v.second.data, true), runtime::munge(v.second.native_name)); - - //if(v.second.unboxed_native_name.is_some()) - //{ - // util::format_to(header_buffer, - // "static constexpr {} const {}{ ", - // detail::gen_constant_type(v.second.data, false), - // runtime::munge(v.second.unboxed_native_name.unwrap())); - // detail::gen_constant(v.second.data, header_buffer, false); - // util::format_to(header_buffer, "};"); - //} } /* TODO: More useful types here. */ From c571a7710c23752162614b1f4ecee444491eb0be Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 21 Nov 2025 11:18:43 -0800 Subject: [PATCH 18/58] More cppgen fixes --- compiler+runtime/include/cpp/jank/prelude.hpp | 2 ++ .../src/cpp/jank/codegen/processor.cpp | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/prelude.hpp b/compiler+runtime/include/cpp/jank/prelude.hpp index 64789c2df..6fd3acb56 100644 --- a/compiler+runtime/include/cpp/jank/prelude.hpp +++ b/compiler+runtime/include/cpp/jank/prelude.hpp @@ -10,3 +10,5 @@ #include #include #include + +extern "C" void *jank_unbox(char const *type, void *o); diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 0c7d9ebe7..e7e4f60e5 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1677,13 +1677,19 @@ namespace jank::codegen arg_tmps.emplace_back(gen(arg_expr, arity).unwrap()); } + auto const op_name{ cpp_util::operator_name(static_cast(expr->op)).unwrap() }; + if(expr->arg_exprs.size() == 1) + { + util::format_to(body_buffer, "auto &&{}( {}{} );", ret_tmp, op_name, arg_tmps[0].str(false)); + } + else if(op_name == "aget") { util::format_to(body_buffer, - "auto &&{}( {}{} );", + "auto &&{}( {}[{}] );", ret_tmp, - cpp_util::operator_name(static_cast(expr->op)).unwrap(), - arg_tmps[0].str(false)); + arg_tmps[0].str(false), + arg_tmps[1].str(false)); } else { @@ -1691,7 +1697,7 @@ namespace jank::codegen "auto &&{}( {} {} {} );", ret_tmp, arg_tmps[0].str(false), - cpp_util::operator_name(static_cast(expr->op)).unwrap(), + op_name, arg_tmps[1].str(false)); } @@ -1734,13 +1740,15 @@ namespace jank::codegen { auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_unbox")) }; auto value_tmp{ gen(expr->value_expr, arity) }; + auto const type_name{ cpp_util::get_qualified_type_name(expr->type) }; util::format_to(body_buffer, "auto {}{ " - "static_cast<{}>(jank::runtime::try_object({})-" - ">data.data) };", + "static_cast<{}>(jank_unbox(\"{}\", {}.data)" + ") };", ret_tmp, - Cpp::GetTypeAsString(expr->type), + type_name, + type_name, value_tmp.unwrap().str(false)); if(expr->position == expression_position::tail) From d4cc3aaa1d70a10f0978ea8d708d398cce917f22 Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 21 Nov 2025 11:31:49 -0800 Subject: [PATCH 19/58] Don't wrap cpp_raw during eval --- compiler+runtime/src/cpp/jank/evaluate.cpp | 3 ++- compiler+runtime/src/cpp/jank/jit/processor.cpp | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/evaluate.cpp b/compiler+runtime/src/cpp/jank/evaluate.cpp index bf0abb321..83837010f 100644 --- a/compiler+runtime/src/cpp/jank/evaluate.cpp +++ b/compiler+runtime/src/cpp/jank/evaluate.cpp @@ -723,7 +723,8 @@ namespace jank::evaluate object_ref eval(expr::cpp_raw_ref const expr) { - return dynamic_call(eval(wrap_expression(expr, "cpp_raw", {}))); + __rt_ctx->jit_prc.eval_string(expr->code); + return runtime::jank_nil; } object_ref eval(expr::cpp_type_ref const) diff --git a/compiler+runtime/src/cpp/jank/jit/processor.cpp b/compiler+runtime/src/cpp/jank/jit/processor.cpp index eebf61ffd..7030654ed 100644 --- a/compiler+runtime/src/cpp/jank/jit/processor.cpp +++ b/compiler+runtime/src/cpp/jank/jit/processor.cpp @@ -230,8 +230,11 @@ namespace jank::jit profile::timer const timer{ "jit eval_string" }; //util::println("// eval_string:\n{}\n", s); auto err(interpreter->ParseAndExecute({ s.data(), s.size() })); - /* TODO: Throw on errors. */ - llvm::logAllUnhandledErrors(std::move(err), llvm::errs(), "error: "); + if(err) + { + llvm::logAllUnhandledErrors(std::move(err), llvm::errs(), "error: "); + throw std::runtime_error{ "Failed to evaluate C++ code." }; + } register_jit_stack_frames(); } From 02032b958475e9fe05cc4b6f3c700fdc2c9bffd4 Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 21 Nov 2025 12:25:10 -0800 Subject: [PATCH 20/58] More cppgen fixes --- .../include/cpp/jank/codegen/processor.hpp | 1 + .../src/cpp/jank/codegen/processor.cpp | 105 +++++++++--------- .../pass-packing-exceeds-max-params.jank | 4 +- 3 files changed, 54 insertions(+), 56 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index 05cdb814f..a8832e187 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -101,6 +101,7 @@ namespace jank::codegen jtl::option gen(analyze::expr::call_ref const, analyze::expr::function_arity const &); jtl::option gen(analyze::expr::primitive_literal_ref const, analyze::expr::function_arity const &); + jtl::option gen(analyze::expr::list_ref const, analyze::expr::function_arity const &); jtl::option gen(analyze::expr::vector_ref const, analyze::expr::function_arity const &); jtl::option gen(analyze::expr::map_ref const, analyze::expr::function_arity const &); jtl::option gen(analyze::expr::set_ref const, analyze::expr::function_arity const &); diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index e7e4f60e5..e89b0a54e 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -589,22 +589,11 @@ namespace jank::codegen "auto const {}(jank::runtime::dynamic_call({}", ret_tmp, source_tmp); - for(size_t i{}; i < runtime::max_params && i < arg_tmps.size(); ++i) + for(size_t i{}; i < arg_tmps.size(); ++i) { util::format_to(body_buffer, ", {}", arg_tmps[i].str(true)); } - if(runtime::max_params < arg_tmps.size()) - { - util::format_to( - body_buffer, - ", jank::runtime::make_box(std::in_place"); - for(size_t i{ runtime::max_params }; i < arg_tmps.size(); ++i) - { - util::format_to(body_buffer, ", {}", arg_tmps[i].str(true)); - } - util::format_to(body_buffer, ")"); - } util::format_to(body_buffer, "));"); } @@ -654,6 +643,42 @@ namespace jank::codegen } } + jtl::option + processor::gen(analyze::expr::list_ref const expr, analyze::expr::function_arity const &fn_arity) + { + native_vector data_tmps; + data_tmps.reserve(expr->data_exprs.size()); + for(auto const &data_expr : expr->data_exprs) + { + data_tmps.emplace_back(gen(data_expr, fn_arity).unwrap()); + } + + auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("list"))); + util::format_to(body_buffer, + "auto const {}(jank::runtime::make_box(", + ret_tmp); + if(expr->meta.is_some()) + { + detail::gen_constant(expr->meta.unwrap(), body_buffer, true); + util::format_to(body_buffer, ", "); + } + util::format_to(body_buffer, "std::in_place "); + for(auto const &tmp : data_tmps) + { + util::format_to(body_buffer, ", "); + util::format_to(body_buffer, "{}", tmp.str(true)); + } + util::format_to(body_buffer, "));"); + + if(expr->position == analyze::expression_position::tail) + { + util::format_to(body_buffer, "return {};", ret_tmp); + return none; + } + + return ret_tmp; + } + jtl::option processor::gen(analyze::expr::vector_ref const expr, analyze::expr::function_arity const &fn_arity) { @@ -951,11 +976,10 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::let_ref const expr, analyze::expr::function_arity const &fn_arity) { - handle const ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")), - expr->needs_box }; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")) }; bool used_option{}; - if(expr->needs_box) + //if(expr->needs_box) { /* TODO: The type may not be default constructible so this may fail. We likely * want an array the same size as the desired type. When we have the last expression, @@ -972,26 +996,16 @@ namespace jank::codegen if(cpp_util::is_untyped_object(last_expr_type)) { type_name = "object_ref"; - util::format_to(body_buffer, "{} {}{ }; {", type_name, ret_tmp.str(expr->needs_box)); + util::format_to(body_buffer, "{} {}{ }; {", type_name, ret_tmp); } else { used_option = true; type_name = Cpp::GetTypeAsString(Cpp::GetNonReferenceType(last_expr_type)); /* TODO: Test for this with something non-default constructible. */ - util::format_to(body_buffer, - "jtl::option<{}> {}{ }; {", - type_name, - ret_tmp.str(expr->needs_box)); + util::format_to(body_buffer, "jtl::option<{}> {}{ }; {", type_name, ret_tmp); } } - else - { - util::format_to(body_buffer, - "auto const {}([&](){}{", - ret_tmp.str(expr->needs_box), - (expr->needs_box ? "-> object_ref" : "")); - } for(auto const &pair : expr->pairs) { @@ -1059,22 +1073,15 @@ namespace jank::codegen /* We ignore all values but the last. */ if(++it == expr->body->values.end() && val_tmp.is_some()) { - if(expr->needs_box) - { - /* The last expression tmp needs to be movable. */ - util::format_to(body_buffer, - "{} = std::move({});", - ret_tmp.str(true), - val_tmp.unwrap().str(expr->needs_box)); + /* The last expression tmp needs to be movable. */ + util::format_to(body_buffer, + "{} = std::move({});", + ret_tmp, + val_tmp.unwrap().str(expr->needs_box)); - if(expr->is_loop) - { - util::format_to(body_buffer, " break;"); - } - } - else + if(expr->is_loop) { - util::format_to(body_buffer, "return {};", val_tmp.unwrap().str(expr->needs_box)); + util::format_to(body_buffer, " break;"); } } } @@ -1089,25 +1096,15 @@ namespace jank::codegen util::format_to(body_buffer, "}"); } - if(expr->needs_box) - { - util::format_to(body_buffer, "}"); - } - else - { - util::format_to(body_buffer, "}());"); - } + util::format_to(body_buffer, "}"); if(expr->position == analyze::expression_position::tail) { - util::format_to(body_buffer, - "return {}{};", - ret_tmp.str(expr->needs_box), - (used_option ? ".unwrap()" : "")); + util::format_to(body_buffer, "return {}{};", ret_tmp, (used_option ? ".unwrap()" : "")); return none; } - return util::format("{}{}", ret_tmp.str(expr->needs_box), (used_option ? ".unwrap()" : "")); + return util::format("{}{}", ret_tmp, (used_option ? ".unwrap()" : "")); } jtl::option diff --git a/compiler+runtime/test/jank/form/fn/arity/variadic/pass-packing-exceeds-max-params.jank b/compiler+runtime/test/jank/form/fn/arity/variadic/pass-packing-exceeds-max-params.jank index 2986e57b1..eac3700fa 100644 --- a/compiler+runtime/test/jank/form/fn/arity/variadic/pass-packing-exceeds-max-params.jank +++ b/compiler+runtime/test/jank/form/fn/arity/variadic/pass-packing-exceeds-max-params.jank @@ -1,5 +1,5 @@ (def variadic - (fn* + (fn* variadic ([& args] args))) (assert (= (variadic 1 2 3 4 5 6 7 8 9) [1 2 3 4 5 6 7 8 9])) (assert (= (variadic 1 2 3 4 5 6 7 8 9 10) [1 2 3 4 5 6 7 8 9 10])) @@ -8,7 +8,7 @@ ; This will be code-generated, not evaluated. (def foo - (fn* [] + (fn* foo [] (assert (= (variadic 1 2 3 4 5 6 7 8 9 10 11) [1 2 3 4 5 6 7 8 9 10 11])) (assert (= (variadic 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) [1 2 3 4 5 6 7 8 9 10 11 12 13 14 15])))) (foo) From 03ab3445c26e4db8560661e892c5c79d427dda4b Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 21 Nov 2025 15:23:15 -0800 Subject: [PATCH 21/58] Implement cppgen for letfn --- .../src/cpp/jank/analyze/processor.cpp | 4 +- .../src/cpp/jank/codegen/processor.cpp | 155 +++++++++++++----- 2 files changed, 117 insertions(+), 42 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 6826c40df..7b719cd29 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -2259,7 +2259,7 @@ namespace jank::analyze /* All bindings in a letfn appear simultaneously and may be mutually recursive. * This makes creating a letfn locals frame a bit more involved than let, where locals - * are introduced left-to-right. For example, each binding in (letfn [(a [] b) (b [] a)]) + * are introduced left-to-right. For example, each binding in (letfn [(a [] b) (b [] a)]) * requires the other to be in scope in order to be analyzed. * * We tackle this in two steps. First, we create empty local bindings for all names. @@ -2308,7 +2308,7 @@ namespace jank::analyze /* Populate the local frame we prepared for sym in the previous loop with its binding. */ auto it(ret->pairs.emplace_back(sym, fexpr)); - auto local(ret->frame->locals.find(sym)->second); + auto &local(ret->frame->locals.find(sym)->second); local.value_expr = some(it.second); local.needs_box = it.second->needs_box; } diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index e89b0a54e..f1c04f040 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -906,12 +906,6 @@ namespace jank::codegen { auto const &pair{ let->pairs[i] }; auto const local(expr->frame->find_local_or_capture(pair.first)); - if(local.is_none()) - { - throw std::runtime_error{ util::format("ICE: unable to find local: {}", - pair.first->to_string()) }; - } - auto const &local_name(runtime::munge(local.unwrap().binding->native_name)); auto const &val_name(arg_tmp_it->str(true)); @@ -979,43 +973,24 @@ namespace jank::codegen auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")) }; bool used_option{}; - //if(expr->needs_box) + auto const last_expr_type{ cpp_util::expression_type( + expr->body->values[expr->body->values.size() - 1]) }; + + auto const &type_name{ cpp_util::get_qualified_type_name( + Cpp::GetNonReferenceType(last_expr_type)) }; + if(cpp_util::is_any_object(last_expr_type)) { - /* TODO: The type may not be default constructible so this may fail. We likely - * want an array the same size as the desired type. When we have the last expression, - * we can then do a placement new with the move ctor. - * - * Also add a test for this. */ - auto const last_expr_type{ cpp_util::expression_type( - expr->body->values[expr->body->values.size() - 1]) }; - - jtl::immutable_string type_name; - /* In analysis, we treat untyped objects as object*, since that's easier for IR. - * However, for C++, we want to normalize that to object_ref to take full advantage - * of richer types. */ - if(cpp_util::is_untyped_object(last_expr_type)) - { - type_name = "object_ref"; - util::format_to(body_buffer, "{} {}{ }; {", type_name, ret_tmp); - } - else - { - used_option = true; - type_name = Cpp::GetTypeAsString(Cpp::GetNonReferenceType(last_expr_type)); - /* TODO: Test for this with something non-default constructible. */ - util::format_to(body_buffer, "jtl::option<{}> {}{ }; {", type_name, ret_tmp); - } + util::format_to(body_buffer, "{} {}{ }; {", type_name, ret_tmp); + } + else + { + used_option = true; + util::format_to(body_buffer, "jtl::option<{}> {}{ }; {", type_name, ret_tmp); } for(auto const &pair : expr->pairs) { auto const local(expr->frame->find_local_or_capture(pair.first)); - if(local.is_none()) - { - throw std::runtime_error{ util::format("ICE: unable to find local: {}", - pair.first->to_string()) }; - } - auto const local_type{ cpp_util::expression_type(pair.second) }; auto const &val_tmp(gen(pair.second, fn_arity)); auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); @@ -1108,9 +1083,108 @@ namespace jank::codegen } jtl::option - processor::gen(analyze::expr::letfn_ref const, analyze::expr::function_arity const &) + processor::gen(analyze::expr::letfn_ref const expr, analyze::expr::function_arity const &fn_arity) { - return none; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")) }; + bool used_option{}; + + auto const last_expr_type{ cpp_util::expression_type( + expr->body->values[expr->body->values.size() - 1]) }; + + auto const &type_name{ cpp_util::get_qualified_type_name( + Cpp::GetNonReferenceType(last_expr_type)) }; + if(cpp_util::is_any_object(last_expr_type)) + { + util::format_to(body_buffer, "{} {}{ }; {", type_name, ret_tmp); + } + else + { + used_option = true; + util::format_to(body_buffer, "jtl::option<{}> {}{ }; {", type_name, ret_tmp); + } + + /* We don't handle shadowed bindings very well, so we can run into problems where our + * codegen doesn't work. For letfn, we detect shadowed bindings and get around potential + * assignment issues by just using an object_ref. This can be removed once we + * properly give shadowed bindings individual local_binding entries or we have some other + * mechanism for tracking them. */ + bool has_shadowed_bindings{}; + native_set seen_names; + for(auto const &pair : expr->pairs) + { + auto const local(expr->frame->find_local_or_capture(pair.first)); + auto const &name{ local.unwrap().binding->native_name }; + if(seen_names.contains(name)) + { + has_shadowed_bindings = true; + break; + } + seen_names.emplace(name); + } + + for(auto const &pair : expr->pairs) + { + auto const local(expr->frame->find_local_or_capture(pair.first)); + auto const val_expr(llvm::cast(pair.second.data)); + auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); + auto const type_name{ ( + has_shadowed_bindings + ? "jank::runtime::object_ref" + : util::format("jank::runtime::oref<{}>", runtime::munge(val_expr->unique_name))) }; + util::format_to(body_buffer, "{ {} {};", type_name, munged_name); + } + + for(auto const &pair : expr->pairs) + { + auto const local(expr->frame->find_local_or_capture(pair.first)); + auto const &val_tmp(gen(pair.second, fn_arity)); + auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); + + util::format_to(body_buffer, "{} = {}; ", munged_name, val_tmp.unwrap().str(false)); + } + + for(auto const &pair : expr->pairs) + { + auto const local(expr->frame->find_local_or_capture(pair.first)); + + auto const &munged_name(runtime::munge(local.unwrap().binding->native_name)); + auto const val_expr(llvm::cast(pair.second.data)); + for(auto const &capture_pair : val_expr->captures()) + { + auto const &capture_name(runtime::munge(capture_pair.second->native_name)); + util::format_to(body_buffer, "{}->{} = {}; ", munged_name, capture_name, capture_name); + } + } + + for(auto it(expr->body->values.begin()); it != expr->body->values.end();) + { + auto const &val_tmp(gen(*it, fn_arity)); + + /* We ignore all values but the last. */ + if(++it == expr->body->values.end() && val_tmp.is_some()) + { + /* The last expression tmp needs to be movable. */ + util::format_to(body_buffer, + "{} = std::move({});", + ret_tmp, + val_tmp.unwrap().str(expr->needs_box)); + } + } + for(auto const &_ : expr->pairs) + { + static_cast(_); + util::format_to(body_buffer, "}"); + } + + util::format_to(body_buffer, "}"); + + if(expr->position == analyze::expression_position::tail) + { + util::format_to(body_buffer, "return {}{};", ret_tmp, (used_option ? ".unwrap()" : "")); + return none; + } + + return util::format("{}{}", ret_tmp, (used_option ? ".unwrap()" : "")); } jtl::option @@ -1914,8 +1988,9 @@ namespace jank::codegen } used_captures.emplace(v.first->to_hash()); + /* Captures aren't const since they could be late-assigned, in the case of a letfn. */ util::format_to(header_buffer, - "jank::runtime::object_ref const {};", + "jank::runtime::object_ref {};", runtime::munge(v.second.native_name)); } } From c5bc9efd1e090642375a5745846c76e867bf61cb Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 21 Nov 2025 18:30:10 -0800 Subject: [PATCH 22/58] Optimize core libs during phase 1 --- compiler+runtime/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler+runtime/CMakeLists.txt b/compiler+runtime/CMakeLists.txt index 2809c8f42..f2ade1f6f 100644 --- a/compiler+runtime/CMakeLists.txt +++ b/compiler+runtime/CMakeLists.txt @@ -888,7 +888,7 @@ add_custom_command( DEPENDS ${CMAKE_BINARY_DIR}/jank-phase-1 ${CMAKE_SOURCE_DIR}/src/jank/clojure/core.jank ${jank_incremental_pch_flag} OUTPUT ${jank_core_libraries_flag} BYPRODUCTS ${jank_clojure_core_o} - COMMAND ${CMAKE_BINARY_DIR}/jank-phase-1 compile-module -o ${jank_clojure_core_o} clojure.core + COMMAND ${CMAKE_BINARY_DIR}/jank-phase-1 compile-module -O3 -o ${jank_clojure_core_o} clojure.core COMMAND touch ${jank_core_libraries_flag} ) add_custom_target( From e026ce961076c160e517a11b1cc6cc7ce62b641a Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 21 Nov 2025 18:37:38 -0800 Subject: [PATCH 23/58] Add cppgen for case --- compiler+runtime/include/cpp/jank/prelude.hpp | 3 +- compiler+runtime/src/cpp/jank/c_api.cpp | 4 +- .../src/cpp/jank/codegen/processor.cpp | 67 ++++++++++++++++++- .../test/jank/form/case/pass-real.jank | 13 ++-- 4 files changed, 73 insertions(+), 14 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/prelude.hpp b/compiler+runtime/include/cpp/jank/prelude.hpp index 6fd3acb56..0049b7bd5 100644 --- a/compiler+runtime/include/cpp/jank/prelude.hpp +++ b/compiler+runtime/include/cpp/jank/prelude.hpp @@ -10,5 +10,4 @@ #include #include #include - -extern "C" void *jank_unbox(char const *type, void *o); +#include diff --git a/compiler+runtime/src/cpp/jank/c_api.cpp b/compiler+runtime/src/cpp/jank/c_api.cpp index 80e0e7d9b..faed7c122 100644 --- a/compiler+runtime/src/cpp/jank/c_api.cpp +++ b/compiler+runtime/src/cpp/jank/c_api.cpp @@ -940,8 +940,8 @@ extern "C" { if(o_obj->type == object_type::integer) { - /* We don't hash the integer if it's an int32 value. This is to be consistent with how keys are hashed in jank's - * case macro. */ + /* We don't hash the integer if it's within an i32 value. + * This is to be consistent with how keys are hashed in jank's case macro. */ integer = (integer >= std::numeric_limits::min() && integer <= std::numeric_limits::max()) ? integer diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index f1c04f040..8e7c2ceed 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -186,6 +186,25 @@ namespace jank::codegen "f64>({}))", typed_o->data); } + else if constexpr(std::same_as) + { + util::format_to(buffer, + "jank::runtime::make_box(\"{}\")", + typed_o->to_string()); + } + else if constexpr(std::same_as) + { + util::format_to(buffer, + "jank::runtime::make_box(\"{}\")", + typed_o->to_string()); + } + else if constexpr(std::same_as) + { + util::format_to(buffer, + "jank::runtime::obj::ratio::create({}, {})", + typed_o->data.numerator, + typed_o->data.denominator); + } else if constexpr(std::same_as) { if(typed_o->meta.is_some()) @@ -1085,7 +1104,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::letfn_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")) }; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("letfn")) }; bool used_option{}; auto const last_expr_type{ cpp_util::expression_type( @@ -1347,9 +1366,51 @@ namespace jank::codegen } jtl::option - processor::gen(analyze::expr::case_ref const, analyze::expr::function_arity const &) + processor::gen(analyze::expr::case_ref const expr, analyze::expr::function_arity const &fn_arity) { - return none; + auto const is_tail{ expr->position == analyze::expression_position::tail }; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("case")) }; + + util::format_to(body_buffer, "jank::runtime::object_ref {}{ };", ret_tmp); + + auto const &value_tmp{ gen(expr->value_expr, fn_arity) }; + + util::format_to(body_buffer, + "switch(jank_shift_mask_case_integer({}.erase(), {}, {})) {", + value_tmp.unwrap().str(true), + expr->shift, + expr->mask); + + jank_debug_assert(expr->keys.size() == expr->exprs.size()); + for(usize i{}; i < expr->keys.size(); ++i) + { + util::format_to(body_buffer, "case {}: {", expr->keys[i]); + + auto const &case_tmp{ gen(expr->exprs[i], fn_arity) }; + if(!is_tail) + { + util::format_to(body_buffer, "{} = {};", ret_tmp, case_tmp.unwrap().str(true)); + } + util::format_to(body_buffer, "break; }"); + } + + util::format_to(body_buffer, "default: {"); + + auto const &default_tmp{ gen(expr->default_expr, fn_arity) }; + if(!is_tail) + { + util::format_to(body_buffer, "{} = {};", ret_tmp, default_tmp.unwrap().str(true)); + } + + util::format_to(body_buffer, "} }"); + + if(is_tail) + { + util::format_to(body_buffer, "return {};", ret_tmp); + return none; + } + + return ret_tmp; } jtl::option processor::gen(expr::cpp_raw_ref const expr, expr::function_arity const &) diff --git a/compiler+runtime/test/jank/form/case/pass-real.jank b/compiler+runtime/test/jank/form/case/pass-real.jank index efb4ac9b3..4ebeabfb6 100644 --- a/compiler+runtime/test/jank/form/case/pass-real.jank +++ b/compiler+runtime/test/jank/form/case/pass-real.jank @@ -1,9 +1,8 @@ -(assert - (= - (case 3.14 3.14 :pi - 2.71 :e - 1.61 :phi - :default) - :pi)) +(assert (= (case 3.14 + 3.14 :pi + 2.71 :e + 1.61 :phi + :default) + :pi)) :success From 8796a4e73145f0568fe904ac6f57dd9a9ab14ad7 Mon Sep 17 00:00:00 2001 From: jeaye Date: Sat, 22 Nov 2025 12:22:05 -0800 Subject: [PATCH 24/58] Skip static member test for now It's blocked on this Clang bug: https://github.com/llvm/llvm-project/issues/146956 --- compiler+runtime/src/cpp/jank/codegen/processor.cpp | 2 +- .../test/jank/cpp/member/{pass-static.jank => skip-static.jank} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename compiler+runtime/test/jank/cpp/member/{pass-static.jank => skip-static.jank} (100%) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 8e7c2ceed..28c8ed371 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1455,7 +1455,7 @@ namespace jank::codegen return util::format("{}", val); } - auto tmp{ Cpp::GetQualifiedCompleteName(expr->scope) }; + auto const tmp{ Cpp::GetQualifiedCompleteName(expr->scope) }; if(expr->position == expression_position::tail) { diff --git a/compiler+runtime/test/jank/cpp/member/pass-static.jank b/compiler+runtime/test/jank/cpp/member/skip-static.jank similarity index 100% rename from compiler+runtime/test/jank/cpp/member/pass-static.jank rename to compiler+runtime/test/jank/cpp/member/skip-static.jank From 9c3de6c977d4a24c83e597474f8b798771f9c2c6 Mon Sep 17 00:00:00 2001 From: jeaye Date: Sat, 22 Nov 2025 13:09:25 -0800 Subject: [PATCH 25/58] Detect unsupported static member access --- compiler+runtime/include/cpp/jank/error.hpp | 3 ++ .../include/cpp/jank/error/analyze.hpp | 3 ++ .../src/cpp/jank/analyze/processor.cpp | 18 ++++--- compiler+runtime/src/cpp/jank/error.cpp | 2 + .../src/cpp/jank/error/analyze.cpp | 47 +++++++++++-------- .../jank/cpp/global-value/pass-static.jank | 12 ++--- 6 files changed, 53 insertions(+), 32 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/error.hpp b/compiler+runtime/include/cpp/jank/error.hpp index 746dd0f40..accc37a99 100644 --- a/compiler+runtime/include/cpp/jank/error.hpp +++ b/compiler+runtime/include/cpp/jank/error.hpp @@ -103,6 +103,7 @@ namespace jank::error analyze_invalid_cpp_member_access, analyze_invalid_cpp_capture, analyze_mismatched_if_types, + analyze_known_issue, internal_analyze_failure, internal_codegen_failure, @@ -301,6 +302,8 @@ namespace jank::error return "analyze/invalid-cpp-capture"; case kind::analyze_mismatched_if_types: return "analyze/mismatched-if-types"; + case kind::analyze_known_issue: + return "analyze/known-issue"; case kind::internal_analyze_failure: return "internal/analysis-failure"; diff --git a/compiler+runtime/include/cpp/jank/error/analyze.hpp b/compiler+runtime/include/cpp/jank/error/analyze.hpp index 280d4312e..0c5dfda7f 100644 --- a/compiler+runtime/include/cpp/jank/error/analyze.hpp +++ b/compiler+runtime/include/cpp/jank/error/analyze.hpp @@ -150,6 +150,9 @@ namespace jank::error error_ref analyze_invalid_cpp_member_access(jtl::immutable_string const &message, read::source const &source, runtime::object_ref expansion); + error_ref analyze_known_issue(jtl::immutable_string const &message, + read::source const &source, + runtime::object_ref expansion); error_ref internal_analyze_failure(jtl::immutable_string const &message, runtime::object_ref expansion); error_ref internal_analyze_failure(jtl::immutable_string const &message, diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 7b719cd29..5ead51ee5 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -3345,12 +3345,8 @@ namespace jank::analyze if(Cpp::IsVariable(scope)) { vk = expr::cpp_value::value_kind::variable; - /* TODO: A Clang bug prevents us from supporting references to static members. - * https://github.com/llvm/llvm-project/issues/146956 - */ - if(!Cpp::IsStaticDatamember(scope) && !Cpp::IsPointerType(type)) + if(!Cpp::IsPointerType(type)) { - /* TODO: Error if it's static and non-primitive. */ type = Cpp::GetLValueReferenceType(type); } } @@ -4355,8 +4351,18 @@ namespace jank::analyze ->add_usage(read::parse::reparse_nth(l, 0)); } + if(Cpp::IsStaticDatamember(member_scope)) + { + return error::analyze_known_issue( + "A blocking Clang bug prevents access to static members in some scenarios. See " + "https://github.com/llvm/llvm-project/issues/146956 for details.", + object_source(member), + latest_expansion(macro_expansions)) + ->add_usage(read::parse::reparse_nth(l, 0)); + } + val->val_kind = expr::cpp_value::value_kind::variable; - val->type = Cpp::GetTypeFromScope(member_scope); + val->type = Cpp::GetLValueReferenceType(Cpp::GetTypeFromScope(member_scope)); val->scope = member_scope; return val; } diff --git a/compiler+runtime/src/cpp/jank/error.cpp b/compiler+runtime/src/cpp/jank/error.cpp index 732e4d384..45c131497 100644 --- a/compiler+runtime/src/cpp/jank/error.cpp +++ b/compiler+runtime/src/cpp/jank/error.cpp @@ -184,6 +184,8 @@ namespace jank::error return "Invalid C++ delete."; case kind::analyze_invalid_cpp_member_access: return "Invalid C++ member access."; + case kind::analyze_known_issue: + return "Known issue."; case kind::internal_analyze_failure: return "Internal analysis failure."; diff --git a/compiler+runtime/src/cpp/jank/error/analyze.cpp b/compiler+runtime/src/cpp/jank/error/analyze.cpp index 53c0ae82c..f6e181327 100644 --- a/compiler+runtime/src/cpp/jank/error/analyze.cpp +++ b/compiler+runtime/src/cpp/jank/error/analyze.cpp @@ -153,7 +153,7 @@ namespace jank::error note &&extra, runtime::object_ref const expansion) { - return make_error(kind::analyze_invalid_try, message, source, std::move(extra), expansion); + return make_error(kind::analyze_invalid_try, message, source, jtl::move(extra), expansion); } error_ref analyze_unresolved_var(jtl::immutable_string const &message, @@ -229,28 +229,28 @@ namespace jank::error error_ref analyze_invalid_cpp_operator_call(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_operator_call, message, source, expansion); } error_ref analyze_invalid_cpp_constructor_call(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_constructor_call, message, source, expansion); } error_ref analyze_invalid_cpp_member_call(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_member_call, message, source, expansion); } error_ref analyze_invalid_cpp_capture(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_capture, message, @@ -261,109 +261,116 @@ namespace jank::error error_ref analyze_mismatched_if_types(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_mismatched_if_types, message, source, expansion); } error_ref analyze_invalid_cpp_function_call(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_function_call, message, source, expansion); } error_ref analyze_invalid_cpp_call(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_call, message, source, expansion); } error_ref analyze_invalid_cpp_conversion(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_conversion, message, source, expansion); } error_ref analyze_invalid_cpp_symbol(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_symbol, message, source, expansion); } error_ref analyze_unresolved_cpp_symbol(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_unresolved_cpp_symbol, message, source, expansion); } error_ref analyze_invalid_cpp_raw(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_raw, message, source, expansion); } error_ref analyze_invalid_cpp_type(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_type, message, source, expansion); } error_ref analyze_invalid_cpp_value(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_value, message, source, expansion); } error_ref analyze_invalid_cpp_cast(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_cast, message, source, expansion); } error_ref analyze_invalid_cpp_box(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_box, message, source, expansion); } error_ref analyze_invalid_cpp_unbox(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_unbox, message, source, expansion); } error_ref analyze_invalid_cpp_new(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_new, message, source, expansion); } error_ref analyze_invalid_cpp_delete(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_delete, message, source, expansion); } error_ref analyze_invalid_cpp_member_access(jtl::immutable_string const &message, read::source const &source, - runtime::object_ref expansion) + runtime::object_ref const expansion) { return make_error(kind::analyze_invalid_cpp_member_access, message, source, expansion); } + error_ref analyze_known_issue(jtl::immutable_string const &message, + read::source const &source, + runtime::object_ref const expansion) + { + return make_error(kind::analyze_known_issue, message, source, expansion); + } + error_ref internal_analyze_failure(jtl::immutable_string const &message, runtime::object_ref const expansion) { diff --git a/compiler+runtime/test/jank/cpp/global-value/pass-static.jank b/compiler+runtime/test/jank/cpp/global-value/pass-static.jank index ccbe5224a..54a3ab721 100644 --- a/compiler+runtime/test/jank/cpp/global-value/pass-static.jank +++ b/compiler+runtime/test/jank/cpp/global-value/pass-static.jank @@ -1,10 +1,10 @@ (cpp/raw "namespace jank::cpp::global_value::pass_static - { - struct foo - { - static int const boop{ 5 }; - }; - }") + { + struct foo + { + static int const boop{ 5 }; + }; + }") (if (= 5 cpp/jank.cpp.global_value.pass_static.foo.boop) :success :failure) From f97dc5a8228f07767ee277c8ad1e07806a32ef3e Mon Sep 17 00:00:00 2001 From: jeaye Date: Sat, 22 Nov 2025 13:09:55 -0800 Subject: [PATCH 26/58] Add another static test --- .../pass-static-non-copyable.jank | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 compiler+runtime/test/jank/cpp/global-value/pass-static-non-copyable.jank diff --git a/compiler+runtime/test/jank/cpp/global-value/pass-static-non-copyable.jank b/compiler+runtime/test/jank/cpp/global-value/pass-static-non-copyable.jank new file mode 100644 index 000000000..f01aca9d6 --- /dev/null +++ b/compiler+runtime/test/jank/cpp/global-value/pass-static-non-copyable.jank @@ -0,0 +1,24 @@ +(cpp/raw "namespace jank::cpp::global_value::pass_static_non_copyable + { + struct foo + { + foo() = delete; + foo(foo const&) = delete; + foo(foo &&) = delete; + foo(int i) + : data{ i } + { } + + int data{}; + }; + + struct bar + { + static foo const boop; + }; + foo const bar::boop{ 5 }; + }") +(let* [foo cpp/jank.cpp.global_value.pass_static_non_copyable.bar.boop] + (if (= 5 (cpp/.-data foo)) + :success + :failure)) From bd95aabb8a6ff2c90b9b56e108a74f9203c48726 Mon Sep 17 00:00:00 2001 From: jeaye Date: Sat, 22 Nov 2025 13:28:45 -0800 Subject: [PATCH 27/58] Fix tidy issues --- compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp | 2 +- compiler+runtime/src/cpp/jank/codegen/processor.cpp | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index e39af6c3b..654a4ef8a 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -3749,7 +3749,7 @@ namespace jank::codegen * as well, to detect cases where we're not restoring in the correct order. */ return { [stack_ptr, this]() { ssize found{ -1 }; - for(ssize i{}; i != static_cast(stack_saves.size()); ++i) + for(ssize i{}; std::cmp_not_equal(i, stack_saves.size()); ++i) { if(stack_saves[i].data == stack_ptr) { diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 28c8ed371..ea5b7b9d4 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1455,7 +1455,7 @@ namespace jank::codegen return util::format("{}", val); } - auto const tmp{ Cpp::GetQualifiedCompleteName(expr->scope) }; + auto tmp{ Cpp::GetQualifiedCompleteName(expr->scope) }; if(expr->position == expression_position::tail) { @@ -1529,7 +1529,7 @@ namespace jank::codegen util::format_to(body_buffer, "{}(", Cpp::GetQualifiedCompleteName(source->scope)); bool need_comma{}; - for(u8 arg_idx{}; arg_idx < expr->arg_exprs.size(); ++arg_idx) + for(usize arg_idx{}; arg_idx < expr->arg_exprs.size(); ++arg_idx) { auto const arg_expr{ expr->arg_exprs[arg_idx] }; auto const arg_type{ cpp_util::expression_type(arg_expr) }; @@ -2271,7 +2271,6 @@ namespace jank::codegen if(!generated_expression) { - jtl::immutable_string close = ")"; util::format_to(expression_buffer, "jank::runtime::make_box<{}>(", runtime::module::nest_native_ns(module_ns, runtime::munge(struct_name.name))); @@ -2327,7 +2326,7 @@ namespace jank::codegen } } - util::format_to(expression_buffer, "{}", close); + util::format_to(expression_buffer, ")"); generated_expression = true; } From d4153d20fbfb8859a8bdaf752220b462aeb7350e Mon Sep 17 00:00:00 2001 From: jeaye Date: Sat, 22 Nov 2025 23:02:37 -0800 Subject: [PATCH 28/58] Fix some string gc bits --- compiler+runtime/cmake/dependency/bdwgc.cmake | 10 ++++++++-- .../cpp/jank/runtime/convert/builtin.hpp | 10 ++++++++++ .../include/cpp/jtl/immutable_string.hpp | 20 ++++++++++--------- .../src/cpp/jank/jit/processor.cpp | 10 ++++++++-- .../src/cpp/jank/util/clang_format.cpp | 3 ++- 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/compiler+runtime/cmake/dependency/bdwgc.cmake b/compiler+runtime/cmake/dependency/bdwgc.cmake index 034bab9c6..0176e2692 100644 --- a/compiler+runtime/cmake/dependency/bdwgc.cmake +++ b/compiler+runtime/cmake/dependency/bdwgc.cmake @@ -2,8 +2,14 @@ set(CMAKE_C_FLAGS_OLD "${CMAKE_C_FLAGS}") set(CMAKE_CXX_FLAGS_OLD "${CMAKE_CXX_FLAGS}") set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS}) set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY}) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") + + if(NOT APPLE) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS") + endif() + set(BUILD_SHARED_LIBS OFF) set(CMAKE_CXX_CLANG_TIDY "") set(enable_cplusplus ON CACHE BOOL "Enable C++") diff --git a/compiler+runtime/include/cpp/jank/runtime/convert/builtin.hpp b/compiler+runtime/include/cpp/jank/runtime/convert/builtin.hpp index eb8745bc8..148f0f7c3 100644 --- a/compiler+runtime/include/cpp/jank/runtime/convert/builtin.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/convert/builtin.hpp @@ -35,10 +35,20 @@ namespace jank::runtime return const_cast(t); } + static constexpr object *into_object(object_ref const t) + { + return t.erase(); + } + static constexpr object *from_object(T t) { return const_cast(t); } + + static constexpr object *from_object(object_ref const t) + { + return t.erase(); + } }; /* Any typed object can convert to/from itself easily. */ diff --git a/compiler+runtime/include/cpp/jtl/immutable_string.hpp b/compiler+runtime/include/cpp/jtl/immutable_string.hpp index afc94dc41..13034501f 100644 --- a/compiler+runtime/include/cpp/jtl/immutable_string.hpp +++ b/compiler+runtime/include/cpp/jtl/immutable_string.hpp @@ -54,12 +54,10 @@ namespace jtl struct immutable_string { using value_type = char; - using allocator_type = jank::native_allocator; - using allocator_traits = std::allocator_traits; - using size_type = allocator_traits::size_type; using traits_type = std::char_traits; using pointer_type = value_type *; using const_pointer_type = value_type const *; + using size_type = usize; using iterator = pointer_type; using const_iterator = const_pointer_type; using reverse_iterator = std::reverse_iterator; @@ -705,7 +703,7 @@ namespace jtl * 3. As a large_storage instance, containing a pointer, size, and capacity */ /* NOLINTNEXTLINE(cppcoreguidelines-pro-type-member-init) */ - struct storage : allocator_type + struct storage { /* TODO: What if we store a max of 22 chars and dedicate a byte for flags with no masking? */ union { @@ -724,7 +722,7 @@ namespace jtl /* NOTE: No performance difference between if/switch here. */ if(get_category() == category::large_owned) { - allocator_traits::deallocate(store, store.large.data, store.large.size + 1); + GC_free(store.large.data); } } @@ -838,7 +836,8 @@ namespace jtl { jank_debug_assert(max_small_size < size); /* TODO: Apply gnu::malloc to this fn. */ - store.large.data = std::assume_aligned(store.allocate(size + 1)); + store.large.data = std::assume_aligned( + static_cast(GC_malloc_atomic(size + 1))); traits_type::copy(store.large.data, data, size); store.large.data[size] = 0; store.large.size = size; @@ -849,7 +848,8 @@ namespace jtl constexpr void init_large_fill(value_type const fill, u8 const size) noexcept { jank_debug_assert(max_small_size < size); - store.large.data = std::assume_aligned(store.allocate(size + 1)); + store.large.data = std::assume_aligned( + static_cast(GC_malloc_atomic(size + 1))); traits_type::assign(store.large.data, size, fill); store.large.data[size] = 0; store.large.size = size; @@ -864,7 +864,8 @@ namespace jtl { auto const size(lhs_size + rhs_size); jank_debug_assert(max_small_size < size); - store.large.data = std::assume_aligned(store.allocate(size + 1)); + store.large.data = std::assume_aligned( + static_cast(GC_malloc_atomic(size + 1))); traits_type::copy(store.large.data, lhs, lhs_size); traits_type::copy(store.large.data + lhs_size, rhs, rhs_size); store.large.data[size] = 0; @@ -878,7 +879,8 @@ namespace jtl { auto const size(std::distance(begin, end)); jank_debug_assert(max_small_size < size); - store.large.data = std::assume_aligned(store.allocate(size + 1)); + store.large.data = std::assume_aligned( + static_cast(GC_malloc_atomic(size + 1))); std::copy(begin, end, store.large.data); store.large.data[size] = 0; store.large.size = size; diff --git a/compiler+runtime/src/cpp/jank/jit/processor.cpp b/compiler+runtime/src/cpp/jank/jit/processor.cpp index 7030654ed..3001f093d 100644 --- a/compiler+runtime/src/cpp/jank/jit/processor.cpp +++ b/compiler+runtime/src/cpp/jank/jit/processor.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -152,6 +153,9 @@ namespace jank::jit args.emplace_back("-include-pch"); args.emplace_back(strdup(pch_path_str.c_str())); + args.emplace_back("-w"); + args.emplace_back("-Wno-c++11-narrowing"); + util::add_system_flags(args); /********* Every flag after this line is user-provided. *********/ @@ -228,8 +232,10 @@ namespace jank::jit void processor::eval_string(jtl::immutable_string const &s) const { profile::timer const timer{ "jit eval_string" }; - //util::println("// eval_string:\n{}\n", s); - auto err(interpreter->ParseAndExecute({ s.data(), s.size() })); + auto const &formatted{ s }; + //auto const &formatted{ util::format_cpp_source(s).expect_ok() }; + //util::println("// eval_string:\n{}\n", formatted); + auto err(interpreter->ParseAndExecute({ formatted.data(), formatted.size() })); if(err) { llvm::logAllUnhandledErrors(std::move(err), llvm::errs(), "error: "); diff --git a/compiler+runtime/src/cpp/jank/util/clang_format.cpp b/compiler+runtime/src/cpp/jank/util/clang_format.cpp index 820a2aaf9..87837b733 100644 --- a/compiler+runtime/src/cpp/jank/util/clang_format.cpp +++ b/compiler+runtime/src/cpp/jank/util/clang_format.cpp @@ -68,6 +68,7 @@ namespace jank::util { return err(llvm::toString(formatted_code.takeError())); } - return ok(jtl::immutable_string{ *formatted_code }); + jtl::immutable_string ret{ *formatted_code }; + return ok(ret); } } From 381a8bba4e3ca1551f9e14789606adba02ea84c8 Mon Sep 17 00:00:00 2001 From: jeaye Date: Mon, 24 Nov 2025 13:05:03 -0800 Subject: [PATCH 29/58] Bypass one layer of typedefs manually --- .../src/cpp/jank/analyze/cpp_util.cpp | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp b/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp index a16daf244..ff883c5a1 100644 --- a/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/cpp_util.cpp @@ -300,6 +300,26 @@ namespace jank::analyze::cpp_util } /* TODO: Handle typed object refs, too. */ + /* TODO: We probably want a recursive approach to this, for types and scopes. */ + auto const qual_type{ clang::QualType::getFromOpaquePtr(type) }; + if(auto const *alias{ + llvm::dyn_cast_or_null(qual_type.getTypePtrOrNull()) }; + alias) + { + if(auto const *alias_decl{ alias->getDecl() }; alias_decl) + { + auto alias_name{ alias_decl->getQualifiedNameAsString() }; + if(!alias_name.empty()) + { + if(Cpp::IsPointerType(type)) + { + alias_name += "*"; + } + return alias_name; + } + } + } + if(auto const scope{ Cpp::GetScopeFromType(type) }; scope) { auto name{ get_qualified_name(scope) }; From 7f5eb1cc07be5417291253809dff0d12726e0eb0 Mon Sep 17 00:00:00 2001 From: jeaye Date: Mon, 24 Nov 2025 13:12:28 -0800 Subject: [PATCH 30/58] Ensure no implicit conversion for instantiation --- .../cpp/constructor/complex/fail-template-instantiation.jank | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler+runtime/test/jank/cpp/constructor/complex/fail-template-instantiation.jank b/compiler+runtime/test/jank/cpp/constructor/complex/fail-template-instantiation.jank index 59b73e64a..9f60209bc 100644 --- a/compiler+runtime/test/jank/cpp/constructor/complex/fail-template-instantiation.jank +++ b/compiler+runtime/test/jank/cpp/constructor/complex/fail-template-instantiation.jank @@ -10,6 +10,6 @@ std::string a{}; }; }") -(let* [arg (cpp/int 5) +(let* [arg (cpp/int* cpp/nullptr) _ (cpp/jank.cpp.constructor.complex.fail_template_instantiation.foo arg)] :success) From a5fb045c0722678ccee0c2fd645320c435eecf62 Mon Sep 17 00:00:00 2001 From: jeaye Date: Mon, 24 Nov 2025 13:53:12 -0800 Subject: [PATCH 31/58] Fix tidy issues --- .github/workflows/build.yml | 9 +++++---- compiler+runtime/src/cpp/jank/util/clang_format.cpp | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e87c092ac..06e4f89dd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,10 +97,11 @@ jobs: analyze: on ## Debug + sanitization - - name: Ubuntu - address sanitizer - os: ubuntu-24.04 - build_type: Debug - sanitize: address + # TODO: Fix this. GC issue: https://github.com/bdwgc/bdwgc/issues/772 + #- name: Ubuntu - address sanitizer + # os: ubuntu-24.04 + # build_type: Debug + # sanitize: address - name: Ubuntu - undefined behavior sanitizer os: ubuntu-24.04 diff --git a/compiler+runtime/src/cpp/jank/util/clang_format.cpp b/compiler+runtime/src/cpp/jank/util/clang_format.cpp index 87837b733..07176c5cc 100644 --- a/compiler+runtime/src/cpp/jank/util/clang_format.cpp +++ b/compiler+runtime/src/cpp/jank/util/clang_format.cpp @@ -68,7 +68,7 @@ namespace jank::util { return err(llvm::toString(formatted_code.takeError())); } - jtl::immutable_string ret{ *formatted_code }; + jtl::immutable_string const ret{ *formatted_code }; return ok(ret); } } From e32634c87e1cb37317fee8753b69ef5c5428c5e8 Mon Sep 17 00:00:00 2001 From: jeaye Date: Mon, 24 Nov 2025 14:27:25 -0800 Subject: [PATCH 32/58] Remove remaining bpptree references --- compiler+runtime/CMakeLists.txt | 1 - compiler+runtime/cmake/install.cmake | 5 ----- 2 files changed, 6 deletions(-) diff --git a/compiler+runtime/CMakeLists.txt b/compiler+runtime/CMakeLists.txt index f2ade1f6f..6654d5240 100644 --- a/compiler+runtime/CMakeLists.txt +++ b/compiler+runtime/CMakeLists.txt @@ -554,7 +554,6 @@ target_include_directories( PUBLIC "$" "$" - "$" "$" "$" "$" diff --git a/compiler+runtime/cmake/install.cmake b/compiler+runtime/cmake/install.cmake index 9c7f7553c..10b96a7ad 100644 --- a/compiler+runtime/cmake/install.cmake +++ b/compiler+runtime/cmake/install.cmake @@ -61,11 +61,6 @@ jank_glob_install_without_prefix( PATTERN "${CMAKE_SOURCE_DIR}/third-party/folly/folly/*.h" ) -jank_glob_install_without_prefix( - INPUT_PREFIX "${CMAKE_SOURCE_DIR}/third-party/bpptree/" - PATTERN "${CMAKE_SOURCE_DIR}/third-party/bpptree/include/*" -) - jank_glob_install_without_prefix( INPUT_PREFIX "${CMAKE_SOURCE_DIR}/third-party/immer/" OUTPUT_PREFIX "include/" From d8cfc8d1df767beb98619349d3af3cbb312036ae Mon Sep 17 00:00:00 2001 From: jeaye Date: Mon, 24 Nov 2025 18:43:01 -0800 Subject: [PATCH 33/58] Add some missing meta cppgen --- compiler+runtime/cmake/install.cmake | 6 ------ compiler+runtime/include/cpp/jank/prelude.hpp | 1 + .../src/cpp/jank/codegen/processor.cpp | 18 ++++++++++++++---- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/compiler+runtime/cmake/install.cmake b/compiler+runtime/cmake/install.cmake index 10b96a7ad..70121d454 100644 --- a/compiler+runtime/cmake/install.cmake +++ b/compiler+runtime/cmake/install.cmake @@ -77,12 +77,6 @@ jank_glob_install_without_prefix( PATTERN "${CMAKE_SOURCE_DIR}/third-party/ftxui/include/*" ) -jank_glob_install_without_prefix( - INPUT_PREFIX "${CMAKE_SOURCE_DIR}/third-party/libzippp/src/" - OUTPUT_PREFIX "include/" - PATTERN "${CMAKE_SOURCE_DIR}/third-party/libzippp/src/*" -) - jank_glob_install_without_prefix( INPUT_PREFIX "${CMAKE_SOURCE_DIR}/third-party/cpptrace/" PATTERN "${CMAKE_SOURCE_DIR}/third-party/cpptrace/include/*" diff --git a/compiler+runtime/include/cpp/jank/prelude.hpp b/compiler+runtime/include/cpp/jank/prelude.hpp index 0049b7bd5..2618f3e30 100644 --- a/compiler+runtime/include/cpp/jank/prelude.hpp +++ b/compiler+runtime/include/cpp/jank/prelude.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index ea5b7b9d4..13e2f87dd 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -361,7 +362,7 @@ namespace jank::codegen util::format_to( buffer, "jank::runtime::make_box(std::in_place"); - for(auto it : runtime::make_sequence_range(typed_o)) + for(auto const it : runtime::make_sequence_range(typed_o)) { util::format_to(buffer, ", "); gen_constant(it, buffer, true); @@ -1853,11 +1854,17 @@ namespace jank::codegen util::format_to( body_buffer, - "auto {}{ jank::runtime::make_box({}, \"{}\") };", + "auto {}{ jank::runtime::make_box({}, \"{}\") };\n", ret_tmp, value_tmp.unwrap().str(false), type_str); + auto const meta{ runtime::source_to_meta(expr->source) }; + util::format_to(body_buffer, + "jank::runtime::reset_meta({}, jank::runtime::__rt_ctx->read_string(\"{}\"));", + ret_tmp, + util::escape(runtime::to_code_string(meta))); + if(expr->position == expression_position::tail) { util::format_to(body_buffer, "return {};", ret_tmp); @@ -1873,15 +1880,18 @@ namespace jank::codegen auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_unbox")) }; auto value_tmp{ gen(expr->value_expr, arity) }; auto const type_name{ cpp_util::get_qualified_type_name(expr->type) }; + auto const meta{ runtime::source_to_meta(expr->source) }; util::format_to(body_buffer, "auto {}{ " - "static_cast<{}>(jank_unbox(\"{}\", {}.data)" + "static_cast<{}>(jank_unbox_with_source(\"{}\", {}.data, " + "jank::runtime::__rt_ctx->read_string(\"{}\").data)" ") };", ret_tmp, type_name, type_name, - value_tmp.unwrap().str(false)); + value_tmp.unwrap().str(false), + util::escape(runtime::to_code_string(meta))); if(expr->position == expression_position::tail) { From 1d72ec36ad45a7418e71c4fea73858e3eaf55179 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 25 Nov 2025 13:34:30 -0800 Subject: [PATCH 34/58] Intern ns in jank_load fn for cppgen --- compiler+runtime/include/cpp/jank/c_api.h | 2 ++ compiler+runtime/src/cpp/jank/c_api.cpp | 11 +++++++++++ compiler+runtime/src/cpp/jank/codegen/processor.cpp | 1 + 3 files changed, 14 insertions(+) diff --git a/compiler+runtime/include/cpp/jank/c_api.h b/compiler+runtime/include/cpp/jank/c_api.h index a66d5b9b5..ee4d6ceee 100644 --- a/compiler+runtime/include/cpp/jank/c_api.h +++ b/compiler+runtime/include/cpp/jank/c_api.h @@ -49,6 +49,8 @@ extern "C" jank_object_ref jank_read_string(jank_object_ref s); jank_object_ref jank_read_string_c(char const * const s); + jank_object_ref jank_ns_intern(jank_object_ref sym); + jank_object_ref jank_ns_intern_c(char const * const sym); void jank_ns_set_symbol_counter(char const * const ns, jank_u64 const count); jank_object_ref jank_var_intern(jank_object_ref ns, jank_object_ref name); diff --git a/compiler+runtime/src/cpp/jank/c_api.cpp b/compiler+runtime/src/cpp/jank/c_api.cpp index faed7c122..0be7e6aad 100644 --- a/compiler+runtime/src/cpp/jank/c_api.cpp +++ b/compiler+runtime/src/cpp/jank/c_api.cpp @@ -68,6 +68,17 @@ extern "C" return __rt_ctx->read_string(s).erase(); } + jank_object_ref jank_ns_intern(jank_object_ref const sym) + { + auto const sym_obj(try_object(reinterpret_cast(sym))); + return __rt_ctx->intern_ns(sym_obj).erase(); + } + + jank_object_ref jank_ns_intern_c(char const * const sym) + { + return __rt_ctx->intern_ns(sym).erase(); + } + void jank_ns_set_symbol_counter(char const * const ns, jank_u64 const count) { auto const ns_obj(__rt_ctx->intern_ns(ns)); diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 13e2f87dd..975eb4b3a 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -2267,6 +2267,7 @@ namespace jank::codegen util::format_to(footer_buffer, "extern \"C\" void* {}(){", runtime::module::module_to_load_function(module)); + util::format_to(footer_buffer, "jank_ns_intern_c(\"{}\");", module); util::format_to(footer_buffer, "return {}::{}{ }.call().erase();", runtime::module::module_to_native_ns(module), From 892800a7630ae0e2a46028b4543fefca79ac71ad Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 25 Nov 2025 13:42:56 -0800 Subject: [PATCH 35/58] Remove bpptree entirely --- .gitmodules | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index f4d195697..4bc396c22 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "compiler+runtime/third-party/folly"] path = compiler+runtime/third-party/folly url = https://github.com/jank-lang/folly.git -[submodule "compiler+runtime/third-party/bpptree"] - path = compiler+runtime/third-party/bpptree - url = https://github.com/jank-lang/BppTree.git [submodule "compiler+runtime/third-party/immer"] path = compiler+runtime/third-party/immer url = https://github.com/jank-lang/immer.git From a64bc41fd32f32b0c9439839e0bf4a1e28640813 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 25 Nov 2025 15:58:41 -0800 Subject: [PATCH 36/58] Fix aot cppgen for vars --- .../include/cpp/jank/analyze/local_frame.hpp | 3 +- .../include/cpp/jank/runtime/var.hpp | 1 + .../src/cpp/jank/analyze/local_frame.cpp | 23 ++++---------- .../src/cpp/jank/analyze/processor.cpp | 30 +++++++++++++------ .../src/cpp/jank/codegen/processor.cpp | 8 ++--- compiler+runtime/src/cpp/jank/runtime/var.cpp | 5 ++++ 6 files changed, 36 insertions(+), 34 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp b/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp index 9b389951d..a76a19852 100644 --- a/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp +++ b/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp @@ -8,6 +8,7 @@ namespace jank::runtime { struct context; + using var_ref = oref; namespace obj { @@ -135,7 +136,7 @@ namespace jank::analyze static bool within_same_fn(jtl::ptr, jtl::ptr); - runtime::obj::symbol_ref lift_var(runtime::obj::symbol_ref const &); + void lift_var(runtime::var_ref const &); jtl::option> find_lifted_var(runtime::obj::symbol_ref const &) const; diff --git a/compiler+runtime/include/cpp/jank/runtime/var.hpp b/compiler+runtime/include/cpp/jank/runtime/var.hpp index 4a8c5b442..b111a57ac 100644 --- a/compiler+runtime/include/cpp/jank/runtime/var.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/var.hpp @@ -58,6 +58,7 @@ namespace jank::runtime var_ref set_dynamic(bool dyn); + obj::symbol_ref to_qualified_symbol() const; var_thread_binding_ref get_thread_binding() const; /* behavior::derefable */ diff --git a/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp b/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp index c3b5f4072..7d2456b5e 100644 --- a/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp @@ -190,34 +190,21 @@ namespace jank::analyze return &find_closest_fn_frame(*l) == &find_closest_fn_frame(*r); } - obj::symbol_ref local_frame::lift_var(obj::symbol_ref const &sym) + void local_frame::lift_var(var_ref const &var) { auto &closest_fn(find_closest_fn_frame(*this)); - auto const &found(closest_fn.lifted_vars.find(sym)); + auto const qualified_sym{ make_box(var->n->name->name, var->name->name) }; + auto const &found(closest_fn.lifted_vars.find(qualified_sym)); if(found != closest_fn.lifted_vars.end()) { - return found->first; - } - - obj::symbol_ref qualified_sym{}; - if(sym->ns.empty()) - { - qualified_sym - = make_box(expect_object(__rt_ctx->current_ns_var->deref())->name->name, - sym->name); - } - else - { - qualified_sym = make_box(*sym); + return; } /* We use unique native names, just so var names don't clash with the underlying C++ API. */ lifted_var lv{ __rt_ctx->unique_namespaced_string(munge(qualified_sym->name)), qualified_sym }; - closest_fn.lifted_vars.emplace(qualified_sym, std::move(lv)); - return qualified_sym; + closest_fn.lifted_vars.emplace(qualified_sym, jtl::move(lv)); } - /* TODO: These are not used in IR gen. Remove entirely? */ jtl::option> local_frame::find_lifted_var(obj::symbol_ref const &sym) const { diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 5ead51ee5..82fa54925 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -1282,17 +1282,19 @@ namespace jank::analyze ->add_usage(read::parse::reparse_nth(l, 1)); } - auto qualified_sym(current_frame->lift_var(sym)); + auto qualified_sym(runtime::__rt_ctx->qualify_symbol(sym)); qualified_sym->meta = sym->meta; /* We always def in the current ns, so we want an owned var. */ - auto const var(__rt_ctx->intern_owned_var(qualified_sym)); - if(var.is_err()) + auto const var_res(__rt_ctx->intern_owned_var(qualified_sym)); + if(var_res.is_err()) { - return error::internal_analyze_failure(var.expect_err(), + return error::internal_analyze_failure(var_res.expect_err(), meta_source(sym), latest_expansion(macro_expansions)); } + current_frame->lift_var(var_res.expect_ok()); + jtl::option> value_expr; bool const has_value{ 3 <= length }; bool const has_docstring{ 4 <= length }; @@ -1316,7 +1318,7 @@ namespace jank::analyze } value_expr = some(value_result.expect_ok()); - vars.insert_or_assign(var.expect_ok(), value_expr.unwrap()); + vars.insert_or_assign(var_res.expect_ok(), value_expr.unwrap()); } if(has_docstring) @@ -1599,7 +1601,7 @@ namespace jank::analyze auto const macro_kw(__rt_ctx->intern_keyword("", "macro", true).expect_ok()); if(var->meta.is_none() || get(var->meta.unwrap(), macro_kw).is_nil()) { - current_frame->lift_var(qualified_sym); + current_frame->lift_var(var); } return jtl::make_ref(position, current_frame, true, qualified_sym, var); } @@ -2544,7 +2546,7 @@ namespace jank::analyze auto const arg_sym(runtime::expect_object(arg)); - auto const qualified_sym(current_frame->lift_var(arg_sym)); + auto const qualified_sym{ __rt_ctx->qualify_symbol(arg_sym) }; auto const found_var(__rt_ctx->find_var(qualified_sym)); if(found_var.is_nil()) { @@ -2553,6 +2555,7 @@ namespace jank::analyze meta_source(o->meta), latest_expansion(macro_expansions)); } + current_frame->lift_var(found_var); return jtl::make_ref(position, current_frame, true, qualified_sym, found_var); } @@ -2566,8 +2569,17 @@ namespace jank::analyze { auto const pop_macro_expansions{ push_macro_expansions(*this, o) }; - auto const qualified_sym( - current_frame->lift_var(make_box(o->n->name->name, o->name->name))); + auto const qualified_sym(__rt_ctx->qualify_symbol(o->to_qualified_symbol())); + + auto const found_var(__rt_ctx->find_var(qualified_sym)); + if(found_var.is_nil()) + { + return error::analyze_unresolved_var( + util::format("Unable to resolve var '{}'.", qualified_sym->to_string()), + meta_source(o->meta), + latest_expansion(macro_expansions)); + } + current_frame->lift_var(found_var); return jtl::make_ref(position, current_frame, true, qualified_sym, o); } diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 975eb4b3a..e591aebf4 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -555,7 +555,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::var_deref_ref const expr, analyze::expr::function_arity const &) { - auto const &var(expr->frame->find_lifted_var(expr->qualified_name).unwrap().get()); + auto const &var(expr->frame->find_lifted_var(expr->var->to_qualified_symbol()).unwrap().get()); switch(expr->position) { case analyze::expression_position::statement: @@ -2256,11 +2256,7 @@ namespace jank::codegen util::format_to(footer_buffer, "};"); /* Namespace. */ - //if(!runtime::module::is_nested_module(module)) - //if(target == compilation_target::module) - { - util::format_to(footer_buffer, "}"); - } + util::format_to(footer_buffer, "}"); if(target == compilation_target::module) { diff --git a/compiler+runtime/src/cpp/jank/runtime/var.cpp b/compiler+runtime/src/cpp/jank/runtime/var.cpp index 8894d37e8..788748899 100644 --- a/compiler+runtime/src/cpp/jank/runtime/var.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/var.cpp @@ -143,6 +143,11 @@ namespace jank::runtime return this; } + obj::symbol_ref var::to_qualified_symbol() const + { + return make_box(n->name->name, name->name); + } + var_thread_binding_ref var::get_thread_binding() const { if(!thread_bound.load()) From 90d5c0f7e8b5ffbcc744fd6b54f1c6a69a8cd883 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 25 Nov 2025 19:12:31 -0800 Subject: [PATCH 37/58] Fix var_ref cppgen --- .../include/cpp/jank/analyze/expr/var_ref.hpp | 6 ++++++ .../src/cpp/jank/analyze/processor.cpp | 17 ++++++----------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/analyze/expr/var_ref.hpp b/compiler+runtime/include/cpp/jank/analyze/expr/var_ref.hpp index 272b71803..6ddae3921 100644 --- a/compiler+runtime/include/cpp/jank/analyze/expr/var_ref.hpp +++ b/compiler+runtime/include/cpp/jank/analyze/expr/var_ref.hpp @@ -26,6 +26,12 @@ namespace jank::analyze::expr runtime::object_ref to_runtime_data() const override; + /* Holds the fully qualified name for the originally resolved var. + * It will be useful to know that the var ref happened through a + * referred var, for static analysis and error reporting. + * + * For all the other purposes, `var` member should be used that points + * to the actual value of the var.. */ runtime::obj::symbol_ref qualified_name{}; runtime::var_ref var{}; }; diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 82fa54925..b0c3f215f 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -2557,7 +2557,11 @@ namespace jank::analyze } current_frame->lift_var(found_var); - return jtl::make_ref(position, current_frame, true, qualified_sym, found_var); + return jtl::make_ref(position, + current_frame, + true, + found_var->to_qualified_symbol(), + found_var); } processor::expression_result @@ -2570,16 +2574,7 @@ namespace jank::analyze auto const pop_macro_expansions{ push_macro_expansions(*this, o) }; auto const qualified_sym(__rt_ctx->qualify_symbol(o->to_qualified_symbol())); - - auto const found_var(__rt_ctx->find_var(qualified_sym)); - if(found_var.is_nil()) - { - return error::analyze_unresolved_var( - util::format("Unable to resolve var '{}'.", qualified_sym->to_string()), - meta_source(o->meta), - latest_expansion(macro_expansions)); - } - current_frame->lift_var(found_var); + current_frame->lift_var(o); return jtl::make_ref(position, current_frame, true, qualified_sym, o); } From d233b773e3bc6cf5cc20d19a2f8e3a181a764e0d Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 25 Nov 2025 23:56:40 -0800 Subject: [PATCH 38/58] Clear up static const issue --- compiler+runtime/src/cpp/jank/analyze/processor.cpp | 10 ---------- .../cpp/member/{skip-static.jank => pass-static.jank} | 3 ++- .../test/jank/cpp/member/skip-static-const.jank | 11 +++++++++++ 3 files changed, 13 insertions(+), 11 deletions(-) rename compiler+runtime/test/jank/cpp/member/{skip-static.jank => pass-static.jank} (76%) create mode 100644 compiler+runtime/test/jank/cpp/member/skip-static-const.jank diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index b0c3f215f..ac1fdffd5 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -4358,16 +4358,6 @@ namespace jank::analyze ->add_usage(read::parse::reparse_nth(l, 0)); } - if(Cpp::IsStaticDatamember(member_scope)) - { - return error::analyze_known_issue( - "A blocking Clang bug prevents access to static members in some scenarios. See " - "https://github.com/llvm/llvm-project/issues/146956 for details.", - object_source(member), - latest_expansion(macro_expansions)) - ->add_usage(read::parse::reparse_nth(l, 0)); - } - val->val_kind = expr::cpp_value::value_kind::variable; val->type = Cpp::GetLValueReferenceType(Cpp::GetTypeFromScope(member_scope)); val->scope = member_scope; diff --git a/compiler+runtime/test/jank/cpp/member/skip-static.jank b/compiler+runtime/test/jank/cpp/member/pass-static.jank similarity index 76% rename from compiler+runtime/test/jank/cpp/member/skip-static.jank rename to compiler+runtime/test/jank/cpp/member/pass-static.jank index 56afb7ecd..582cb3460 100644 --- a/compiler+runtime/test/jank/cpp/member/skip-static.jank +++ b/compiler+runtime/test/jank/cpp/member/pass-static.jank @@ -2,8 +2,9 @@ { struct bar { - static const int a{ 5 }; + static int const a; }; + int const bar::a{ 5 }; }") (let* [b (cpp/jank.cpp.member.pass_static.bar.) a (cpp/.-a b)] diff --git a/compiler+runtime/test/jank/cpp/member/skip-static-const.jank b/compiler+runtime/test/jank/cpp/member/skip-static-const.jank new file mode 100644 index 000000000..757315a21 --- /dev/null +++ b/compiler+runtime/test/jank/cpp/member/skip-static-const.jank @@ -0,0 +1,11 @@ +(cpp/raw "namespace jank::cpp::member::pass_static_const + { + struct bar + { + static int const a{ 5 }; + }; + }") +(let* [b (cpp/jank.cpp.member.pass_static_const.bar.) + a (cpp/.-a b)] + (if (= 5 a) + :success)) From 7c7ae86e7ad3a062574ab55197e7257b0fe3b243 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 12:34:47 -0800 Subject: [PATCH 39/58] Fix cppgen for throw --- compiler+runtime/src/cpp/jank/codegen/processor.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index e591aebf4..d84c0090e 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -1294,7 +1294,13 @@ namespace jank::codegen util::format_to(body_buffer, "throw static_cast({});", value_tmp.unwrap().str(true)); - return none; + + if(expr->position == analyze::expression_position::tail) + { + util::format_to(body_buffer, "return jank::runtime::jank_nil;"); + } + + return "jank::runtime::jank_nil"; } jtl::option From 87aaa10a4d181cb4beb9f53f8c3fd3be93f055ce Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 13:19:48 -0800 Subject: [PATCH 40/58] Fix inf/nan cppgen --- .../include/cpp/jank/error/report.hpp | 1 + .../src/cpp/jank/codegen/processor.cpp | 18 ++++++++++++++++-- compiler+runtime/src/cpp/jank/error/report.cpp | 11 +++++++++++ compiler+runtime/src/cpp/jank/runtime/ns.cpp | 15 +++++++-------- compiler+runtime/src/cpp/jank/util/cli.cpp | 7 +++++++ .../src/cpp/jtl/string_builder.cpp | 6 +++--- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/error/report.hpp b/compiler+runtime/include/cpp/jank/error/report.hpp index 04b38c62a..46418aa46 100644 --- a/compiler+runtime/include/cpp/jank/error/report.hpp +++ b/compiler+runtime/include/cpp/jank/error/report.hpp @@ -5,4 +5,5 @@ namespace jank::error { void report(error_ref e); + void warn(jtl::immutable_string const &); } diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index d84c0090e..56fb27662 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -184,8 +184,22 @@ namespace jank::codegen { util::format_to(buffer, "jank::runtime::make_box(static_cast({}))", - typed_o->data); + "f64>("); + + if(std::isinf(typed_o->data)) + { + util::format_to(buffer, "INFINITY"); + } + else if(std::isnan(typed_o->data)) + { + util::format_to(buffer, "NAN"); + } + else + { + util::format_to(buffer, "{}", typed_o->data); + } + + util::format_to(buffer, "))"); } else if constexpr(std::same_as) { diff --git a/compiler+runtime/src/cpp/jank/error/report.cpp b/compiler+runtime/src/cpp/jank/error/report.cpp index a75d74538..5bbbc0069 100644 --- a/compiler+runtime/src/cpp/jank/error/report.cpp +++ b/compiler+runtime/src/cpp/jank/error/report.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include #include @@ -592,4 +594,13 @@ namespace jank::error report(e->cause.as_ref()); } } + + void warn(jtl::immutable_string const &msg) + { + util::println(stderr, + "{}warning:{} {}", + jtl::terminal_style::yellow, + jtl::terminal_style::reset, + msg); + } } diff --git a/compiler+runtime/src/cpp/jank/runtime/ns.cpp b/compiler+runtime/src/cpp/jank/runtime/ns.cpp index fd971c22a..2306fe325 100644 --- a/compiler+runtime/src/cpp/jank/runtime/ns.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/ns.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace jank::runtime { @@ -85,14 +86,12 @@ namespace jank::runtime if(redefined) { auto const v{ expect_object(*found_var) }; - /* TODO: Util for warning. */ - util::println( - stderr, - "WARNING: '{}' already referred to {} in namespace '{}' but has been replaced by {}", - unqualified_sym->to_string(), - v->to_code_string(), - name->to_string(), - new_var->to_code_string()); + error::warn( + util::format("'{}' already referred to {} in namespace '{}' but has been replaced by {}", + unqualified_sym->to_string(), + v->to_code_string(), + name->to_string(), + new_var->to_code_string())); } *locked_vars = make_box((*locked_vars)->data.set(unqualified_sym, new_var)); diff --git a/compiler+runtime/src/cpp/jank/util/cli.cpp b/compiler+runtime/src/cpp/jank/util/cli.cpp index 8752374d4..ab4e6dfc0 100644 --- a/compiler+runtime/src/cpp/jank/util/cli.cpp +++ b/compiler+runtime/src/cpp/jank/util/cli.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace jank::util::cli { @@ -175,6 +176,12 @@ namespace jank::util::cli opts.command = command::check_health; } + if(opts.codegen == codegen_type::llvm_ir) + { + error::warn( + "LLVM IR code generation is currently unstable and incomplete. Use at your own risk."); + } + return ok(); } diff --git a/compiler+runtime/src/cpp/jtl/string_builder.cpp b/compiler+runtime/src/cpp/jtl/string_builder.cpp index 3c683c534..3cd52f8c1 100644 --- a/compiler+runtime/src/cpp/jtl/string_builder.cpp +++ b/compiler+runtime/src/cpp/jtl/string_builder.cpp @@ -122,18 +122,18 @@ namespace jtl string_builder &string_builder::operator()(float const d) & { - /* snprintf %f implicitly casts to double anyway. */ + /* snprintf %G implicitly casts to double anyway. */ return (*this)(static_cast(d)); } string_builder &string_builder::operator()(double const d) & { /* NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) */ - auto const required{ snprintf(nullptr, 0, "%f", d) }; + auto const required{ snprintf(nullptr, 0, "%G", d) }; maybe_realloc(*this, required); /* NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) */ - snprintf(buffer + pos, capacity - pos, "%f", d); + snprintf(buffer + pos, capacity - pos, "%G", d); pos += required; return *this; From 5bb730626da19469c6d47a15057fb2add6663094 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 13:33:48 -0800 Subject: [PATCH 41/58] Lift macro derefs too This was needed to support some `is` usages within the clojure-test-suite. --- compiler+runtime/src/cpp/jank/analyze/processor.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index ac1fdffd5..2f4663a6e 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -1597,12 +1597,7 @@ namespace jank::analyze latest_expansion(macro_expansions)); } - /* Macros aren't lifted, since they're not used during runtime. */ - auto const macro_kw(__rt_ctx->intern_keyword("", "macro", true).expect_ok()); - if(var->meta.is_none() || get(var->meta.unwrap(), macro_kw).is_nil()) - { - current_frame->lift_var(var); - } + current_frame->lift_var(var); return jtl::make_ref(position, current_frame, true, qualified_sym, var); } From d416973dd5ca682f4df5260c9949f877bed91f47 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 13:52:49 -0800 Subject: [PATCH 42/58] Test string_builder inf/nan --- .../src/cpp/jtl/string_builder.cpp | 6 +++--- .../test/cpp/jtl/string_builder.cpp | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/compiler+runtime/src/cpp/jtl/string_builder.cpp b/compiler+runtime/src/cpp/jtl/string_builder.cpp index 3cd52f8c1..3c683c534 100644 --- a/compiler+runtime/src/cpp/jtl/string_builder.cpp +++ b/compiler+runtime/src/cpp/jtl/string_builder.cpp @@ -122,18 +122,18 @@ namespace jtl string_builder &string_builder::operator()(float const d) & { - /* snprintf %G implicitly casts to double anyway. */ + /* snprintf %f implicitly casts to double anyway. */ return (*this)(static_cast(d)); } string_builder &string_builder::operator()(double const d) & { /* NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) */ - auto const required{ snprintf(nullptr, 0, "%G", d) }; + auto const required{ snprintf(nullptr, 0, "%f", d) }; maybe_realloc(*this, required); /* NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) */ - snprintf(buffer + pos, capacity - pos, "%G", d); + snprintf(buffer + pos, capacity - pos, "%f", d); pos += required; return *this; diff --git a/compiler+runtime/test/cpp/jtl/string_builder.cpp b/compiler+runtime/test/cpp/jtl/string_builder.cpp index 61f0d4b94..bcfcdde96 100644 --- a/compiler+runtime/test/cpp/jtl/string_builder.cpp +++ b/compiler+runtime/test/cpp/jtl/string_builder.cpp @@ -89,6 +89,24 @@ namespace jtl CHECK_EQ("3.140000", sb.view()); } + TEST_CASE("infinity") + { + string_builder sb; + sb(INFINITY); + CHECK_EQ(3, sb.pos); + CHECK_EQ(initial_capacity, sb.capacity); + CHECK_EQ("inf", sb.view()); + } + + TEST_CASE("nan") + { + string_builder sb; + sb(NAN); + CHECK_EQ(3, sb.pos); + CHECK_EQ(initial_capacity, sb.capacity); + CHECK_EQ("nan", sb.view()); + } + TEST_CASE("char32_t") { string_builder sb; From 3e0ef7a7d8923307a72429a4aa43f6fef9f8cedf Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 15:15:28 -0800 Subject: [PATCH 43/58] Improve profile output --- .../include/cpp/jank/jit/processor.hpp | 6 +++++ .../src/cpp/jank/codegen/llvm_processor.cpp | 2 +- .../src/cpp/jank/codegen/processor.cpp | 25 ++++++++++++++++++- compiler+runtime/src/cpp/jank/evaluate.cpp | 15 +++-------- .../src/cpp/jank/jit/processor.cpp | 12 ++++++--- .../src/cpp/jank/runtime/context.cpp | 2 ++ 6 files changed, 46 insertions(+), 16 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/jit/processor.hpp b/compiler+runtime/include/cpp/jank/jit/processor.hpp index ca870111a..b133f45f2 100644 --- a/compiler+runtime/include/cpp/jank/jit/processor.hpp +++ b/compiler+runtime/include/cpp/jank/jit/processor.hpp @@ -17,6 +17,11 @@ namespace llvm } } +namespace clang +{ + class Value; +} + namespace Cpp { class Interpreter; @@ -30,6 +35,7 @@ namespace jank::jit ~processor(); void eval_string(jtl::immutable_string const &s) const; + void eval_string(jtl::immutable_string const &s, clang::Value *) const; void load_object(jtl::immutable_string_view const &path) const; void load_dynamic_library(jtl::immutable_string const &path) const; void load_ir_module(llvm::orc::ThreadSafeModule &&m) const; diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index 654a4ef8a..ac3a1df04 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -735,7 +735,7 @@ namespace jank::codegen jtl::string_result llvm_processor::impl::gen() { - profile::timer const timer{ "ir gen" }; + profile::timer const timer{ util::format("ir gen {}", root_fn->name) }; if(target != compilation_target::function) { create_global_ctor(); diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 56fb27662..b572e539b 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include /* The strategy for codegen to C++ is quite simple. Codegen always happens on a @@ -1436,7 +1437,7 @@ namespace jank::codegen jtl::option processor::gen(expr::cpp_raw_ref const expr, expr::function_arity const &) { - util::format_to(deps_buffer, "{}\n", expr->code); + util::format_to(deps_buffer, "\n{}\n", expr->code); if(expr->position == analyze::expression_position::tail) { @@ -1996,6 +1997,7 @@ namespace jank::codegen { if(!generated_declaration) { + profile::timer const timer{ util::format("cpp gen {}", root_fn->name) }; build_header(); build_body(); build_footer(); @@ -2283,7 +2285,28 @@ namespace jank::codegen util::format_to(footer_buffer, "extern \"C\" void* {}(){", runtime::module::module_to_load_function(module)); + + /* First thing we do when loading this module is to intern our ns. Everything else will + * build on that. */ util::format_to(footer_buffer, "jank_ns_intern_c(\"{}\");", module); + + /* This dance is performed to keep symbol names unique across all the modules. + * Considering LLVM JIT symbols to be global, we need to define them with + * unique names to avoid conflicts during JIT recompilation/reloading. + * + * The approach, right now, is for each namespace, we will keep a counter + * and will increase it every time we define a new symbol. When we JIT reload + * the same namespace again, we will define new symbols. + * + * This IR codegen for calling `jank_ns_set_symbol_counter`, is to set the counter + * on an initial load. + */ + auto const current_ns{ __rt_ctx->current_ns() }; + util::format_to(footer_buffer, + "jank_ns_set_symbol_counter(\"{}\", {});", + current_ns->name->get_name(), + current_ns->symbol_counter.load()); + util::format_to(footer_buffer, "return {}::{}{ }.call().erase();", runtime::module::module_to_native_ns(module), diff --git a/compiler+runtime/src/cpp/jank/evaluate.cpp b/compiler+runtime/src/cpp/jank/evaluate.cpp index 83837010f..aa2c1aab5 100644 --- a/compiler+runtime/src/cpp/jank/evaluate.cpp +++ b/compiler+runtime/src/cpp/jank/evaluate.cpp @@ -20,7 +20,6 @@ #include #include #include -#include namespace jank::evaluate { @@ -267,7 +266,8 @@ namespace jank::evaluate object_ref eval(expression_ref const ex) { - profile::timer const timer{ "eval ast node" }; + profile::timer const timer{ util::format("eval ast node {}", + analyze::expression_kind_str(ex->kind)) }; object_ref ret{}; visit_expr([&ret](auto const typed_ex) { ret = eval(typed_ex); }, ex); return ret; @@ -580,6 +580,7 @@ namespace jank::evaluate object_ref eval(expr::function_ref const expr) { + profile::timer const timer{ util::format("eval jit function {}", expr->name) }; auto const &module( module::nest_module(expect_object(__rt_ctx->current_ns_var->deref())->to_string(), munge(expr->unique_name))); @@ -616,15 +617,7 @@ namespace jank::evaluate __rt_ctx->jit_prc.eval_string(cg_prc.declaration_str()); auto const expr_str{ cg_prc.expression_str() + ".erase()" }; clang::Value v; - auto res( - __rt_ctx->jit_prc.interpreter->ParseAndExecute({ expr_str.data(), expr_str.size() }, &v)); - if(res) - { - /* TODO: Helper to turn an llvm::Error into a string. */ - jtl::immutable_string const msg{ "Unable to compile/eval C++ source." }; - llvm::logAllUnhandledErrors(jtl::move(res), llvm::errs(), "error: "); - throw error::internal_codegen_failure(msg); - } + __rt_ctx->jit_prc.eval_string({ expr_str.data(), expr_str.size() }, &v); return try_object(v.convertTo()); } } diff --git a/compiler+runtime/src/cpp/jank/jit/processor.cpp b/compiler+runtime/src/cpp/jank/jit/processor.cpp index 3001f093d..8dcc7512b 100644 --- a/compiler+runtime/src/cpp/jank/jit/processor.cpp +++ b/compiler+runtime/src/cpp/jank/jit/processor.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace jank::jit { @@ -230,16 +231,21 @@ namespace jank::jit } void processor::eval_string(jtl::immutable_string const &s) const + { + return eval_string(s, nullptr); + } + + void processor::eval_string(jtl::immutable_string const &s, clang::Value * const ret) const { profile::timer const timer{ "jit eval_string" }; auto const &formatted{ s }; //auto const &formatted{ util::format_cpp_source(s).expect_ok() }; //util::println("// eval_string:\n{}\n", formatted); - auto err(interpreter->ParseAndExecute({ formatted.data(), formatted.size() })); + auto err(interpreter->ParseAndExecute({ formatted.data(), formatted.size() }, ret)); if(err) { - llvm::logAllUnhandledErrors(std::move(err), llvm::errs(), "error: "); - throw std::runtime_error{ "Failed to evaluate C++ code." }; + llvm::logAllUnhandledErrors(jtl::move(err), llvm::errs(), "error: "); + throw error::internal_codegen_failure("Unable to compile C++ source."); } register_jit_stack_frames(); } diff --git a/compiler+runtime/src/cpp/jank/runtime/context.cpp b/compiler+runtime/src/cpp/jank/runtime/context.cpp index 3af7ab700..347f0b145 100644 --- a/compiler+runtime/src/cpp/jank/runtime/context.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/context.cpp @@ -181,6 +181,7 @@ namespace jank::runtime * targeted at AOT and doesn't have access to what's loaded in the JIT runtime. */ if(truthy(compile_files_var->deref())) { + profile::timer const timer{ "rt compile-module" }; auto const &module(runtime::to_string(current_module_var->deref())); auto const name{ module::module_to_load_function(module) }; @@ -203,6 +204,7 @@ namespace jank::runtime } else { + profile::timer const timer{ "rt compile-module parse + write" }; codegen::processor cg_prc{ fn, module, codegen::compilation_target::module }; //util::println("{}\n", util::format_cpp_source(cg_prc.declaration_str()).expect_ok()); auto const code{ cg_prc.declaration_str() }; From 5b47b3c8c09f3b4ea46e2ad1de1e969e59a3af6b Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 16:28:52 -0800 Subject: [PATCH 44/58] Start optimizing cppgen for size 1. Dedupe lifted constants/vars across arities 2. Remove lifting of nil/bools 3. Remove baked in `using namespace` usages Much, much more to go. --- .../include/cpp/jank/codegen/processor.hpp | 2 - .../src/cpp/jank/analyze/processor.cpp | 6 +- .../src/cpp/jank/codegen/processor.cpp | 150 +++++++----------- 3 files changed, 61 insertions(+), 97 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index a8832e187..4a2166a67 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -154,8 +154,6 @@ namespace jank::codegen void build_footer(); jtl::immutable_string expression_str(); - jtl::immutable_string module_init_str(jtl::immutable_string const &module); - void format_dynamic_call(jtl::immutable_string const &source_tmp, jtl::immutable_string const &ret_tmp, native_vector const &arg_exprs, diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 2f4663a6e..b926056d8 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -2820,7 +2820,11 @@ namespace jank::analyze { auto const pop_macro_expansions{ push_macro_expansions(*this, o) }; - current_frame->lift_constant(o); + /* There's no need to lift these, since we'll just codgen the public constants for them. */ + if(o->type != object_type::nil && o->type != object_type::boolean) + { + current_frame->lift_constant(o); + } return jtl::make_ref(position, current_frame, needs_box, o); } diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index b572e539b..c9bb984d9 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -654,13 +655,26 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::primitive_literal_ref const expr, analyze::expr::function_arity const &) { - auto const &constant(expr->frame->find_lifted_constant(expr->data).unwrap().get()); - - handle ret{ runtime::munge(constant.native_name) }; - if(constant.unboxed_native_name.is_some()) + handle ret; + if(expr->data->type == runtime::object_type::nil) + { + ret = handle{ "jank::runtime::jank_nil" }; + } + else if(expr->data->type == runtime::object_type::boolean) { - ret = { runtime::munge(constant.native_name), - runtime::munge(constant.unboxed_native_name.unwrap()) }; + ret = handle{ runtime::truthy(expr->data) ? "jank::runtime::jank_true" + : "jank::runtime::jank_false" }; + } + else + { + auto const &constant(expr->frame->find_lifted_constant(expr->data).unwrap().get()); + + ret = runtime::munge(constant.native_name); + if(constant.unboxed_native_name.is_some()) + { + ret = { runtime::munge(constant.native_name), + runtime::munge(constant.unboxed_native_name.unwrap()) }; + } } switch(expr->position) @@ -1040,7 +1054,7 @@ namespace jank::codegen if(cpp_util::is_any_object(local_type)) { util::format_to(body_buffer, - "{ object_ref {}({}); ", + "{ jank::runtime::object_ref {}({}); ", munged_name, val_tmp.unwrap().str(true)); } @@ -1323,7 +1337,7 @@ namespace jank::codegen { auto const has_catch{ expr->catch_body.is_some() }; auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("try"))); - util::format_to(body_buffer, "object_ref {}{ };", ret_tmp); + util::format_to(body_buffer, "jank::runtime::object_ref {}{ };", ret_tmp); util::format_to(body_buffer, "{"); if(expr->finally_body.is_some()) @@ -2020,17 +2034,7 @@ namespace jank::codegen void processor::build_header() { - /* TODO: We don't want this for nested modules, but we do if they're in their own file. - * Do we need three module compilation targets? Top-level, nested, local? - * - * Local fns are within a struct already, so we can't enter the ns again. */ - //if(!runtime::module::is_nested_module(module)) - //if(target == compilation_target::module) - { - util::format_to(header_buffer, - "namespace {} {", - runtime::module::module_to_native_ns(module)); - } + util::format_to(header_buffer, "namespace {} {", runtime::module::module_to_native_ns(module)); util::format_to(header_buffer, R"( @@ -2041,16 +2045,17 @@ namespace jank::codegen { /* TODO: Constants and vars are not shared across arities. We'd need stable names. */ - native_set used_vars, used_constants, used_captures; + native_set used_vars, used_constants, used_captures; for(auto const &arity : root_fn->arities) { for(auto const &v : arity.frame->lifted_vars) { - if(used_vars.contains(v.second.native_name.to_hash())) + auto const hash{ v.first->to_hash() }; + if(used_vars.contains(hash)) { continue; } - used_vars.emplace(v.second.native_name.to_hash()); + used_vars.emplace(hash); util::format_to(header_buffer, "jank::runtime::var_ref const {};", @@ -2059,13 +2064,13 @@ namespace jank::codegen for(auto const &v : arity.frame->lifted_constants) { - if(used_constants.contains(v.second.native_name.to_hash())) + if(used_constants.contains(runtime::to_hash(v.first))) { continue; } - used_constants.emplace(v.second.native_name.to_hash()); + used_constants.emplace(runtime::to_hash(v.first)); - /* TODO: Typed lifted constants. */ + /* TODO: Typed lifted constants (in analysis). */ util::format_to(header_buffer, "{} const {};", detail::gen_constant_type(v.second.data, true), @@ -2075,11 +2080,12 @@ namespace jank::codegen /* TODO: More useful types here. */ for(auto const &v : arity.frame->captures) { - if(used_captures.contains(v.first->to_hash())) + auto const hash{ v.first->to_hash() }; + if(used_captures.contains(hash)) { continue; } - used_captures.emplace(v.first->to_hash()); + used_captures.emplace(hash); /* Captures aren't const since they could be late-assigned, in the case of a letfn. */ util::format_to(header_buffer, @@ -2090,7 +2096,7 @@ namespace jank::codegen } { - native_set used_captures; + native_set used_captures; util::format_to(header_buffer, "{}(", runtime::munge(struct_name.name)); bool need_comma{}; @@ -2098,11 +2104,12 @@ namespace jank::codegen { for(auto const &v : arity.frame->captures) { - if(used_captures.contains(v.first->to_hash())) + auto const hash{ v.first->to_hash() }; + if(used_captures.contains(hash)) { continue; } - used_captures.emplace(v.first->to_hash()); + used_captures.emplace(hash); /* TODO: More useful types here. */ util::format_to(header_buffer, @@ -2115,7 +2122,9 @@ namespace jank::codegen } { - native_set used_vars, used_constants, used_captures; + native_set used_captures; + native_map used_vars; + native_map used_constants; util::format_to(header_buffer, ") : jank::runtime::obj::jit_function{ "); /* TODO: All of the meta in clojure.core alone costs 2s to JIT compile at run-time. * How can this be faster? */ @@ -2124,13 +2133,16 @@ namespace jank::codegen for(auto const &arity : root_fn->arities) { - for(auto const &v : arity.frame->lifted_vars) + for(auto &v : arity.frame->lifted_vars) { - if(used_vars.contains(v.second.native_name.to_hash())) + auto const hash{ v.first->to_hash() }; + auto const existing{ used_vars.find(hash) }; + if(existing != used_vars.end()) { + v.second = existing->second; continue; } - used_vars.emplace(v.second.native_name.to_hash()); + used_vars.emplace(hash, v.second); util::format_to(header_buffer, R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}", "{}").expect_ok() })", @@ -2139,13 +2151,16 @@ namespace jank::codegen v.second.var_name->name); } - for(auto const &v : arity.frame->lifted_constants) + for(auto &v : arity.frame->lifted_constants) { - if(used_constants.contains(v.second.native_name.to_hash())) + auto const hash{ runtime::to_hash(v.first) }; + auto const existing{ used_constants.find(hash) }; + if(existing != used_constants.end()) { + v.second = existing->second; continue; } - used_constants.emplace(v.second.native_name.to_hash()); + used_constants.emplace(hash, v.second); util::format_to(header_buffer, ", {}{", runtime::munge(v.second.native_name)); detail::gen_constant(v.second.data, header_buffer, true); @@ -2154,11 +2169,12 @@ namespace jank::codegen for(auto const &v : arity.frame->captures) { - if(used_captures.contains(v.first->to_hash())) + auto const hash{ v.first->to_hash() }; + if(used_captures.contains(hash)) { continue; } - used_captures.emplace(v.first->to_hash()); + used_captures.emplace(hash); auto const name{ runtime::munge(v.second.native_name) }; util::format_to(header_buffer, ", {}{ {} }", name, name); @@ -2205,12 +2221,7 @@ namespace jank::codegen param_shadows_fn |= param->name == root_fn->name; } - util::format_to(body_buffer, - R"( - ) final { - using namespace jank; - using namespace jank::runtime; - )"); + util::format_to(body_buffer, ") final {"); //util::format_to(body_buffer, "jank::profile::timer __timer{ \"{}\" };", root_fn->name); @@ -2382,53 +2393,4 @@ namespace jank::codegen } return { expression_buffer.data(), expression_buffer.size() }; } - - /* TODO: Not sure if we want any of this. The module dependency loading feels wrong, - * since it should be tied to calls to require instead. */ - jtl::immutable_string processor::module_init_str(jtl::immutable_string const &module) - { - jtl::string_builder module_buffer; - - util::format_to(module_buffer, "namespace {} {", runtime::module::module_to_native_ns(module)); - - util::format_to(module_buffer, - R"( - struct __ns__init - { - )"); - - util::format_to(module_buffer, "static void __init(){"); - //util::format_to(module_buffer, "jank::profile::timer __timer{ \"ns __init\" };"); - util::format_to(module_buffer, - "constexpr auto const deps(jank::util::make_array("); - bool needs_comma{}; - for(auto const &dep : __rt_ctx->module_dependencies[module]) - { - if(needs_comma) - { - util::format_to(module_buffer, ", "); - } - util::format_to(module_buffer, "\"/{}\"", dep); - needs_comma = true; - } - util::format_to(module_buffer, "));"); - - util::format_to(module_buffer, "for(auto const &dep : deps){"); - util::format_to(module_buffer, "jank::runtime::__rt_ctx->load_module(dep).expect_ok();"); - util::format_to(module_buffer, "}"); - - /* __init fn */ - util::format_to(module_buffer, "}"); - - /* Struct */ - util::format_to(module_buffer, "};"); - - /* Namespace */ - util::format_to(module_buffer, "}"); - - native_transient_string ret; - ret.reserve(module_buffer.size()); - ret += jtl::immutable_string_view{ module_buffer.data(), module_buffer.size() }; - return ret; - } } From ac27ef94729160afb9ccc962d65d9b0b1122dd03 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 18:59:55 -0800 Subject: [PATCH 45/58] Move lifted constants and vars to codegen We want to dedupe them by arity, which gives us just as much work in codegen as tracking them normally would, if not more. This also gives each codegen processor its own flexibility about which things to lift and at which scope to do it. --- .../include/cpp/jank/analyze/local_frame.hpp | 33 --- .../include/cpp/jank/codegen/processor.hpp | 6 + .../include/cpp/jank/runtime/context.hpp | 23 ++- .../src/cpp/jank/analyze/local_frame.cpp | 89 +------- .../src/cpp/jank/analyze/processor.cpp | 29 +-- .../src/cpp/jank/codegen/llvm_processor.cpp | 88 ++++---- .../src/cpp/jank/codegen/processor.cpp | 194 +++++++++--------- compiler+runtime/src/cpp/jank/evaluate.cpp | 13 +- .../src/cpp/jank/runtime/context.cpp | 36 ++-- 9 files changed, 190 insertions(+), 321 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp b/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp index a76a19852..4c02ddcdb 100644 --- a/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp +++ b/compiler+runtime/include/cpp/jank/analyze/local_frame.hpp @@ -30,25 +30,6 @@ namespace jank::analyze using function_context_ref = jtl::ref; } - struct lifted_var - { - jtl::immutable_string native_name{}; - runtime::obj::symbol_ref var_name{}; - - runtime::object_ref to_runtime_data() const; - }; - - /* TODO: Track constant usages to figure out if boxing is needed at all, - * rather than just doing both. */ - struct lifted_constant - { - jtl::immutable_string native_name{}; - jtl::option unboxed_native_name{}; - runtime::object_ref data{}; - - runtime::object_ref to_runtime_data() const; - }; - struct local_binding { runtime::obj::symbol_ref name{}; @@ -136,14 +117,6 @@ namespace jank::analyze static bool within_same_fn(jtl::ptr, jtl::ptr); - void lift_var(runtime::var_ref const &); - jtl::option> - find_lifted_var(runtime::obj::symbol_ref const &) const; - - void lift_constant(runtime::object_ref); - jtl::option> - find_lifted_constant(runtime::object_ref) const; - static local_frame const &find_closest_fn_frame(local_frame const &frame); static local_frame &find_closest_fn_frame(local_frame &frame); @@ -153,12 +126,6 @@ namespace jank::analyze jtl::option> parent; native_unordered_map locals; native_unordered_map captures; - native_unordered_map lifted_vars; - native_unordered_map, - runtime::very_equal_to> - lifted_constants; /* This is only set if the frame type is fn. */ jtl::ptr fn_ctx; }; diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index 4a2166a67..51d40ee45 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -170,6 +170,12 @@ namespace jank::codegen jtl::string_builder footer_buffer; jtl::string_builder expression_buffer; jtl::immutable_string expression_fn_name; + native_unordered_map lifted_vars; + native_unordered_map, + runtime::very_equal_to> + lifted_constants; bool generated_declaration{}; bool generated_expression{}; }; diff --git a/compiler+runtime/include/cpp/jank/runtime/context.hpp b/compiler+runtime/include/cpp/jank/runtime/context.hpp index c1ac657ff..50adda9ea 100644 --- a/compiler+runtime/include/cpp/jank/runtime/context.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/context.hpp @@ -59,7 +59,8 @@ namespace jank::runtime obj::symbol_ref qualify_symbol(obj::symbol_ref const &) const; jtl::option find_local(obj::symbol_ref const &); - jtl::result intern_var(obj::symbol_ref const &); + jtl::result intern_var(obj::symbol_ref const &qualified_name); + jtl::result intern_var(jtl::immutable_string const &); jtl::result intern_var(jtl::immutable_string const &ns, jtl::immutable_string const &name); jtl::result intern_owned_var(obj::symbol_ref const &); @@ -79,11 +80,11 @@ namespace jank::runtime object_ref macroexpand(object_ref o); object_ref eval_file(jtl::immutable_string const &path); - object_ref eval_string(jtl::immutable_string_view const &code); - jtl::result eval_cpp_string(jtl::immutable_string_view const &code) const; - object_ref read_string(jtl::immutable_string_view const &code); + object_ref eval_string(jtl::immutable_string const &code); + jtl::result eval_cpp_string(jtl::immutable_string const &code) const; + object_ref read_string(jtl::immutable_string const &code); native_vector - analyze_string(jtl::immutable_string_view const &code, bool const eval = true); + analyze_string(jtl::immutable_string const &code, bool const eval = true); /* Finds the specified module on the module path and loads it. If * the module is already loaded, nothing is done. @@ -97,10 +98,10 @@ namespace jank::runtime * Module meow.cat refers to foo.bar$meow.cat */ jtl::result - load_module(jtl::immutable_string_view const &module, module::origin ori); + load_module(jtl::immutable_string const &module, module::origin ori); /* Does all the same work as load_module, but also writes compiled files to the file system. */ - jtl::result compile_module(jtl::immutable_string_view const &module); + jtl::result compile_module(jtl::immutable_string const &module); object_ref eval(object_ref const o); @@ -110,11 +111,11 @@ namespace jank::runtime /* Generates a unique name for use with anything from codgen structs, * lifted vars, to shadowed locals. Prefixes with current namespace. */ jtl::immutable_string unique_namespaced_string() const; - jtl::immutable_string unique_namespaced_string(jtl::immutable_string_view const &prefix) const; - jtl::immutable_string unique_munged_string() const; - jtl::immutable_string unique_munged_string(jtl::immutable_string_view const &prefix) const; + jtl::immutable_string unique_namespaced_string(jtl::immutable_string const &prefix) const; + jtl::immutable_string unique_string() const; + jtl::immutable_string unique_string(jtl::immutable_string const &prefix) const; obj::symbol unique_symbol() const; - obj::symbol unique_symbol(jtl::immutable_string_view const &prefix) const; + obj::symbol unique_symbol(jtl::immutable_string const &prefix) const; folly::Synchronized> namespaces; folly::Synchronized> keywords; diff --git a/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp b/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp index 7d2456b5e..523014dbf 100644 --- a/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/local_frame.cpp @@ -14,16 +14,6 @@ namespace jank::analyze { using namespace jank::runtime; - object_ref lifted_var::to_runtime_data() const - { - return obj::persistent_array_map::create_unique(make_box("var_name"), var_name); - } - - object_ref lifted_constant::to_runtime_data() const - { - return obj::persistent_array_map::create_unique(make_box("data"), data); - } - object_ref local_binding::to_runtime_data() const { return obj::persistent_array_map::create_unique( @@ -190,78 +180,15 @@ namespace jank::analyze return &find_closest_fn_frame(*l) == &find_closest_fn_frame(*r); } - void local_frame::lift_var(var_ref const &var) - { - auto &closest_fn(find_closest_fn_frame(*this)); - auto const qualified_sym{ make_box(var->n->name->name, var->name->name) }; - auto const &found(closest_fn.lifted_vars.find(qualified_sym)); - if(found != closest_fn.lifted_vars.end()) - { - return; - } - - /* We use unique native names, just so var names don't clash with the underlying C++ API. */ - lifted_var lv{ __rt_ctx->unique_namespaced_string(munge(qualified_sym->name)), qualified_sym }; - closest_fn.lifted_vars.emplace(qualified_sym, jtl::move(lv)); - } - - jtl::option> - local_frame::find_lifted_var(obj::symbol_ref const &sym) const - { - auto const &closest_fn(find_closest_fn_frame(*this)); - auto const &found(closest_fn.lifted_vars.find(sym)); - if(found != closest_fn.lifted_vars.end()) - { - return some(std::ref(found->second)); - } - return none; - } - - void local_frame::lift_constant(object_ref const constant) - { - auto &closest_fn(find_closest_fn_frame(*this)); - auto const &found(closest_fn.lifted_constants.find(constant)); - if(found != closest_fn.lifted_constants.end()) - { - return; - } - - auto const name(__rt_ctx->unique_symbol("const")); - auto const unboxed_name{ visit_number_like( - [&](auto const) -> jtl::option { return name.name + "__unboxed"; }, - []() -> jtl::option { return none; }, - constant) }; - - lifted_constant l{ name.name, unboxed_name, constant }; - closest_fn.lifted_constants.emplace(constant, std::move(l)); - } - - jtl::option> - local_frame::find_lifted_constant(object_ref const o) const - { - auto const &closest_fn(find_closest_fn_frame(*this)); - auto const &found(closest_fn.lifted_constants.find(o)); - if(found != closest_fn.lifted_constants.end()) - { - return some(std::ref(found->second)); - } - return none; - } - object_ref local_frame::to_runtime_data() const { - return obj::persistent_array_map::create_unique( - make_box("type"), - make_box(frame_type_str(type)), - make_box("parent"), - jank::detail::to_runtime_data(parent), - make_box("locals"), - jank::detail::to_runtime_data(locals), - make_box("captures"), - jank::detail::to_runtime_data(captures), - make_box("lifted_vars"), - jank::detail::to_runtime_data(lifted_vars), - make_box("lifted_constants"), - jank::detail::to_runtime_data(lifted_constants)); + return obj::persistent_array_map::create_unique(make_box("type"), + make_box(frame_type_str(type)), + make_box("parent"), + jank::detail::to_runtime_data(parent), + make_box("locals"), + jank::detail::to_runtime_data(locals), + make_box("captures"), + jank::detail::to_runtime_data(captures)); } } diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index b926056d8..6c73b33b0 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -942,7 +943,9 @@ namespace jank::analyze latest_expansion(macro_expansions)); } if(is_ctor - && Cpp::IsAggregateConstructible(val->type, arg_types, __rt_ctx->unique_munged_string())) + && Cpp::IsAggregateConstructible(val->type, + arg_types, + runtime::munge(__rt_ctx->unique_namespaced_string()))) { //util::println("using aggregate initializaation"); return jtl::make_ref(position, @@ -1293,8 +1296,6 @@ namespace jank::analyze latest_expansion(macro_expansions)); } - current_frame->lift_var(var_res.expect_ok()); - jtl::option> value_expr; bool const has_value{ 3 <= length }; bool const has_docstring{ 4 <= length }; @@ -1337,13 +1338,6 @@ namespace jank::analyze qualified_sym = qualified_sym->with_meta(meta_with_doc); } - /* Lift this so it can be used during codegen. */ - /* TODO: I don't think lifting meta is actually needed anymore. Verify. */ - if(qualified_sym->meta.is_some()) - { - current_frame->lift_constant(qualified_sym->meta.unwrap()); - } - return jtl::make_ref(position, current_frame, true, qualified_sym, value_expr); } @@ -1597,7 +1591,6 @@ namespace jank::analyze latest_expansion(macro_expansions)); } - current_frame->lift_var(var); return jtl::make_ref(position, current_frame, true, qualified_sym, var); } @@ -2550,7 +2543,6 @@ namespace jank::analyze meta_source(o->meta), latest_expansion(macro_expansions)); } - current_frame->lift_var(found_var); return jtl::make_ref(position, current_frame, @@ -2569,7 +2561,6 @@ namespace jank::analyze auto const pop_macro_expansions{ push_macro_expansions(*this, o) }; auto const qualified_sym(__rt_ctx->qualify_symbol(o->to_qualified_symbol())); - current_frame->lift_var(o); return jtl::make_ref(position, current_frame, true, qualified_sym, o); } @@ -2819,12 +2810,6 @@ namespace jank::analyze bool const needs_box) { auto const pop_macro_expansions{ push_macro_expansions(*this, o) }; - - /* There's no need to lift these, since we'll just codgen the public constants for them. */ - if(o->type != object_type::nil && o->type != object_type::boolean) - { - current_frame->lift_constant(o); - } return jtl::make_ref(position, current_frame, needs_box, o); } @@ -2869,9 +2854,6 @@ namespace jank::analyze jtl::make_ref(position, current_frame, true, std::move(exprs), o->meta)); auto const o(evaluate::eval(pre_eval_expr)); - /* TODO: Order lifted constants. Use sub constants during codegen. */ - current_frame->lift_constant(o); - return jtl::make_ref(position, current_frame, true, o); } @@ -2980,9 +2962,6 @@ namespace jank::analyze typed_o->meta)); auto const constant(evaluate::eval(pre_eval_expr)); - /* TODO: Order lifted constants. Use sub constants during codegen. */ - current_frame->lift_constant(constant); - return jtl::make_ref(position, current_frame, true, constant); } diff --git a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp index ac3a1df04..27d49c296 100644 --- a/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/llvm_processor.cpp @@ -210,6 +210,16 @@ namespace jank::codegen usize alignment{}; }; + static jtl::immutable_string unique_munged_string() + { + return runtime::munge(__rt_ctx->unique_namespaced_string()); + } + + static jtl::immutable_string unique_munged_string(jtl::immutable_string const &prefix) + { + return runtime::munge(__rt_ctx->unique_namespaced_string(prefix)); + } + static llvm::Type *llvm_builtin_type(reusable_context const &ctx, jtl::ref const llvm_ctx, jtl::ptr const type) @@ -409,7 +419,7 @@ namespace jank::codegen || Cpp::IsPointerType(param_type) /*|| Cpp::IsArrayType(param_type)*/) }; - auto const fn_callable{ Cpp::MakeAotCallable(match, __rt_ctx->unique_munged_string()) }; + auto const fn_callable{ Cpp::MakeAotCallable(match, unique_munged_string()) }; link_module(ctx, reinterpret_cast(fn_callable.getModule())); llvm::Value *arg_alloc{ arg }; @@ -515,7 +525,7 @@ namespace jank::codegen reusable_context::reusable_context(jtl::immutable_string const &module_name, std::unique_ptr llvm_ctx) : module_name{ module_name } - , ctor_name{ __rt_ctx->unique_munged_string("jank_global_init") } + , ctor_name{ unique_munged_string("jank_global_init") } //, llvm_ctx{ std::make_unique() } //, llvm_ctx{ reinterpret_cast *>( // reinterpret_cast( @@ -527,8 +537,7 @@ namespace jank::codegen , mam{ std::make_unique() } , pic{ std::make_unique() } { - auto m{ std::make_unique(__rt_ctx->unique_munged_string(module_name).c_str(), - *llvm_ctx) }; + auto m{ std::make_unique(unique_munged_string(module_name).c_str(), *llvm_ctx) }; module = llvm::orc::ThreadSafeModule{ std::move(m), std::move(llvm_ctx) }; auto const raw_ctx{ extract_context(module) }; @@ -1853,7 +1862,7 @@ namespace jank::codegen /* We also need to surface this RTTI upward, to the module level, so it * can end up in the generated object file. */ auto const callable{ - Cpp::MakeRTTICallable(catch_type, exception_rtti, __rt_ctx->unique_munged_string()) + Cpp::MakeRTTICallable(catch_type, exception_rtti, unique_munged_string()) }; global_rtti.emplace(exception_rtti, callable); } @@ -2213,14 +2222,14 @@ namespace jank::codegen return alloc; } - auto const callable{ Cpp::IsFunctionPointerType(expr->type) - /* We pass the type and the scope in here so that unresolved template + auto const callable{ + Cpp::IsFunctionPointerType(expr->type) + /* We pass the type and the scope in here so that unresolved template * scopes can be turned into the correct specialization which matches * the type we have. */ - ? Cpp::MakeFunctionValueAotCallable(expr->scope, - expr->type, - __rt_ctx->unique_munged_string()) - : Cpp::MakeAotCallable(expr->scope, __rt_ctx->unique_munged_string()) }; + ? Cpp::MakeFunctionValueAotCallable(expr->scope, expr->type, unique_munged_string()) + : Cpp::MakeAotCallable(expr->scope, unique_munged_string()) + }; jank_debug_assert(callable); link_module(*ctx, reinterpret_cast(callable.getModule())); @@ -2471,30 +2480,28 @@ namespace jank::codegen if(expr->source_expr->kind == expression_kind::cpp_value) { auto const source{ llvm::cast(expr->source_expr.data) }; - return gen_aot_call( - Cpp::MakeAotCallable(source->scope, arg_types, __rt_ctx->unique_munged_string()), - source->scope, - expr->type, - Cpp::GetName(source->scope), - expr->arg_exprs, - expr->position, - expr->kind, - arity); + return gen_aot_call(Cpp::MakeAotCallable(source->scope, arg_types, unique_munged_string()), + source->scope, + expr->type, + Cpp::GetName(source->scope), + expr->arg_exprs, + expr->position, + expr->kind, + arity); } else { auto const source_type{ cpp_util::expression_type(expr->source_expr) }; auto arg_exprs{ expr->arg_exprs }; arg_exprs.insert(arg_exprs.begin(), expr->source_expr); - return gen_aot_call( - Cpp::MakeApplyCallable(source_type, arg_types, __rt_ctx->unique_munged_string()), - nullptr, - expr->type, - "call", - jtl::move(arg_exprs), - expr->position, - expr->kind, - arity); + return gen_aot_call(Cpp::MakeApplyCallable(source_type, arg_types, unique_munged_string()), + nullptr, + expr->type, + "call", + jtl::move(arg_exprs), + expr->position, + expr->kind, + arity); } } @@ -2512,7 +2519,7 @@ namespace jank::codegen * We can save ourselves the time of JIT compiling more C++ and make the IR easier * to optimize. */ ctor_fn_callable - = Cpp::MakeBuiltinConstructorAotCallable(expr->type, __rt_ctx->unique_munged_string()); + = Cpp::MakeBuiltinConstructorAotCallable(expr->type, unique_munged_string()); } else { @@ -2523,7 +2530,7 @@ namespace jank::codegen ctor_fn_callable = Cpp::MakeBuiltinConstructorAotCallable(expr->type, needs_conversion ? expr->type : arg_type, - __rt_ctx->unique_munged_string()); + unique_munged_string()); } } else if(expr->is_aggregate) @@ -2533,15 +2540,14 @@ namespace jank::codegen { arg_types.emplace_back(cpp_util::expression_type(arg_expr)); } - ctor_fn_callable - = Cpp::MakeAggregateInitializationAotCallable(expr->type, - arg_types, - __rt_ctx->unique_munged_string()); + ctor_fn_callable = Cpp::MakeAggregateInitializationAotCallable(expr->type, + arg_types, + unique_munged_string()); } else { jank_debug_assert(expr->fn); - ctor_fn_callable = Cpp::MakeAotCallable(expr->fn, __rt_ctx->unique_munged_string()); + ctor_fn_callable = Cpp::MakeAotCallable(expr->fn, unique_munged_string()); } jank_debug_assert(ctor_fn_callable); @@ -2565,7 +2571,7 @@ namespace jank::codegen llvm::Value * llvm_processor::impl::gen(expr::cpp_member_call_ref const expr, expr::function_arity const &arity) { - return gen_aot_call(Cpp::MakeAotCallable(expr->fn, __rt_ctx->unique_munged_string()), + return gen_aot_call(Cpp::MakeAotCallable(expr->fn, unique_munged_string()), expr->fn, cpp_util::expression_type(expr), Cpp::GetName(expr->fn), @@ -2578,7 +2584,7 @@ namespace jank::codegen llvm::Value *llvm_processor::impl::gen(expr::cpp_member_access_ref const expr, expr::function_arity const &arity) { - return gen_aot_call(Cpp::MakeAotCallable(expr->scope, __rt_ctx->unique_munged_string()), + return gen_aot_call(Cpp::MakeAotCallable(expr->scope, unique_munged_string()), nullptr, expr->type, Cpp::GetName(expr->scope), @@ -2654,7 +2660,7 @@ namespace jank::codegen return gen_aot_call(Cpp::MakeBuiltinOperatorAotCallable(static_cast(expr->op), expr->type, arg_types, - __rt_ctx->unique_munged_string()), + unique_munged_string()), nullptr, expr->type, name.getAsString(), @@ -2755,7 +2761,7 @@ namespace jank::codegen if(!Cpp::IsTriviallyDestructible(expr->type)) { auto const dtor{ Cpp::GetDestructor(Cpp::GetScopeFromType(expr->type)) }; - auto const dtor_callable{ Cpp::MakeAotCallable(dtor, __rt_ctx->unique_munged_string()) }; + auto const dtor_callable{ Cpp::MakeAotCallable(dtor, unique_munged_string()) }; link_module(*ctx, reinterpret_cast(dtor_callable.getModule())); auto const reg_fn_type(llvm::FunctionType::get(ctx->builder->getVoidTy(), @@ -2803,7 +2809,7 @@ namespace jank::codegen if(!Cpp::IsTriviallyDestructible(value_type)) { auto const dtor{ Cpp::GetDestructor(Cpp::GetScopeFromType(value_type)) }; - auto const dtor_callable{ Cpp::MakeAotCallable(dtor, __rt_ctx->unique_munged_string()) }; + auto const dtor_callable{ Cpp::MakeAotCallable(dtor, unique_munged_string()) }; link_module(*ctx, reinterpret_cast(dtor_callable.getModule())); auto const dtor_fn_type( diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index c9bb984d9..09ddef423 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -484,17 +484,53 @@ namespace jank::codegen return ret; } + static jtl::immutable_string + lift_var(native_unordered_map &lifted_vars, + jtl::immutable_string const &qualified_name) + { + auto const existing{ lifted_vars.find(qualified_name) }; + if(existing != lifted_vars.end()) + { + return existing->second; + } + + static jtl::immutable_string const dot{ "\\." }; + auto const us{ __rt_ctx->unique_string(qualified_name) }; + auto const native_name{ runtime::munge_and_replace(us, dot, "_") }; + //util::println("lifting var '{}' with '{}' as '{}'", qualified_name, us, native_name); + lifted_vars.emplace(qualified_name, native_name); + return native_name; + } + + static jtl::immutable_string + lift_constant(native_unordered_map, + runtime::very_equal_to> &lifted_constants, + object_ref const &o) + { + auto const existing{ lifted_constants.find(o) }; + if(existing != lifted_constants.end()) + { + return existing->second; + } + + auto const &native_name{ runtime::munge(__rt_ctx->unique_namespaced_string("const")) }; + //util::println("lifting constant {} as {}", runtime::to_code_string(o), native_name); + lifted_constants.emplace(o, native_name); + return native_name; + } + jtl::option processor::gen(analyze::expr::def_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &var(expr->frame->find_lifted_var(expr->name).unwrap().get()); - auto const &munged_name(runtime::munge(var.native_name)); - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(munged_name))); + auto const &var(lift_var(lifted_vars, expr->name->to_string())); + auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(var))); - jtl::option> meta; + jtl::option meta; if(expr->name->meta.is_some()) { - meta = expr->frame->find_lifted_constant(expr->name->meta.unwrap()).unwrap(); + meta = lift_constant(lifted_constants, expr->name->meta.unwrap()); } /* Forward declarations just intern the var and evaluate to it. */ @@ -503,16 +539,12 @@ namespace jank::codegen if(meta.is_some()) { auto const dynamic{ truthy( - get(meta.unwrap().get().data, __rt_ctx->intern_keyword("dynamic").expect_ok())) }; - return util::format("{}->with_meta({})->set_dynamic({})", - runtime::munge(var.native_name), - runtime::munge(meta.unwrap().get().native_name), - dynamic); + get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; + return util::format("{}->with_meta({})->set_dynamic({})", var, meta.unwrap(), dynamic); } else { - return util::format("{}->with_meta(jank::runtime::jank_nil)", - runtime::munge(var.native_name)); + return util::format("{}->with_meta(jank::runtime::jank_nil)", var); } } @@ -524,17 +556,17 @@ namespace jank::codegen if(meta.is_some()) { auto const dynamic{ truthy( - get(meta.unwrap().get().data, __rt_ctx->intern_keyword("dynamic").expect_ok())) }; + get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; return util::format("{}->bind_root({})->with_meta({})->set_dynamic({})", - runtime::munge(var.native_name), + var, val.str(true), - runtime::munge(meta.unwrap().get().native_name), + meta.unwrap(), dynamic); } else { return util::format("{}->bind_root({})->with_meta(jank::runtime::jank_nil)", - runtime::munge(var.native_name), + var, val.str(true)); } } @@ -548,19 +580,19 @@ namespace jank::codegen if(meta.is_some()) { auto const dynamic{ truthy( - get(meta.unwrap().get().data, __rt_ctx->intern_keyword("dynamic").expect_ok())) }; + get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; util::format_to(body_buffer, "{}->bind_root({})->with_meta({})->set_dynamic({});", - runtime::munge(var.native_name), + var, val.str(true), - runtime::munge(meta.unwrap().get().native_name), + meta.unwrap(), dynamic); } else { util::format_to(body_buffer, "{}->bind_root({})->with_meta(jank::runtime::jank_nil);", - runtime::munge(var.native_name), + var, val.str(true)); } return none; @@ -571,17 +603,17 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::var_deref_ref const expr, analyze::expr::function_arity const &) { - auto const &var(expr->frame->find_lifted_var(expr->var->to_qualified_symbol()).unwrap().get()); + auto const &var(lift_var(lifted_vars, expr->var->to_qualified_symbol()->to_string())); switch(expr->position) { case analyze::expression_position::statement: case analyze::expression_position::value: { - return util::format("{}->deref()", runtime::munge(var.native_name)); + return util::format("{}->deref()", var); } case analyze::expression_position::tail: { - util::format_to(body_buffer, "return {}->deref();", runtime::munge(var.native_name)); + util::format_to(body_buffer, "return {}->deref();", var); return none; } } @@ -590,17 +622,17 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::var_ref_ref const expr, analyze::expr::function_arity const &) { - auto const &var(expr->frame->find_lifted_var(expr->qualified_name).unwrap().get()); + auto const &var(lift_var(lifted_vars, expr->qualified_name->to_string())); switch(expr->position) { case analyze::expression_position::statement: case analyze::expression_position::value: { - return runtime::munge(var.native_name); + return var; } case analyze::expression_position::tail: { - util::format_to(body_buffer, "return {};", runtime::munge(var.native_name)); + util::format_to(body_buffer, "return {};", var); return none; } } @@ -667,14 +699,7 @@ namespace jank::codegen } else { - auto const &constant(expr->frame->find_lifted_constant(expr->data).unwrap().get()); - - ret = runtime::munge(constant.native_name); - if(constant.unboxed_native_name.is_some()) - { - ret = { runtime::munge(constant.native_name), - runtime::munge(constant.unboxed_native_name.unwrap()) }; - } + ret = lift_constant(lifted_constants, expr->data); } switch(expr->position) @@ -2012,8 +2037,8 @@ namespace jank::codegen if(!generated_declaration) { profile::timer const timer{ util::format("cpp gen {}", root_fn->name) }; - build_header(); build_body(); + build_header(); build_footer(); generated_declaration = true; } @@ -2044,39 +2069,9 @@ namespace jank::codegen runtime::munge(struct_name.name)); { - /* TODO: Constants and vars are not shared across arities. We'd need stable names. */ - native_set used_vars, used_constants, used_captures; + native_set used_captures; for(auto const &arity : root_fn->arities) { - for(auto const &v : arity.frame->lifted_vars) - { - auto const hash{ v.first->to_hash() }; - if(used_vars.contains(hash)) - { - continue; - } - used_vars.emplace(hash); - - util::format_to(header_buffer, - "jank::runtime::var_ref const {};", - runtime::munge(v.second.native_name)); - } - - for(auto const &v : arity.frame->lifted_constants) - { - if(used_constants.contains(runtime::to_hash(v.first))) - { - continue; - } - used_constants.emplace(runtime::to_hash(v.first)); - - /* TODO: Typed lifted constants (in analysis). */ - util::format_to(header_buffer, - "{} const {};", - detail::gen_constant_type(v.second.data, true), - runtime::munge(v.second.native_name)); - } - /* TODO: More useful types here. */ for(auto const &v : arity.frame->captures) { @@ -2093,6 +2088,21 @@ namespace jank::codegen runtime::munge(v.second.native_name)); } } + + for(auto const &v : lifted_vars) + { + util::format_to(header_buffer, "jank::runtime::var_ref const {};", v.second); + } + + + for(auto const &v : lifted_constants) + { + /* TODO: Typed lifted constants (in analysis). */ + util::format_to(header_buffer, + "{} const {};", + detail::gen_constant_type(v.first, true), + v.second); + } } { @@ -2123,8 +2133,6 @@ namespace jank::codegen { native_set used_captures; - native_map used_vars; - native_map used_constants; util::format_to(header_buffer, ") : jank::runtime::obj::jit_function{ "); /* TODO: All of the meta in clojure.core alone costs 2s to JIT compile at run-time. * How can this be faster? */ @@ -2133,40 +2141,6 @@ namespace jank::codegen for(auto const &arity : root_fn->arities) { - for(auto &v : arity.frame->lifted_vars) - { - auto const hash{ v.first->to_hash() }; - auto const existing{ used_vars.find(hash) }; - if(existing != used_vars.end()) - { - v.second = existing->second; - continue; - } - used_vars.emplace(hash, v.second); - - util::format_to(header_buffer, - R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}", "{}").expect_ok() })", - runtime::munge(v.second.native_name), - v.second.var_name->ns, - v.second.var_name->name); - } - - for(auto &v : arity.frame->lifted_constants) - { - auto const hash{ runtime::to_hash(v.first) }; - auto const existing{ used_constants.find(hash) }; - if(existing != used_constants.end()) - { - v.second = existing->second; - continue; - } - used_constants.emplace(hash, v.second); - - util::format_to(header_buffer, ", {}{", runtime::munge(v.second.native_name)); - detail::gen_constant(v.second.data, header_buffer, true); - util::format_to(header_buffer, "}"); - } - for(auto const &v : arity.frame->captures) { auto const hash{ v.first->to_hash() }; @@ -2180,6 +2154,22 @@ namespace jank::codegen util::format_to(header_buffer, ", {}{ {} }", name, name); } } + + for(auto const &v : lifted_vars) + { + util::format_to(header_buffer, + R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}").expect_ok() })", + v.second, + v.first); + } + + + for(auto const &v : lifted_constants) + { + util::format_to(header_buffer, ", {}{", v.second); + detail::gen_constant(v.first, header_buffer, true); + util::format_to(header_buffer, "}"); + } } util::format_to(header_buffer, "{ }"); diff --git a/compiler+runtime/src/cpp/jank/evaluate.cpp b/compiler+runtime/src/cpp/jank/evaluate.cpp index aa2c1aab5..ae4632738 100644 --- a/compiler+runtime/src/cpp/jank/evaluate.cpp +++ b/compiler+runtime/src/cpp/jank/evaluate.cpp @@ -129,12 +129,7 @@ namespace jank::evaluate } /* Some expressions don't make sense to eval outright and aren't fns that can be JIT compiled. - * For those, we wrap them in a fn expression and then JIT compile and call them. - * - * There's an oddity here, since that expr wouldn't've been analyzed within a fn frame, so - * its lifted vars/constants, for example, aren't in a fn frame. Instead, they're put in the - * root frame. So, when wrapping this expr, we give the fn the root frame, but change its - * type to a fn frame. */ + * For those, we wrap them in a fn expression and then JIT compile and call them. */ template static expr::function_ref wrap_expression(jtl::ref const orig_expr, jtl::immutable_string const &name, @@ -147,8 +142,6 @@ namespace jank::evaluate ret->unique_name = __rt_ctx->unique_namespaced_string(ret->name); ret->meta = obj::persistent_hash_map::empty(); - auto const &closest_fn_frame(local_frame::find_closest_fn_frame(*expr->frame)); - auto const frame{ jtl::make_ref(local_frame::frame_type::fn, expr->frame->parent) }; auto const fn_ctx{ jtl::make_ref() }; @@ -158,16 +151,12 @@ namespace jank::evaluate fn_ctx }; expr->frame->parent = arity.frame; ret->frame = arity.frame->parent.unwrap_or(arity.frame); - ret->frame->lift_constant(ret->meta); fn_ctx->name = ret->name; fn_ctx->unique_name = ret->unique_name; fn_ctx->fn = ret; arity.frame->fn_ctx = fn_ctx; arity.fn_ctx = fn_ctx; - arity.frame->lifted_vars = closest_fn_frame.lifted_vars; - arity.frame->lifted_constants = closest_fn_frame.lifted_constants; - arity.fn_ctx->param_count = arity.params.size(); for(auto const sym : arity.params) { diff --git a/compiler+runtime/src/cpp/jank/runtime/context.cpp b/compiler+runtime/src/cpp/jank/runtime/context.cpp index 347f0b145..e2e12f8aa 100644 --- a/compiler+runtime/src/cpp/jank/runtime/context.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/context.cpp @@ -154,7 +154,7 @@ namespace jank::runtime return eval_string(file.expect_ok().view()); } - object_ref context::eval_string(jtl::immutable_string_view const &code) + object_ref context::eval_string(jtl::immutable_string const &code) { profile::timer const timer{ "rt eval_string" }; read::lex::processor l_prc{ code }; @@ -225,8 +225,7 @@ namespace jank::runtime return ret; } - jtl::result - context::eval_cpp_string(jtl::immutable_string_view const &code) const + jtl::result context::eval_cpp_string(jtl::immutable_string const &code) const { profile::timer const timer{ "rt eval_cpp_string" }; @@ -256,7 +255,7 @@ namespace jank::runtime return ok(); } - object_ref context::read_string(jtl::immutable_string_view const &code) + object_ref context::read_string(jtl::immutable_string const &code) { profile::timer const timer{ "rt read_string" }; @@ -278,7 +277,7 @@ namespace jank::runtime } native_vector - context::analyze_string(jtl::immutable_string_view const &code, bool const eval) + context::analyze_string(jtl::immutable_string const &code, bool const eval) { profile::timer const timer{ "rt analyze_string" }; read::lex::processor l_prc{ code }; @@ -308,7 +307,7 @@ namespace jank::runtime } jtl::result - context::load_module(jtl::immutable_string_view const &module, module::origin const ori) + context::load_module(jtl::immutable_string const &module, module::origin const ori) { auto const ns(current_ns()); @@ -345,7 +344,7 @@ namespace jank::runtime } } - jtl::result context::compile_module(jtl::immutable_string_view const &module) + jtl::result context::compile_module(jtl::immutable_string const &module) { module_dependencies.clear(); @@ -418,26 +417,25 @@ namespace jank::runtime return unique_namespaced_string("G_"); } - jtl::immutable_string - context::unique_namespaced_string(jtl::immutable_string_view const &prefix) const + jtl::immutable_string context::unique_namespaced_string(jtl::immutable_string const &prefix) const { static jtl::immutable_string const dot{ "\\." }; auto const ns{ current_ns() }; return util::format("{}-{}-{}", runtime::munge_and_replace(ns->name->get_name(), dot, "_"), - prefix.data(), + prefix.c_str(), ++ns->symbol_counter); } - jtl::immutable_string context::unique_munged_string() const + jtl::immutable_string context::unique_string() const { - return munge(unique_namespaced_string()); + return unique_string("G_"); } - jtl::immutable_string - context::unique_munged_string(jtl::immutable_string_view const &prefix) const + jtl::immutable_string context::unique_string(jtl::immutable_string const &prefix) const { - return munge(unique_namespaced_string(prefix)); + auto const ns{ current_ns() }; + return util::format("{}-{}", prefix.c_str(), ++ns->symbol_counter); } obj::symbol context::unique_symbol() const @@ -445,7 +443,7 @@ namespace jank::runtime return unique_symbol("G-"); } - obj::symbol context::unique_symbol(jtl::immutable_string_view const &prefix) const + obj::symbol context::unique_symbol(jtl::immutable_string const &prefix) const { return { "", unique_namespaced_string(prefix) }; } @@ -514,6 +512,12 @@ namespace jank::runtime return expect_object(current_ns_var->deref()); } + jtl::result + context::intern_var(jtl::immutable_string const &qualified_name) + { + return intern_var(make_box(qualified_name)); + } + jtl::result context::intern_var(jtl::immutable_string const &ns, jtl::immutable_string const &name) { From 65d38b6328ab74cd3c9eb62d10ddf89cd4b3f712 Mon Sep 17 00:00:00 2001 From: jeaye Date: Wed, 26 Nov 2025 19:54:15 -0800 Subject: [PATCH 46/58] Use explicit empty containers in cppgen --- .../src/cpp/jank/codegen/processor.cpp | 193 ++++++++++++------ 1 file changed, 132 insertions(+), 61 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 09ddef423..4de1a0230 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -273,103 +273,174 @@ namespace jank::codegen } else if constexpr(std::same_as) { - util::format_to(buffer, - "jank::runtime::make_box("); - if(typed_o->meta.is_some()) + if(typed_o->data.empty()) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\"), ", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "jank::runtime::obj::persistent_vector::empty()"); + if(typed_o->meta.is_some()) + { + /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ + util::format_to(buffer, + "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } } - util::format_to(buffer, "std::in_place "); - for(auto const &form : typed_o->data) + else { - util::format_to(buffer, ", "); - gen_constant(form, buffer, true); + util::format_to(buffer, + "jank::runtime::make_box("); + if(typed_o->meta.is_some()) + { + /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ + util::format_to(buffer, + "jank::runtime::__rt_ctx->read_string(\"{}\"), ", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } + util::format_to(buffer, "std::in_place "); + for(auto const &form : typed_o->data) + { + util::format_to(buffer, ", "); + gen_constant(form, buffer, true); + } + util::format_to(buffer, ")"); } - util::format_to(buffer, ")"); } else if constexpr(std::same_as) { - util::format_to(buffer, - "jank::runtime::make_box("); - if(typed_o->meta.is_some()) + if(typed_o->data.empty()) { - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\"), ", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "jank::runtime::obj::persistent_list::empty()"); + if(typed_o->meta.is_some()) + { + /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ + util::format_to(buffer, + "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } } - util::format_to(buffer, "std::in_place "); - for(auto const &form : typed_o->data) + else { - util::format_to(buffer, ", "); - gen_constant(form, buffer, true); + util::format_to(buffer, + "jank::runtime::make_box("); + if(typed_o->meta.is_some()) + { + util::format_to(buffer, + "jank::runtime::__rt_ctx->read_string(\"{}\"), ", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } + util::format_to(buffer, "std::in_place "); + for(auto const &form : typed_o->data) + { + util::format_to(buffer, ", "); + gen_constant(form, buffer, true); + } + util::format_to(buffer, ")"); } - util::format_to(buffer, ")"); } else if constexpr(std::same_as) { - util::format_to(buffer, - "jank::runtime::make_box("); - if(typed_o->meta.is_some()) + if(typed_o->data.empty()) { - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\"), ", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "jank::runtime::obj::persistent_hash_set::empty()"); + if(typed_o->meta.is_some()) + { + /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ + util::format_to(buffer, + "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } } - util::format_to(buffer, "std::in_place "); - for(auto const &form : typed_o->data) + else { - util::format_to(buffer, ", "); - gen_constant(form, buffer, true); + util::format_to(buffer, + "jank::runtime::make_box("); + if(typed_o->meta.is_some()) + { + util::format_to(buffer, + "jank::runtime::__rt_ctx->read_string(\"{}\"), ", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } + util::format_to(buffer, "std::in_place "); + for(auto const &form : typed_o->data) + { + util::format_to(buffer, ", "); + gen_constant(form, buffer, true); + } + util::format_to(buffer, ")"); } - util::format_to(buffer, ")"); } else if constexpr(std::same_as) { - bool need_comma{}; - if(typed_o->meta.is_some()) + if(typed_o->data.empty()) { - util::format_to(buffer, - "jank::runtime::obj::persistent_array_map::create_unique_with_meta("); - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\")", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); - need_comma = true; + util::format_to(buffer, "jank::runtime::obj::persistent_array_map::empty()"); + if(typed_o->meta.is_some()) + { + /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ + util::format_to(buffer, + "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } } else { - util::format_to(buffer, "jank::runtime::obj::persistent_array_map::create_unique("); - } - for(auto const &form : typed_o->data) - { - if(need_comma) + bool need_comma{}; + if(typed_o->meta.is_some()) + { + util::format_to( + buffer, + "jank::runtime::obj::persistent_array_map::create_unique_with_meta("); + util::format_to(buffer, + "jank::runtime::__rt_ctx->read_string(\"{}\")", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + need_comma = true; + } + else + { + util::format_to(buffer, "jank::runtime::obj::persistent_array_map::create_unique("); + } + for(auto const &form : typed_o->data) { + if(need_comma) + { + util::format_to(buffer, ", "); + } + need_comma = true; + gen_constant(form.first, buffer, true); util::format_to(buffer, ", "); + gen_constant(form.second, buffer, true); } - need_comma = true; - gen_constant(form.first, buffer, true); - util::format_to(buffer, ", "); - gen_constant(form.second, buffer, true); + util::format_to(buffer, ")"); } - util::format_to(buffer, ")"); } else if constexpr(std::same_as) { - auto const has_meta{ typed_o->meta.is_some() }; - if(has_meta) + if(typed_o->data.empty()) { - util::format_to(buffer, "jank::runtime::reset_meta("); + util::format_to(buffer, "jank::runtime::obj::persistent_hash_map::empty()"); + if(typed_o->meta.is_some()) + { + /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ + util::format_to(buffer, + "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } } - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\")", - util::escape(typed_o->to_code_string())); - if(has_meta) + else { + auto const has_meta{ typed_o->meta.is_some() }; + if(has_meta) + { + util::format_to(buffer, "jank::runtime::with_meta("); + } util::format_to(buffer, - ", jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + "jank::runtime::__rt_ctx->read_string(\"{}\")", + util::escape(typed_o->to_code_string())); + if(has_meta) + { + util::format_to(buffer, + ", jank::runtime::__rt_ctx->read_string(\"{}\"))", + util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + } } } /* Cons, etc. */ From 636a38a39db5ae4caf0edeefdb6bb545c430dcbf Mon Sep 17 00:00:00 2001 From: jeaye Date: Thu, 27 Nov 2025 15:37:00 -0800 Subject: [PATCH 47/58] Add empty string cppgen --- compiler+runtime/src/cpp/jank/codegen/processor.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 4de1a0230..a90f719d3 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -267,9 +267,16 @@ namespace jank::codegen } else if constexpr(std::same_as) { - util::format_to(buffer, - "jank::runtime::make_box({})", - typed_o->to_code_string()); + if(typed_o->data.empty()) + { + util::format_to(buffer, "jank::runtime::obj::persistent_string::empty()"); + } + else + { + util::format_to(buffer, + "jank::runtime::make_box({})", + typed_o->to_code_string()); + } } else if constexpr(std::same_as) { From b28b6c2b59d071f037b27b8f72d1c81dc46733fc Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 28 Nov 2025 17:52:45 -0800 Subject: [PATCH 48/58] More cppgen fixes --- .../include/cpp/jank/codegen/processor.hpp | 9 +++- compiler+runtime/include/cpp/jank/error.hpp | 10 ++++ .../include/cpp/jank/error/runtime.hpp | 1 + .../include/cpp/jank/runtime/context.hpp | 1 + compiler+runtime/include/cpp/jtl/result.hpp | 51 ++++++++++++++++--- .../src/cpp/jank/codegen/processor.cpp | 42 +++++++++------ compiler+runtime/src/cpp/jank/error.cpp | 12 +++++ .../src/cpp/jank/error/runtime.cpp | 7 +++ .../src/cpp/jank/runtime/context.cpp | 12 ++++- .../src/cpp/jank/runtime/module/loader.cpp | 15 +++++- 10 files changed, 135 insertions(+), 25 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index 51d40ee45..559f46ed4 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -170,7 +170,14 @@ namespace jank::codegen jtl::string_builder footer_buffer; jtl::string_builder expression_buffer; jtl::immutable_string expression_fn_name; - native_unordered_map lifted_vars; + + struct lifted_var + { + jtl::immutable_string native_name; + bool owned{}; + }; + + native_unordered_map lifted_vars; native_unordered_map, diff --git a/compiler+runtime/include/cpp/jank/error.hpp b/compiler+runtime/include/cpp/jank/error.hpp index accc37a99..7526cea62 100644 --- a/compiler+runtime/include/cpp/jank/error.hpp +++ b/compiler+runtime/include/cpp/jank/error.hpp @@ -381,6 +381,8 @@ namespace jank::error * is because cpptrace doesn't use our GC allocator. */ struct base : gc_cleanup { + static constexpr bool is_error{ true }; + base() = delete; base(base const &) = delete; base(base &&) noexcept = default; @@ -460,4 +462,12 @@ namespace jank { return jtl::make_ref(jtl::forward(args)...); } + + namespace error + { + error_ref internal_failure(jtl::immutable_string const &message); + /* This can be used by jtl helpers which can't reach into jank but which fail. */ + [[noreturn]] + void throw_internal_failure(jtl::immutable_string const &message); + } } diff --git a/compiler+runtime/include/cpp/jank/error/runtime.hpp b/compiler+runtime/include/cpp/jank/error/runtime.hpp index 1deab795f..706a2e5e0 100644 --- a/compiler+runtime/include/cpp/jank/error/runtime.hpp +++ b/compiler+runtime/include/cpp/jank/error/runtime.hpp @@ -9,6 +9,7 @@ namespace jank::error error_ref runtime_unable_to_open_file(jtl::immutable_string const &message); error_ref runtime_invalid_cpp_eval(); error_ref runtime_unable_to_load_module(jtl::immutable_string const &message); + error_ref runtime_unable_to_load_module(error_ref cause); error_ref internal_runtime_failure(jtl::immutable_string const &message); error_ref runtime_invalid_unbox(jtl::immutable_string const &message, read::source const &unbox_source); diff --git a/compiler+runtime/include/cpp/jank/runtime/context.hpp b/compiler+runtime/include/cpp/jank/runtime/context.hpp index 50adda9ea..80efc9405 100644 --- a/compiler+runtime/include/cpp/jank/runtime/context.hpp +++ b/compiler+runtime/include/cpp/jank/runtime/context.hpp @@ -64,6 +64,7 @@ namespace jank::runtime jtl::result intern_var(jtl::immutable_string const &ns, jtl::immutable_string const &name); jtl::result intern_owned_var(obj::symbol_ref const &); + jtl::result intern_owned_var(jtl::immutable_string const &); jtl::result intern_owned_var(jtl::immutable_string const &ns, jtl::immutable_string const &name); var_ref find_var(obj::symbol_ref const &); diff --git a/compiler+runtime/include/cpp/jtl/result.hpp b/compiler+runtime/include/cpp/jtl/result.hpp index 6162aa035..946ff95d5 100644 --- a/compiler+runtime/include/cpp/jtl/result.hpp +++ b/compiler+runtime/include/cpp/jtl/result.hpp @@ -6,6 +6,12 @@ #include #include +namespace jank::error +{ + [[noreturn]] + void throw_internal_failure(jtl::immutable_string const &message); +} + namespace jtl { namespace detail @@ -30,6 +36,37 @@ namespace jtl struct result { }; + + template + [[noreturn]] + constexpr void panic(Result const &r) + { + using E = typename Result::error_type; + + /* A result can hold any type of error type, but when we expect a value + * and it's not there, we only want to throw a jank::error_ref. This + * not only makes catching easier, it also fits into our error reporting. + * + * So we need to do some work here to see if we have an error_ref, something + * we can use to build an error_ref (like a string), or just something else. */ + + /* This is a roundabout way of looking for error_ref. */ + if constexpr(requires(E t) { E::value_type::is_error; }) + { + throw r.expect_err(); + } + else if constexpr(jtl::is_same) + { + jank::error::throw_internal_failure(r.expect_err()); + } + else + { + immutable_string s{ "Unexpected result<" }; + s = s + type_name().data(); + s = s + ">"; + jank::error::throw_internal_failure(s); + } + } } constexpr detail::result ok() noexcept @@ -54,6 +91,9 @@ namespace jtl { static_assert(!std::same_as, "Result and error type must be different."); + using value_type = R; + using error_type = E; + constexpr result(detail::result &&r) noexcept : data{ R{ std::move(r.data) } } { @@ -112,9 +152,7 @@ namespace jtl return; } - /* TODO: Update all of these throws to throw a consistent type, regardless of the - * error type. This simplifies our catching logic. */ - throw expect_err(); + detail::panic(*this); } constexpr R const &expect_ok() const @@ -183,9 +221,8 @@ namespace jtl constexpr R unwrap_move() { if(!is_ok()) - /* TODO: Panic function. */ { - throw expect_err(); + detail::panic(*this); } return std::move(std::get(data)); } @@ -245,6 +282,8 @@ namespace jtl template struct [[nodiscard]] result { + using error_type = E; + constexpr result(detail::result &&) noexcept : data{ void_t{} } { @@ -288,7 +327,7 @@ namespace jtl return; } - throw expect_err(); + detail::panic(*this); } constexpr void expect_ok() const diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index a90f719d3..3de59e04c 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -23,8 +23,8 @@ * jank fn has a nested fn, it becomes a nested struct, since this whole * generation works recursively. * - * Analysis lifts constants and vars, so those just become members which are - * initialized in the ctor. + * During codegen, we lift constants and vars, so those just become members which + * are initialized in the ctor. * * The most interesting part is the translation of expressions into statements, * so that something like `(println (if foo bar spam))` can become sane C++. @@ -563,20 +563,21 @@ namespace jank::codegen } static jtl::immutable_string - lift_var(native_unordered_map &lifted_vars, - jtl::immutable_string const &qualified_name) + lift_var(native_unordered_map &lifted_vars, + jtl::immutable_string const &qualified_name, + bool const owned) { auto const existing{ lifted_vars.find(qualified_name) }; if(existing != lifted_vars.end()) { - return existing->second; + return existing->second.native_name; } static jtl::immutable_string const dot{ "\\." }; auto const us{ __rt_ctx->unique_string(qualified_name) }; auto const native_name{ runtime::munge_and_replace(us, dot, "_") }; //util::println("lifting var '{}' with '{}' as '{}'", qualified_name, us, native_name); - lifted_vars.emplace(qualified_name, native_name); + lifted_vars.emplace(qualified_name, processor::lifted_var{ native_name, owned }); return native_name; } @@ -602,7 +603,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::def_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &var(lift_var(lifted_vars, expr->name->to_string())); + auto const &var(lift_var(lifted_vars, expr->name->to_string(), true)); auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(var))); jtl::option meta; @@ -681,7 +682,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::var_deref_ref const expr, analyze::expr::function_arity const &) { - auto const &var(lift_var(lifted_vars, expr->var->to_qualified_symbol()->to_string())); + auto const &var(lift_var(lifted_vars, expr->var->to_qualified_symbol()->to_string(), false)); switch(expr->position) { case analyze::expression_position::statement: @@ -700,7 +701,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::var_ref_ref const expr, analyze::expr::function_arity const &) { - auto const &var(lift_var(lifted_vars, expr->qualified_name->to_string())); + auto const &var(lift_var(lifted_vars, expr->qualified_name->to_string(), false)); switch(expr->position) { case analyze::expression_position::statement: @@ -2169,7 +2170,7 @@ namespace jank::codegen for(auto const &v : lifted_vars) { - util::format_to(header_buffer, "jank::runtime::var_ref const {};", v.second); + util::format_to(header_buffer, "jank::runtime::var_ref const {};", v.second.native_name); } @@ -2235,10 +2236,20 @@ namespace jank::codegen for(auto const &v : lifted_vars) { - util::format_to(header_buffer, - R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}").expect_ok() })", - v.second, - v.first); + if(v.second.owned) + { + util::format_to(header_buffer, + R"(, {}{ jank::runtime::__rt_ctx->intern_owned_var("{}").expect_ok() })", + v.second.native_name, + v.first); + } + else + { + util::format_to(header_buffer, + R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}").expect_ok() })", + v.second.native_name, + v.first); + } } @@ -2362,7 +2373,7 @@ namespace jank::codegen if(target == compilation_target::module) { util::format_to(footer_buffer, - "extern \"C\" void* {}(){", + "void* {}(){", runtime::module::module_to_load_function(module)); /* First thing we do when loading this module is to intern our ns. Everything else will @@ -2390,6 +2401,7 @@ namespace jank::codegen "return {}::{}{ }.call().erase();", runtime::module::module_to_native_ns(module), runtime::munge(struct_name.name)); + util::format_to(footer_buffer, "}"); } } diff --git a/compiler+runtime/src/cpp/jank/error.cpp b/compiler+runtime/src/cpp/jank/error.cpp index 45c131497..cae8deb79 100644 --- a/compiler+runtime/src/cpp/jank/error.cpp +++ b/compiler+runtime/src/cpp/jank/error.cpp @@ -471,4 +471,16 @@ namespace jank::error { return os << "error(" << kind_str(e.kind) << " - " << e.source << ", \"" << e.message << "\")"; } + + error_ref internal_failure(jtl::immutable_string const &message) + { + auto const e{ make_error(kind::internal_failure, message, read::source::unknown) }; + e->trace = std::make_unique(cpptrace::generate_trace()); + return e; + } + + void throw_internal_failure(jtl::immutable_string const &message) + { + throw internal_failure(message); + } } diff --git a/compiler+runtime/src/cpp/jank/error/runtime.cpp b/compiler+runtime/src/cpp/jank/error/runtime.cpp index f84a5327f..bac4d49ab 100644 --- a/compiler+runtime/src/cpp/jank/error/runtime.cpp +++ b/compiler+runtime/src/cpp/jank/error/runtime.cpp @@ -29,6 +29,13 @@ namespace jank::error return make_error(kind::runtime_unable_to_load_module, message, read::source::unknown); } + error_ref runtime_unable_to_load_module(error_ref const cause) + { + auto const e{ make_error(kind::runtime_unable_to_load_module, read::source::unknown) }; + e->cause = cause; + return e; + } + error_ref internal_runtime_failure(jtl::immutable_string const &message) { return make_error(kind::internal_runtime_failure, message, read::source::unknown); diff --git a/compiler+runtime/src/cpp/jank/runtime/context.cpp b/compiler+runtime/src/cpp/jank/runtime/context.cpp index e2e12f8aa..c7595fb3f 100644 --- a/compiler+runtime/src/cpp/jank/runtime/context.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/context.cpp @@ -338,10 +338,14 @@ namespace jank::runtime { return error::runtime_unable_to_load_module(e.what()); } - catch(object_ref const &e) + catch(object_ref const e) { return error::runtime_unable_to_load_module(runtime::to_code_string(e)); } + catch(error_ref const e) + { + return error::runtime_unable_to_load_module(e); + } } jtl::result context::compile_module(jtl::immutable_string const &module) @@ -551,6 +555,12 @@ namespace jank::runtime return intern_owned_var(make_box(ns, name)); } + jtl::result + context::intern_owned_var(jtl::immutable_string const &qualified_name) + { + return intern_owned_var(make_box(qualified_name)); + } + jtl::result context::intern_owned_var(obj::symbol_ref const &qualified_sym) { diff --git a/compiler+runtime/src/cpp/jank/runtime/module/loader.cpp b/compiler+runtime/src/cpp/jank/runtime/module/loader.cpp index 2687ff978..178c6fe4f 100644 --- a/compiler+runtime/src/cpp/jank/runtime/module/loader.cpp +++ b/compiler+runtime/src/cpp/jank/runtime/module/loader.cpp @@ -992,8 +992,19 @@ namespace jank::runtime::module __rt_ctx->jit_prc.load_object(entry.path); } - auto const load{ __rt_ctx->jit_prc.find_symbol(load_function_name).expect_ok() }; - reinterpret_cast(load)(); + /* For C++ codegen, we don't use extern "C" for the load fn, since it's UB + * to throw exceptions across C boundaries. Instead, we just declare the function + * and then try to call it. */ + auto const load_fn_res{ __rt_ctx->jit_prc.find_symbol(load_function_name) }; + if(load_fn_res.is_ok()) + { + reinterpret_cast(load_fn_res.expect_ok())(); + } + else + { + __rt_ctx->jit_prc.eval_string( + util::format("void* {}(); {}();", load_function_name, load_function_name)); + } return ok(); } From 5a162b2aabc6f1e744b647b2b92105e23644f9d8 Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 28 Nov 2025 18:11:57 -0800 Subject: [PATCH 49/58] Fix var interning order for cppgen --- .../src/cpp/jank/codegen/processor.cpp | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 3de59e04c..f275dd0a6 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -603,8 +603,17 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::def_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &var(lift_var(lifted_vars, expr->name->to_string(), true)); - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(var))); + /* def uses a var, but we don't lift it. Even if it's lifted by another usage, + * it'll be re-interned here as an owned var. This needs to happen at the point + * of the def, rather than prior (i.e. due to lifting), since there could be + * some other var-related effects such as refer which need to happen before + * def. */ + auto var_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("var"))); + util::format_to( + body_buffer, + R"(auto const {}(jank::runtime::__rt_ctx->intern_owned_var("{}").expect_ok());)", + var_tmp, + expr->name->to_string()); jtl::option meta; if(expr->name->meta.is_some()) @@ -619,11 +628,11 @@ namespace jank::codegen { auto const dynamic{ truthy( get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; - return util::format("{}->with_meta({})->set_dynamic({})", var, meta.unwrap(), dynamic); + return util::format("{}->with_meta({})->set_dynamic({})", var_tmp, meta.unwrap(), dynamic); } else { - return util::format("{}->with_meta(jank::runtime::jank_nil)", var); + return util::format("{}->with_meta(jank::runtime::jank_nil)", var_tmp); } } @@ -637,7 +646,7 @@ namespace jank::codegen auto const dynamic{ truthy( get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; return util::format("{}->bind_root({})->with_meta({})->set_dynamic({})", - var, + var_tmp, val.str(true), meta.unwrap(), dynamic); @@ -645,7 +654,7 @@ namespace jank::codegen else { return util::format("{}->bind_root({})->with_meta(jank::runtime::jank_nil)", - var, + var_tmp, val.str(true)); } } @@ -662,7 +671,7 @@ namespace jank::codegen get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; util::format_to(body_buffer, "{}->bind_root({})->with_meta({})->set_dynamic({});", - var, + var_tmp, val.str(true), meta.unwrap(), dynamic); @@ -671,7 +680,7 @@ namespace jank::codegen { util::format_to(body_buffer, "{}->bind_root({})->with_meta(jank::runtime::jank_nil);", - var, + var_tmp, val.str(true)); } return none; From 5f9a6e31d1cce2e7142dc11e523f06e681c146f2 Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 28 Nov 2025 18:27:36 -0800 Subject: [PATCH 50/58] Remove some dead code --- .../src/cpp/jank/codegen/processor.cpp | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index f275dd0a6..0d7207e8d 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -471,11 +471,6 @@ namespace jank::codegen }, o); } - - static jtl::immutable_string boxed_local_name(jtl::immutable_string const &local_name) - { - return local_name; // + "__boxed"; - } } handle::handle(jtl::immutable_string const &name, bool const boxed) @@ -518,7 +513,7 @@ namespace jank::codegen else if(binding->has_boxed_usage) { unboxed_name = runtime::munge(binding->native_name); - boxed_name = detail::boxed_local_name(unboxed_name); + boxed_name = unboxed_name; } else { @@ -996,17 +991,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::local_reference_ref const expr, analyze::expr::function_arity const &) { - auto const munged_name(runtime::munge(expr->binding->native_name)); - - handle ret; - if(expr->binding->needs_box) - { - ret = munged_name; - } - else - { - ret = handle{ detail::boxed_local_name(munged_name), munged_name }; - } + auto const ret(runtime::munge(expr->binding->native_name)); switch(expr->position) { @@ -1017,7 +1002,7 @@ namespace jank::codegen } case analyze::expression_position::tail: { - util::format_to(body_buffer, "return {};", ret.str(expr->needs_box)); + util::format_to(body_buffer, "return {};", ret); return none; } } From a1ead4f49820fd05c98443f778c5260b1833ae08 Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 28 Nov 2025 18:38:46 -0800 Subject: [PATCH 51/58] Clean up some cppgen --- .../src/cpp/jank/codegen/processor.cpp | 212 +++++++----------- 1 file changed, 76 insertions(+), 136 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 0d7207e8d..c0f6e59a8 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -75,73 +74,45 @@ namespace jank::codegen switch(o->type) { case jank::runtime::object_type::nil: - { - return "jank::runtime::obj::nil_ref"; - } + return "jank::runtime::obj::nil_ref"; case jank::runtime::object_type::boolean: - { - return "jank::runtime::obj::boolean_ref"; - } + return "jank::runtime::obj::boolean_ref"; case jank::runtime::object_type::integer: + if(boxed) { - if(boxed) - { - return "jank::runtime::obj::integer_ref"; - } - return "jank::i64"; + return "jank::runtime::obj::integer_ref"; } + return "jank::i64"; case jank::runtime::object_type::character: + if(boxed) { - if(boxed) - { - return "jank::runtime::obj::character_ref"; - } - return "jank::runtime::obj::character"; + return "jank::runtime::obj::character_ref"; } + return "jank::runtime::obj::character"; case jank::runtime::object_type::real: + if(boxed) { - if(boxed) - { - return "jank::runtime::obj::real_ref"; - } - return "jank::f64"; + return "jank::runtime::obj::real_ref"; } + return "jank::f64"; case jank::runtime::object_type::symbol: - { - return "jank::runtime::obj::symbol_ref"; - } + return "jank::runtime::obj::symbol_ref"; case jank::runtime::object_type::keyword: - { - return "jank::runtime::obj::keyword_ref"; - } + return "jank::runtime::obj::keyword_ref"; case jank::runtime::object_type::persistent_string: - { - return "jank::runtime::obj::persistent_string_ref"; - } + return "jank::runtime::obj::persistent_string_ref"; case jank::runtime::object_type::persistent_list: - { - return "jank::runtime::obj::persistent_list_ref"; - } + return "jank::runtime::obj::persistent_list_ref"; case jank::runtime::object_type::persistent_vector: - { - return "jank::runtime::obj::persistent_vector_ref"; - } + return "jank::runtime::obj::persistent_vector_ref"; case jank::runtime::object_type::persistent_hash_set: - { - return "jank::runtime::obj::persistent_hash_set_ref"; - } + return "jank::runtime::obj::persistent_hash_set_ref"; case jank::runtime::object_type::persistent_array_map: - { - return "jank::runtime::obj::persistent_array_map_ref"; - } + return "jank::runtime::obj::persistent_array_map_ref"; case jank::runtime::object_type::var: - { - return "jank::runtime::var_ref"; - } + return "jank::runtime::var_ref"; default: - { - return "jank::runtime::object_ref"; - } + return "jank::runtime::object_ref"; } #pragma clang diagnostic pop } @@ -635,51 +606,45 @@ namespace jank::codegen switch(expr->position) { case analyze::expression_position::value: + if(meta.is_some()) { - if(meta.is_some()) - { - auto const dynamic{ truthy( - get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; - return util::format("{}->bind_root({})->with_meta({})->set_dynamic({})", - var_tmp, - val.str(true), - meta.unwrap(), - dynamic); - } - else - { - return util::format("{}->bind_root({})->with_meta(jank::runtime::jank_nil)", - var_tmp, - val.str(true)); - } + auto const dynamic{ truthy( + get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; + return util::format("{}->bind_root({})->with_meta({})->set_dynamic({})", + var_tmp, + val.str(true), + meta.unwrap(), + dynamic); } - case analyze::expression_position::tail: + else { - util::format_to(body_buffer, "return "); + return util::format("{}->bind_root({})->with_meta(jank::runtime::jank_nil)", + var_tmp, + val.str(true)); } + case analyze::expression_position::tail: + util::format_to(body_buffer, "return "); [[fallthrough]]; case analyze::expression_position::statement: + if(meta.is_some()) { - if(meta.is_some()) - { - auto const dynamic{ truthy( - get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; - util::format_to(body_buffer, - "{}->bind_root({})->with_meta({})->set_dynamic({});", - var_tmp, - val.str(true), - meta.unwrap(), - dynamic); - } - else - { - util::format_to(body_buffer, - "{}->bind_root({})->with_meta(jank::runtime::jank_nil);", - var_tmp, - val.str(true)); - } - return none; + auto const dynamic{ truthy( + get(expr->name->meta.unwrap(), __rt_ctx->intern_keyword("dynamic").expect_ok())) }; + util::format_to(body_buffer, + "{}->bind_root({})->with_meta({})->set_dynamic({});", + var_tmp, + val.str(true), + meta.unwrap(), + dynamic); } + else + { + util::format_to(body_buffer, + "{}->bind_root({})->with_meta(jank::runtime::jank_nil);", + var_tmp, + val.str(true)); + } + return none; } } @@ -691,14 +656,10 @@ namespace jank::codegen { case analyze::expression_position::statement: case analyze::expression_position::value: - { - return util::format("{}->deref()", var); - } + return util::format("{}->deref()", var); case analyze::expression_position::tail: - { - util::format_to(body_buffer, "return {}->deref();", var); - return none; - } + util::format_to(body_buffer, "return {}->deref();", var); + return none; } } @@ -710,14 +671,10 @@ namespace jank::codegen { case analyze::expression_position::statement: case analyze::expression_position::value: - { - return var; - } + return var; case analyze::expression_position::tail: - { - util::format_to(body_buffer, "return {};", var); - return none; - } + util::format_to(body_buffer, "return {};", var); + return none; } } @@ -789,14 +746,10 @@ namespace jank::codegen { case analyze::expression_position::statement: case analyze::expression_position::value: - { - return ret; - } + return ret; case analyze::expression_position::tail: - { - util::format_to(body_buffer, "return {};", ret.str(expr->needs_box)); - return none; - } + util::format_to(body_buffer, "return {};", ret.str(expr->needs_box)); + return none; } } @@ -997,14 +950,10 @@ namespace jank::codegen { case analyze::expression_position::statement: case analyze::expression_position::value: - { - return ret; - } + return ret; case analyze::expression_position::tail: - { - util::format_to(body_buffer, "return {};", ret); - return none; - } + util::format_to(body_buffer, "return {};", ret); + return none; } } @@ -1023,15 +972,10 @@ namespace jank::codegen { case analyze::expression_position::statement: case analyze::expression_position::value: - /* TODO: Return a handle. */ - { - return prc.expression_str(); - } + return prc.expression_str(); case analyze::expression_position::tail: - { - util::format_to(body_buffer, "return {};", prc.expression_str()); - return none; - } + util::format_to(body_buffer, "return {};", prc.expression_str()); + return none; } } @@ -1347,21 +1291,17 @@ namespace jank::codegen { case analyze::expression_position::statement: case analyze::expression_position::value: + return last; + case analyze::expression_position::tail: + if(last.is_none()) { - return last; + util::format_to(body_buffer, "return jank::runtime::jank_nil;"); } - case analyze::expression_position::tail: + else { - if(last.is_none()) - { - util::format_to(body_buffer, "return jank::runtime::jank_nil;"); - } - else - { - util::format_to(body_buffer, "return {};", last.unwrap().str(expr->needs_box)); - } - return none; + util::format_to(body_buffer, "return {};", last.unwrap().str(expr->needs_box)); } + return none; } } @@ -2110,6 +2050,9 @@ namespace jank::codegen if(!generated_declaration) { profile::timer const timer{ util::format("cpp gen {}", root_fn->name) }; + + /* We generate the body first so that we know what we need for the header. This is + * necessary since we end up lifting vars and constants while building the body. */ build_body(); build_header(); build_footer(); @@ -2124,9 +2067,6 @@ namespace jank::codegen ret += jtl::immutable_string_view{ body_buffer.data(), body_buffer.size() }; ret += jtl::immutable_string_view{ footer_buffer.data(), footer_buffer.size() }; - //ret = util::format_cpp_source(ret).expect_ok(); - - //util::println("codegen declaration {}", ret); return ret; } From f2ad20e239e29a17fdf4916bb1c6bfec322e7f8e Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 28 Nov 2025 18:44:18 -0800 Subject: [PATCH 52/58] Remove some dead cppgen code --- .../src/cpp/jank/codegen/processor.cpp | 53 +++---------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index c0f6e59a8..762a406f3 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -66,7 +66,6 @@ namespace jank::codegen * the actual param names as mutable locals outside of the while loop. */ constexpr jtl::immutable_string_view const recur_suffix{ "__recur" }; - /* TODO: Consider making this a on the typed object: the C++ name. */ static jtl::immutable_string gen_constant_type(runtime::object_ref const o, bool const boxed) { #pragma clang diagnostic push @@ -444,18 +443,10 @@ namespace jank::codegen } } - handle::handle(jtl::immutable_string const &name, bool const boxed) + handle::handle(jtl::immutable_string const &name, bool const) { - if(boxed) - { - boxed_name = name; - unboxed_name = boxed_name; - } - else - { - unboxed_name = name; - boxed_name = util::format("jank::runtime::make_box({})", unboxed_name); - } + boxed_name = name; + unboxed_name = boxed_name; } handle::handle(jtl::immutable_string const &boxed_name) @@ -470,43 +461,19 @@ namespace jank::codegen { if(this->boxed_name.empty()) { - this->boxed_name = util::format("jank::runtime::make_box({})", unboxed_name); + this->boxed_name = unboxed_name; } } handle::handle(analyze::local_binding_ptr const binding) { - if(binding->needs_box) - { - boxed_name = runtime::munge(binding->native_name); - unboxed_name = boxed_name; - } - else if(binding->has_boxed_usage) - { - unboxed_name = runtime::munge(binding->native_name); - boxed_name = unboxed_name; - } - else - { - unboxed_name = runtime::munge(binding->native_name); - } + boxed_name = runtime::munge(binding->native_name); + unboxed_name = boxed_name; } - jtl::immutable_string handle::str([[maybe_unused]] bool const needs_box) const + jtl::immutable_string handle::str(bool const) const { return boxed_name; - //if(needs_box) - //{ - // if(boxed_name.empty()) - // { - // throw std::runtime_error{ util::format("Missing boxed name for handle {}", unboxed_name) }; - // } - // return boxed_name; - //} - //else - //{ - // return unboxed_name; - //} } processor::processor(analyze::expr::function_ref const expr, @@ -542,7 +509,6 @@ namespace jank::codegen static jtl::immutable_string const dot{ "\\." }; auto const us{ __rt_ctx->unique_string(qualified_name) }; auto const native_name{ runtime::munge_and_replace(us, dot, "_") }; - //util::println("lifting var '{}' with '{}' as '{}'", qualified_name, us, native_name); lifted_vars.emplace(qualified_name, processor::lifted_var{ native_name, owned }); return native_name; } @@ -561,7 +527,6 @@ namespace jank::codegen } auto const &native_name{ runtime::munge(__rt_ctx->unique_namespaced_string("const")) }; - //util::println("lifting constant {} as {}", runtime::to_code_string(o), native_name); lifted_constants.emplace(o, native_name); return native_name; } @@ -624,6 +589,7 @@ namespace jank::codegen } case analyze::expression_position::tail: util::format_to(body_buffer, "return "); + [[fallthrough]]; case analyze::expression_position::statement: if(meta.is_some()) @@ -683,13 +649,10 @@ namespace jank::codegen native_vector const &arg_exprs, analyze::expr::function_arity const &fn_arity) { - //util::println("format_dynamic_call source {}", source_tmp); native_vector arg_tmps; arg_tmps.reserve(arg_exprs.size()); for(auto const &arg_expr : arg_exprs) { - //util::println("\tformat_dynamic_call arg {}", - // runtime::to_code_string(arg_expr->to_runtime_data())); arg_tmps.emplace_back(gen(arg_expr, fn_arity).unwrap()); } From 1374093ce979186c2d5b3153ff772663256e0dc7 Mon Sep 17 00:00:00 2001 From: jeaye Date: Fri, 28 Nov 2025 18:52:32 -0800 Subject: [PATCH 53/58] Don't use namespaced strings for cppgen --- .../src/cpp/jank/codegen/processor.cpp | 48 +++++++++---------- .../src/cpp/jank/jit/processor.cpp | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 762a406f3..986ee5b72 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -526,7 +526,7 @@ namespace jank::codegen return existing->second; } - auto const &native_name{ runtime::munge(__rt_ctx->unique_namespaced_string("const")) }; + auto const &native_name{ runtime::munge(__rt_ctx->unique_string("const")) }; lifted_constants.emplace(o, native_name); return native_name; } @@ -539,7 +539,7 @@ namespace jank::codegen * of the def, rather than prior (i.e. due to lifting), since there could be * some other var-related effects such as refer which need to happen before * def. */ - auto var_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("var"))); + auto var_tmp(runtime::munge(__rt_ctx->unique_string("var"))); util::format_to( body_buffer, R"(auto const {}(jank::runtime::__rt_ctx->intern_owned_var("{}").expect_ok());)", @@ -671,7 +671,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::call_ref const expr, analyze::expr::function_arity const &fn_arity) { - handle ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("call")) }; + handle ret_tmp{ runtime::munge(__rt_ctx->unique_string("call")) }; auto const &source_tmp(gen(expr->source_expr, fn_arity)); format_dynamic_call(source_tmp.unwrap().str(true), ret_tmp.str(true), @@ -726,7 +726,7 @@ namespace jank::codegen data_tmps.emplace_back(gen(data_expr, fn_arity).unwrap()); } - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("list"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("list"))); util::format_to(body_buffer, "auto const {}(jank::runtime::make_box(", ret_tmp); @@ -762,7 +762,7 @@ namespace jank::codegen data_tmps.emplace_back(gen(data_expr, fn_arity).unwrap()); } - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("vec"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("vec"))); util::format_to(body_buffer, "auto const {}(jank::runtime::make_box(", ret_tmp); @@ -799,7 +799,7 @@ namespace jank::codegen gen(data_expr.second, fn_arity).unwrap()); } - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("map"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("map"))); /* Jump right to a hash map, if we have enough values. */ if(expr->data_exprs.size() <= runtime::obj::persistent_array_map::max_size) @@ -877,7 +877,7 @@ namespace jank::codegen data_tmps.emplace_back(gen(data_expr, fn_arity).unwrap()); } - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("set"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("set"))); util::format_to( body_buffer, "auto const {}(jank::runtime::make_box(", @@ -1003,7 +1003,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::named_recursion_ref const expr, analyze::expr::function_arity const &fn_arity) { - handle ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("named_recursion")) }; + handle ret_tmp{ runtime::munge(__rt_ctx->unique_string("named_recursion")) }; auto const &source_tmp( gen(jtl::ref{ &expr->recursion_ref }, fn_arity)); @@ -1024,7 +1024,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::let_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("let")) }; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_string("let")) }; bool used_option{}; auto const last_expr_type{ cpp_util::expression_type( @@ -1139,7 +1139,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::letfn_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("letfn")) }; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_string("letfn")) }; bool used_option{}; auto const last_expr_type{ cpp_util::expression_type( @@ -1271,7 +1271,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::if_ref const expr, analyze::expr::function_arity const &fn_arity) { - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("if"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("if"))); auto const expr_type{ cpp_util::expression_type(expr->then) }; util::format_to(body_buffer, "{} {}{ };", @@ -1337,7 +1337,7 @@ namespace jank::codegen processor::gen(analyze::expr::try_ref const expr, analyze::expr::function_arity const &fn_arity) { auto const has_catch{ expr->catch_body.is_some() }; - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("try"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("try"))); util::format_to(body_buffer, "jank::runtime::object_ref {}{ };", ret_tmp); util::format_to(body_buffer, "{"); @@ -1406,7 +1406,7 @@ namespace jank::codegen processor::gen(analyze::expr::case_ref const expr, analyze::expr::function_arity const &fn_arity) { auto const is_tail{ expr->position == analyze::expression_position::tail }; - auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("case")) }; + auto const &ret_tmp{ runtime::munge(__rt_ctx->unique_string("case")) }; util::format_to(body_buffer, "jank::runtime::object_ref {}{ };", ret_tmp); @@ -1506,7 +1506,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_cast_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_cast"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("cpp_cast"))); auto const value_tmp{ gen(expr->value_expr, arity) }; if(Cpp::IsVoid(expr->conversion_type)) @@ -1543,7 +1543,7 @@ namespace jank::codegen if(expr->source_expr->kind == expression_kind::cpp_value) { auto const source{ static_cast(expr->source_expr.data) }; - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_call"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("cpp_call"))); native_vector arg_tmps; arg_tmps.reserve(expr->arg_exprs.size()); @@ -1606,7 +1606,7 @@ namespace jank::codegen } else { - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_call"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("cpp_call"))); auto const source_tmp{ gen(expr->source_expr, arity).unwrap() }; @@ -1665,7 +1665,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_constructor_call_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_ctor"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("cpp_ctor"))); native_vector arg_tmps; arg_tmps.reserve(expr->arg_exprs.size()); @@ -1751,7 +1751,7 @@ namespace jank::codegen analyze::expr::function_arity const &arity) { auto const fn_name{ Cpp::GetName(expr->fn) }; - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(fn_name))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string(fn_name))); native_vector arg_tmps; arg_tmps.reserve(expr->arg_exprs.size()); @@ -1815,7 +1815,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_member_access_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string(expr->name))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string(expr->name))); auto obj_tmp(gen(expr->obj_expr, arity)); util::format_to(body_buffer, @@ -1837,7 +1837,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_builtin_operator_call_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp(runtime::munge(__rt_ctx->unique_namespaced_string("cpp_operator"))); + auto ret_tmp(runtime::munge(__rt_ctx->unique_string("cpp_operator"))); native_vector arg_tmps; arg_tmps.reserve(expr->arg_exprs.size()); @@ -1882,7 +1882,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_box_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_box")) }; + auto ret_tmp{ runtime::munge(__rt_ctx->unique_string("cpp_box")) }; auto value_tmp{ gen(expr->value_expr, arity) }; auto const value_expr_type{ cpp_util::expression_type(expr->value_expr) }; auto const type_str{ Cpp::GetTypeAsString( @@ -1913,7 +1913,7 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_unbox_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_unbox")) }; + auto ret_tmp{ runtime::munge(__rt_ctx->unique_string("cpp_unbox")) }; auto value_tmp{ gen(expr->value_expr, arity) }; auto const type_name{ cpp_util::get_qualified_type_name(expr->type) }; auto const meta{ runtime::source_to_meta(expr->source) }; @@ -1941,8 +1941,8 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::cpp_new_ref const expr, analyze::expr::function_arity const &arity) { - auto ret_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("cpp_new")) }; - auto finalizer_tmp{ runtime::munge(__rt_ctx->unique_namespaced_string("finalizer")) }; + auto ret_tmp{ runtime::munge(__rt_ctx->unique_string("cpp_new")) }; + auto finalizer_tmp{ runtime::munge(__rt_ctx->unique_string("finalizer")) }; auto value_tmp{ gen(expr->value_expr, arity) }; auto const type_name{ cpp_util::get_qualified_type_name(expr->type) }; diff --git a/compiler+runtime/src/cpp/jank/jit/processor.cpp b/compiler+runtime/src/cpp/jank/jit/processor.cpp index 8dcc7512b..82443ad8e 100644 --- a/compiler+runtime/src/cpp/jank/jit/processor.cpp +++ b/compiler+runtime/src/cpp/jank/jit/processor.cpp @@ -232,7 +232,7 @@ namespace jank::jit void processor::eval_string(jtl::immutable_string const &s) const { - return eval_string(s, nullptr); + eval_string(s, nullptr); } void processor::eval_string(jtl::immutable_string const &s, clang::Value * const ret) const From 4bfdaca2ec5226ab8b73a2ec09be4c91182d61dc Mon Sep 17 00:00:00 2001 From: jeaye Date: Sun, 30 Nov 2025 18:55:06 -0800 Subject: [PATCH 54/58] Start changing module gen --- .../include/cpp/jtl/string_builder.hpp | 1 + .../src/cpp/jank/codegen/processor.cpp | 35 +++++++++++++++---- .../src/cpp/jtl/string_builder.cpp | 5 +++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/compiler+runtime/include/cpp/jtl/string_builder.hpp b/compiler+runtime/include/cpp/jtl/string_builder.hpp index d76874209..22767fd78 100644 --- a/compiler+runtime/include/cpp/jtl/string_builder.hpp +++ b/compiler+runtime/include/cpp/jtl/string_builder.hpp @@ -81,6 +81,7 @@ namespace jtl void reserve(usize capacity); value_type *data() const; usize size() const; + bool empty() const; jtl::immutable_string release(); std::string str() const; diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 986ee5b72..afaae7f37 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -923,11 +923,21 @@ namespace jank::codegen jtl::option processor::gen(analyze::expr::function_ref const expr, analyze::expr::function_arity const &) { - auto const compiling(truthy(__rt_ctx->compile_files_var->deref())); + auto const fn_target((target == compilation_target::eval) ? compilation_target::eval + : compilation_target::function); /* Since each codegen proc handles one callable struct, we create a new one for this fn. */ - processor prc{ expr, - module, - compiling ? compilation_target::function : compilation_target::eval }; + processor prc{ expr, module, fn_target }; + + /* TODO: Share a context instead. */ + prc.lifted_vars = lifted_vars; + prc.lifted_constants = lifted_constants; + + prc.build_body(); + + lifted_vars = jtl::move(prc.lifted_vars); + lifted_constants = jtl::move(prc.lifted_constants); + prc.lifted_vars.clear(); + prc.lifted_constants.clear(); util::format_to(deps_buffer, "{}", prc.declaration_str()); @@ -2035,7 +2045,12 @@ namespace jank::codegen void processor::build_header() { - util::format_to(header_buffer, "namespace {} {", runtime::module::module_to_native_ns(module)); + if(target != compilation_target::function) + { + util::format_to(header_buffer, + "namespace {} {", + runtime::module::module_to_native_ns(module)); + } util::format_to(header_buffer, R"( @@ -2163,6 +2178,11 @@ namespace jank::codegen void processor::build_body() { + if(!body_buffer.empty()) + { + return; + } + analyze::expr::function_arity const *variadic_arity{}; analyze::expr::function_arity const *highest_fixed_arity{}; for(auto const &arity : root_fn->arities) @@ -2265,7 +2285,10 @@ namespace jank::codegen util::format_to(footer_buffer, "};"); /* Namespace. */ - util::format_to(footer_buffer, "}"); + if(target != compilation_target::function) + { + util::format_to(footer_buffer, "}"); + } if(target == compilation_target::module) { diff --git a/compiler+runtime/src/cpp/jtl/string_builder.cpp b/compiler+runtime/src/cpp/jtl/string_builder.cpp index 3c683c534..dd9000f09 100644 --- a/compiler+runtime/src/cpp/jtl/string_builder.cpp +++ b/compiler+runtime/src/cpp/jtl/string_builder.cpp @@ -392,6 +392,11 @@ namespace jtl return pos; } + bool string_builder::empty() const + { + return pos == 0; + } + jtl::immutable_string string_builder::release() { jank_debug_assert(pos < capacity); From ceef6a88d8a9524165b84aea70378be4cdfac3ee Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 2 Dec 2025 21:11:46 -0800 Subject: [PATCH 55/58] Finish module-level cppgen optimizations --- .../include/cpp/jank/codegen/processor.hpp | 2 + .../src/cpp/jank/codegen/processor.cpp | 145 +++++++++++++----- 2 files changed, 110 insertions(+), 37 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index 559f46ed4..c24b3407b 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -164,6 +164,8 @@ namespace jank::codegen compilation_target target{}; runtime::obj::symbol struct_name; + jtl::string_builder module_header_buffer; + jtl::string_builder module_footer_buffer; jtl::string_builder deps_buffer; jtl::string_builder header_buffer; jtl::string_builder body_buffer; diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index afaae7f37..7bc71332a 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -48,10 +48,11 @@ * This is optimized by knowing what position every expression in, so trivial expressions used * as arguments, for example, don't need to be first stored in temporaries. * - * Lastly, this is complicated by tracking boxing requirements so that not everything is an - * `object_ref`. Judicious use of `auto` and semantic analysis alows us to track when unboxing - * is supported, although we very rarely know for certain if something is unboxed. We usually - * only know if it _could_ be. + * Code generation has a target, which is either for eval or for a module. + * When the target is for eval, each generated function is standalone. This + * is the normal operation. However, when doing AOT compilation, our target + * will be a module and we'll do some code size optimizations to group all + * of the functions within a module into one namespace, dedupe constants, etc. */ namespace jank::codegen @@ -928,16 +929,19 @@ namespace jank::codegen /* Since each codegen proc handles one callable struct, we create a new one for this fn. */ processor prc{ expr, module, fn_target }; - /* TODO: Share a context instead. */ - prc.lifted_vars = lifted_vars; - prc.lifted_constants = lifted_constants; + if(fn_target == compilation_target::function) + { + /* TODO: Share a context instead. */ + prc.lifted_vars = lifted_vars; + prc.lifted_constants = lifted_constants; - prc.build_body(); + prc.build_body(); - lifted_vars = jtl::move(prc.lifted_vars); - lifted_constants = jtl::move(prc.lifted_constants); - prc.lifted_vars.clear(); - prc.lifted_constants.clear(); + lifted_vars = jtl::move(prc.lifted_vars); + lifted_constants = jtl::move(prc.lifted_constants); + prc.lifted_vars.clear(); + prc.lifted_constants.clear(); + } util::format_to(deps_buffer, "{}", prc.declaration_str()); @@ -2024,18 +2028,42 @@ namespace jank::codegen { profile::timer const timer{ util::format("cpp gen {}", root_fn->name) }; + /* Module targeting works in a special way, with the goal of + * cutting down the generated code size. Instead of each function + * having its own lifted vars/constants, we have one namespace for + * the module with the lifted globals there, at namespace level. + * Then every function within that module can share the same globals. + * This also makes creating functions cheaper. However, it requires + * some special tracking. */ + if(target == compilation_target::module) + { + util::format_to(module_header_buffer, + "namespace {} {", + runtime::module::module_to_native_ns(module)); + } + + /* We generate the body first so that we know what we need for the header. This is * necessary since we end up lifting vars and constants while building the body. */ build_body(); build_header(); build_footer(); + + if(target == compilation_target::module) + { + /* Namespace. */ + util::format_to(module_footer_buffer, "}"); + } + generated_declaration = true; } native_transient_string ret; - ret.reserve(deps_buffer.size() + header_buffer.size() + body_buffer.size() - + footer_buffer.size()); + ret.reserve(module_header_buffer.size() + module_footer_buffer.size() + deps_buffer.size() + + header_buffer.size() + body_buffer.size() + footer_buffer.size()); + ret += jtl::immutable_string_view{ module_header_buffer.data(), module_header_buffer.size() }; ret += jtl::immutable_string_view{ deps_buffer.data(), deps_buffer.size() }; + ret += jtl::immutable_string_view{ module_footer_buffer.data(), module_footer_buffer.size() }; ret += jtl::immutable_string_view{ header_buffer.data(), header_buffer.size() }; ret += jtl::immutable_string_view{ body_buffer.data(), body_buffer.size() }; ret += jtl::immutable_string_view{ footer_buffer.data(), footer_buffer.size() }; @@ -2080,18 +2108,26 @@ namespace jank::codegen } } + auto &lifted_buffer{ (target == compilation_target::module) ? module_header_buffer + : header_buffer }; + auto const lifted_const{ (target == compilation_target::module) ? "" : "const" }; + for(auto const &v : lifted_vars) { - util::format_to(header_buffer, "jank::runtime::var_ref const {};", v.second.native_name); + util::format_to(lifted_buffer, + "jank::runtime::var_ref {} {};", + lifted_const, + v.second.native_name); } for(auto const &v : lifted_constants) { /* TODO: Typed lifted constants (in analysis). */ - util::format_to(header_buffer, - "{} const {};", + util::format_to(lifted_buffer, + "{} {} {};", detail::gen_constant_type(v.first, true), + lifted_const, v.second); } } @@ -2146,30 +2182,34 @@ namespace jank::codegen } } - for(auto const &v : lifted_vars) + if(target == compilation_target::eval) { - if(v.second.owned) + for(auto const &v : lifted_vars) { - util::format_to(header_buffer, - R"(, {}{ jank::runtime::__rt_ctx->intern_owned_var("{}").expect_ok() })", - v.second.native_name, - v.first); - } - else - { - util::format_to(header_buffer, - R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}").expect_ok() })", - v.second.native_name, - v.first); + if(v.second.owned) + { + util::format_to( + header_buffer, + R"(, {}{ jank::runtime::__rt_ctx->intern_owned_var("{}").expect_ok() })", + v.second.native_name, + v.first); + } + else + { + util::format_to(header_buffer, + R"(, {}{ jank::runtime::__rt_ctx->intern_var("{}").expect_ok() })", + v.second.native_name, + v.first); + } } - } - for(auto const &v : lifted_constants) - { - util::format_to(header_buffer, ", {}{", v.second); - detail::gen_constant(v.first, header_buffer, true); - util::format_to(header_buffer, "}"); + for(auto const &v : lifted_constants) + { + util::format_to(header_buffer, ", {}{", v.second); + detail::gen_constant(v.first, header_buffer, true); + util::format_to(header_buffer, "}"); + } } } @@ -2296,6 +2336,8 @@ namespace jank::codegen "void* {}(){", runtime::module::module_to_load_function(module)); + auto const ns{ runtime::module::module_to_native_ns(module) }; + /* First thing we do when loading this module is to intern our ns. Everything else will * build on that. */ util::format_to(footer_buffer, "jank_ns_intern_c(\"{}\");", module); @@ -2317,9 +2359,38 @@ namespace jank::codegen current_ns->name->get_name(), current_ns->symbol_counter.load()); + for(auto const &v : lifted_vars) + { + if(v.second.owned) + { + util::format_to( + footer_buffer, + R"({}::{} = jank::runtime::__rt_ctx->intern_owned_var("{}").expect_ok();)", + ns, + v.second.native_name, + v.first); + } + else + { + util::format_to(footer_buffer, + R"({}::{} = jank::runtime::__rt_ctx->intern_var("{}").expect_ok();)", + ns, + v.second.native_name, + v.first); + } + } + + + for(auto const &v : lifted_constants) + { + util::format_to(footer_buffer, "{}::{} = ", ns, v.second); + detail::gen_constant(v.first, footer_buffer, true); + util::format_to(footer_buffer, ";"); + } + util::format_to(footer_buffer, "return {}::{}{ }.call().erase();", - runtime::module::module_to_native_ns(module), + ns, runtime::munge(struct_name.name)); util::format_to(footer_buffer, "}"); From 9df0a956c99beaf2bc89bba5e9351e6c2d113a3e Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 2 Dec 2025 21:25:11 -0800 Subject: [PATCH 56/58] Further optimize cppgen --- .../include/cpp/jank/analyze/expr/function.hpp | 6 +++++- compiler+runtime/src/cpp/jank/analyze/processor.cpp | 6 ++++-- compiler+runtime/src/cpp/jank/codegen/processor.cpp | 12 ++++++------ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/analyze/expr/function.hpp b/compiler+runtime/include/cpp/jank/analyze/expr/function.hpp index 9dbe3087b..b6db7de04 100644 --- a/compiler+runtime/include/cpp/jank/analyze/expr/function.hpp +++ b/compiler+runtime/include/cpp/jank/analyze/expr/function.hpp @@ -27,7 +27,11 @@ namespace jank::analyze::expr jtl::immutable_string unique_name; usize param_count{}; bool is_variadic{}; - bool is_tail_recursive{}; + /* Is recur used within this function? */ + bool is_recur_recursive{}; + /* Is there any named recrusion within this function (tail or otherwise)? + * This counts any named recursion reference, not just calls. */ + bool is_named_recursive{}; /* TODO: is_pure */ }; diff --git a/compiler+runtime/src/cpp/jank/analyze/processor.cpp b/compiler+runtime/src/cpp/jank/analyze/processor.cpp index 6c73b33b0..f8d5fecaf 100644 --- a/compiler+runtime/src/cpp/jank/analyze/processor.cpp +++ b/compiler+runtime/src/cpp/jank/analyze/processor.cpp @@ -1574,6 +1574,8 @@ namespace jank::analyze auto &unwrapped_named_recursion(found_named_recursion.unwrap()); local_frame::register_captures(current_frame, unwrapped_named_recursion); + unwrapped_named_recursion.fn_frame->fn_ctx->is_named_recursive = true; + return jtl::make_ref( position, current_frame, @@ -1724,7 +1726,7 @@ namespace jank::analyze /* If it turns out this function uses recur, we need to ensure that its tail expression * is boxed. This is because unboxed values may use IIFE for initialization, which will * not work with the generated while/continue we use for recursion. */ - if(fn_ctx->is_tail_recursive) + if(fn_ctx->is_recur_recursive) { step::force_boxed(body_do); } @@ -2026,7 +2028,7 @@ namespace jank::analyze } else { - fn_ctx.unwrap()->is_tail_recursive = true; + fn_ctx.unwrap()->is_recur_recursive = true; } return jtl::make_ref(position, diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 7bc71332a..626765ca3 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -2238,7 +2238,7 @@ namespace jank::codegen } jtl::immutable_string recur_suffix; - if(arity.fn_ctx->is_tail_recursive) + if(arity.fn_ctx->is_recur_recursive) { recur_suffix = detail::recur_suffix; } @@ -2261,14 +2261,14 @@ namespace jank::codegen //util::format_to(body_buffer, "jank::profile::timer __timer{ \"{}\" };", root_fn->name); - if(!param_shadows_fn) + if(!param_shadows_fn && arity.fn_ctx->is_named_recursive) { util::format_to(body_buffer, "jank::runtime::object_ref const {}{ this };", runtime::munge(root_fn->name)); } - if(arity.fn_ctx->is_tail_recursive) + if(arity.fn_ctx->is_recur_recursive) { util::format_to(body_buffer, "{"); @@ -2295,7 +2295,7 @@ namespace jank::codegen util::format_to(body_buffer, "return jank::runtime::jank_nil;"); } - if(arity.fn_ctx->is_tail_recursive) + if(arity.fn_ctx->is_recur_recursive) { util::format_to(body_buffer, "} }"); } @@ -2311,8 +2311,8 @@ namespace jank::codegen util::format_to(body_buffer, R"( - jank::runtime::behavior::callable::arity_flag_t get_arity_flags() const final - { return jank::runtime::behavior::callable::build_arity_flags({}, true, {}); } + callable::arity_flag_t get_arity_flags() const final + { return callable::build_arity_flags({}, true, {}); } )", variadic_arity->fn_ctx->param_count - 1, variadic_ambiguous); From 2064e91bc660922ee477b28001bfb623b6f3d265 Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 2 Dec 2025 22:19:15 -0800 Subject: [PATCH 57/58] Properly gen all meta --- .../src/cpp/jank/codegen/processor.cpp | 141 +++++++++--------- 1 file changed, 70 insertions(+), 71 deletions(-) diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 626765ca3..027f91b23 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -55,6 +55,13 @@ * of the functions within a module into one namespace, dedupe constants, etc. */ +/* TODO: Size optimizations: + * - Remove extra object->object conversions + * - Add inlining back + * - Remove object requirement for if condition + * - Remove extra if_n = jank_nil on empty branches + */ + namespace jank::codegen { using namespace jank::analyze; @@ -67,6 +74,24 @@ namespace jank::codegen * the actual param names as mutable locals outside of the while loop. */ constexpr jtl::immutable_string_view const recur_suffix{ "__recur" }; + static jtl::immutable_string + lift_constant(native_unordered_map, + runtime::very_equal_to> &lifted_constants, + object_ref const &o) + { + auto const existing{ lifted_constants.find(o) }; + if(existing != lifted_constants.end()) + { + return existing->second; + } + + auto const &native_name{ runtime::munge(__rt_ctx->unique_string("const")) }; + lifted_constants.emplace(o, native_name); + return native_name; + } + static jtl::immutable_string gen_constant_type(runtime::object_ref const o, bool const boxed) { #pragma clang diagnostic push @@ -117,6 +142,11 @@ namespace jank::codegen #pragma clang diagnostic pop } + static bool should_gen_meta(jtl::option const &meta) + { + return meta.is_some() && !runtime::is_empty(meta.unwrap()); + } + static void gen_constant(runtime::object_ref const o, jtl::string_builder &buffer, bool const boxed) { @@ -254,24 +284,21 @@ namespace jank::codegen if(typed_o->data.empty()) { util::format_to(buffer, "jank::runtime::obj::persistent_vector::empty()"); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "->with_meta("); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ")"); } } else { util::format_to(buffer, "jank::runtime::make_box("); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\"), ", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ","); } util::format_to(buffer, "std::in_place "); for(auto const &form : typed_o->data) @@ -287,23 +314,21 @@ namespace jank::codegen if(typed_o->data.empty()) { util::format_to(buffer, "jank::runtime::obj::persistent_list::empty()"); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "->with_meta("); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ")"); } } else { util::format_to(buffer, "jank::runtime::make_box("); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\"), ", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ","); } util::format_to(buffer, "std::in_place "); for(auto const &form : typed_o->data) @@ -319,23 +344,21 @@ namespace jank::codegen if(typed_o->data.empty()) { util::format_to(buffer, "jank::runtime::obj::persistent_hash_set::empty()"); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "->with_meta("); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ")"); } } else { util::format_to(buffer, "jank::runtime::make_box("); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\"), ", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ","); } util::format_to(buffer, "std::in_place "); for(auto const &form : typed_o->data) @@ -351,25 +374,22 @@ namespace jank::codegen if(typed_o->data.empty()) { util::format_to(buffer, "jank::runtime::obj::persistent_array_map::empty()"); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "->with_meta("); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ")"); } } else { bool need_comma{}; - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { util::format_to( buffer, "jank::runtime::obj::persistent_array_map::create_unique_with_meta("); - util::format_to(buffer, - "jank::runtime::__rt_ctx->read_string(\"{}\")", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + gen_constant(typed_o->meta.unwrap(), buffer, true); need_comma = true; } else @@ -395,17 +415,16 @@ namespace jank::codegen if(typed_o->data.empty()) { util::format_to(buffer, "jank::runtime::obj::persistent_hash_map::empty()"); - if(typed_o->meta.is_some()) + if(should_gen_meta(typed_o->meta)) { - /* TODO: If meta is empty, use empty() fn. We'll need a gen helper for this. */ - util::format_to(buffer, - "->with_meta(jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, "->with_meta("); + gen_constant(typed_o->meta.unwrap(), buffer, true); + util::format_to(buffer, ")"); } } else { - auto const has_meta{ typed_o->meta.is_some() }; + auto const has_meta{ should_gen_meta(typed_o->meta) }; if(has_meta) { util::format_to(buffer, "jank::runtime::with_meta("); @@ -415,9 +434,8 @@ namespace jank::codegen util::escape(typed_o->to_code_string())); if(has_meta) { - util::format_to(buffer, - ", jank::runtime::__rt_ctx->read_string(\"{}\"))", - util::escape(runtime::to_code_string(typed_o->meta.unwrap()))); + util::format_to(buffer, ","); + gen_constant(typed_o->meta.unwrap(), buffer, true); } } } @@ -514,24 +532,6 @@ namespace jank::codegen return native_name; } - static jtl::immutable_string - lift_constant(native_unordered_map, - runtime::very_equal_to> &lifted_constants, - object_ref const &o) - { - auto const existing{ lifted_constants.find(o) }; - if(existing != lifted_constants.end()) - { - return existing->second; - } - - auto const &native_name{ runtime::munge(__rt_ctx->unique_string("const")) }; - lifted_constants.emplace(o, native_name); - return native_name; - } - jtl::option processor::gen(analyze::expr::def_ref const expr, analyze::expr::function_arity const &fn_arity) { @@ -550,7 +550,7 @@ namespace jank::codegen jtl::option meta; if(expr->name->meta.is_some()) { - meta = lift_constant(lifted_constants, expr->name->meta.unwrap()); + meta = detail::lift_constant(lifted_constants, expr->name->meta.unwrap()); } /* Forward declarations just intern the var and evaluate to it. */ @@ -703,7 +703,7 @@ namespace jank::codegen } else { - ret = lift_constant(lifted_constants, expr->data); + ret = detail::lift_constant(lifted_constants, expr->data); } switch(expr->position) @@ -1930,18 +1930,17 @@ namespace jank::codegen auto ret_tmp{ runtime::munge(__rt_ctx->unique_string("cpp_unbox")) }; auto value_tmp{ gen(expr->value_expr, arity) }; auto const type_name{ cpp_util::get_qualified_type_name(expr->type) }; - auto const meta{ runtime::source_to_meta(expr->source) }; + auto const meta{ detail::lift_constant(lifted_constants, + runtime::source_to_meta(expr->source)) }; util::format_to(body_buffer, "auto {}{ " - "static_cast<{}>(jank_unbox_with_source(\"{}\", {}.data, " - "jank::runtime::__rt_ctx->read_string(\"{}\").data)" - ") };", + "static_cast<{}>(jank_unbox_with_source(\"{}\", {}.data, {}.data)) };", ret_tmp, type_name, type_name, value_tmp.unwrap().str(false), - util::escape(runtime::to_code_string(meta))); + meta); if(expr->position == expression_position::tail) { From 15fc082db6736c4dd6389cd4c530aca2baaa1e8b Mon Sep 17 00:00:00 2001 From: jeaye Date: Tue, 2 Dec 2025 22:32:49 -0800 Subject: [PATCH 58/58] Optimize cppgen size more --- .../include/cpp/jank/codegen/processor.hpp | 1 + compiler+runtime/src/cpp/jank/codegen/processor.cpp | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/compiler+runtime/include/cpp/jank/codegen/processor.hpp b/compiler+runtime/include/cpp/jank/codegen/processor.hpp index c24b3407b..d7dd037a6 100644 --- a/compiler+runtime/include/cpp/jank/codegen/processor.hpp +++ b/compiler+runtime/include/cpp/jank/codegen/processor.hpp @@ -164,6 +164,7 @@ namespace jank::codegen compilation_target target{}; runtime::obj::symbol struct_name; + jtl::string_builder cpp_raw_buffer; jtl::string_builder module_header_buffer; jtl::string_builder module_footer_buffer; jtl::string_builder deps_buffer; diff --git a/compiler+runtime/src/cpp/jank/codegen/processor.cpp b/compiler+runtime/src/cpp/jank/codegen/processor.cpp index 027f91b23..8b9ad28a7 100644 --- a/compiler+runtime/src/cpp/jank/codegen/processor.cpp +++ b/compiler+runtime/src/cpp/jank/codegen/processor.cpp @@ -57,6 +57,7 @@ /* TODO: Size optimizations: * - Remove extra object->object conversions + * - Typed object to object * - Add inlining back * - Remove object requirement for if condition * - Remove extra if_n = jank_nil on empty branches @@ -500,8 +501,8 @@ namespace jank::codegen compilation_target const target) : root_fn{ expr } , module{ module } - , target{ target } - , struct_name{ root_fn->unique_name } + , target{ target } /* The normal unique name is fully namespaced, which we don't need. */ + , struct_name{ runtime::__rt_ctx->unique_string(root_fn->name) } { assert(root_fn->frame.data); } @@ -1466,7 +1467,7 @@ namespace jank::codegen jtl::option processor::gen(expr::cpp_raw_ref const expr, expr::function_arity const &) { - util::format_to(deps_buffer, "\n{}\n", expr->code); + util::format_to(cpp_raw_buffer, "\n{}\n", expr->code); if(expr->position == analyze::expression_position::tail) { @@ -2058,8 +2059,10 @@ namespace jank::codegen } native_transient_string ret; - ret.reserve(module_header_buffer.size() + module_footer_buffer.size() + deps_buffer.size() - + header_buffer.size() + body_buffer.size() + footer_buffer.size()); + ret.reserve(cpp_raw_buffer.size() + module_header_buffer.size() + module_footer_buffer.size() + + deps_buffer.size() + header_buffer.size() + body_buffer.size() + + footer_buffer.size()); + ret += jtl::immutable_string_view{ cpp_raw_buffer.data(), cpp_raw_buffer.size() }; ret += jtl::immutable_string_view{ module_header_buffer.data(), module_header_buffer.size() }; ret += jtl::immutable_string_view{ deps_buffer.data(), deps_buffer.size() }; ret += jtl::immutable_string_view{ module_footer_buffer.data(), module_footer_buffer.size() };