C++ gen (WIP)

.clang-tidy

@@ -47,6 +47,8 @@ misc-definitions-in-headers,\
 -bugprone-macro-parentheses,\
 -bugprone-throwing-static-initialization,\
 -readability-redundant-member-init,\
+-readability-redundant-typename,\
+-readability-redundant-casting,\
 -readability-implicit-bool-conversion,\
 -readability-magic-numbers,\
 -readability-identifier-length,\

.github/workflows/build.yml

@@ -97,10 +97,11 @@ jobs:
             analyze: on
 
           ## Debug + sanitization
-          - name: Ubuntu - address sanitizer
-            os: ubuntu-24.04
-            build_type: Debug
-            sanitize: address
+          # TODO: Fix this. GC issue: https://github.com/bdwgc/bdwgc/issues/772
+          #- name: Ubuntu - address sanitizer
+          #  os: ubuntu-24.04
+          #  build_type: Debug
+          #  sanitize: address
 
           - name: Ubuntu - undefined behavior sanitizer
             os: ubuntu-24.04
@@ -203,6 +204,12 @@ jobs:
           path: |
             ${{ github.workspace }}/compiler+runtime/.ctcache
           key: ctcache-${{ env.JANK_MATRIX_ID }}-${{ github.ref_name }}
+      - uses: thejerrybao/setup-swap-space@466d59798fb9263b9c2331c83cf71f9b7bfd0c78
+        if: runner.os == 'Linux'
+        with:
+          swap-space-path: /swapfile
+          swap-size-gb: 8
+          remove-existing-swap-files: true
       - name: Build and test jank
         id: jank-build-step
         run: |
@@ -272,26 +279,37 @@ jobs:
         run: |
           scp -i ~/.ssh/jank_id_ed25519 -o StrictHostKeyChecking=no ~/*.tar.gz root@cache.jank-lang.org:/var/www/cache.jank-lang.org/html/
 
-  build-nix:
-    name: "Nix - release"
-    runs-on: ubuntu-latest
-    steps:
-    - name: "Set up nix"
-      uses: cachix/install-nix-action@v31
-      with:
-        nix_path: nixpkgs=channel:nixos-unstable
-    - name: "Set up cachix"
-      uses: cachix/cachix-action@v16
-      with:
-        name: jank-lang
-        # If you chose API tokens for write access OR if you have a private cache
-        authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
-        # Only push to cache on main branch, not PRs
-        skipPush: "${{ github.ref != 'refs/heads/main' }}"
-    - uses: actions/checkout@v4
-      with:
-        submodules: 'recursive'
-    - name: "Build jank"
-      run: nix build .?submodules=1 -L
-    - name: "Check health"
-      run: nix run .?submodules=1 check-health
+  # TODO: Fix Nix in CI!
+  # We're currently running out of memory in CI, due to Clang's memory usage for
+  # JIT compiled C++ code. Nix, unsurprisingly, uses more RAM than others, so it's
+  # blocked while others are merely hindered. We should be able to fix this by
+  # optimizing Clang's memory usage for JIT compiled code. See this gist for
+  # much more details: https://gist.github.com/jeaye/9adaaa56aee50d5912207907a48d1006
+  #
+  # It's also possible that we can just use IR gen for Nix. That would require
+  # some additional CMake support, but is definitely doable. For the alpha, since
+  # we don't have a finished Nix package anyway, I'm not too concerned about this.
+  #
+  #build-nix:
+  #  name: "Nix - release"
+  #  runs-on: ubuntu-latest
+  #  steps:
+  #  - name: "Set up nix"
+  #    uses: cachix/install-nix-action@v31
+  #    with:
+  #      nix_path: nixpkgs=channel:nixos-unstable
+  #  - name: "Set up cachix"
+  #    uses: cachix/cachix-action@v16
+  #    with:
+  #      name: jank-lang
+  #      # If you chose API tokens for write access OR if you have a private cache
+  #      authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+  #      # Only push to cache on main branch, not PRs
+  #      skipPush: "${{ github.ref != 'refs/heads/main' }}"
+  #  - uses: actions/checkout@v4
+  #    with:
+  #      submodules: 'recursive'
+  #  - name: "Build jank"
+  #    run: nix build .?submodules=1 -L
+  #  - name: "Check health"
+  #    run: nix run .?submodules=1 check-health

.gitignore

@@ -6,7 +6,6 @@ a.out
 .jank-repl-history
 .envrc
 .direnv
-/notes
 
 # Vim files
 /.ycm_extra_conf.py*

.gitmodules

@@ -1,15 +1,9 @@
 [submodule "compiler+runtime/third-party/folly"]
 	path = compiler+runtime/third-party/folly
 	url = https://github.com/jank-lang/folly.git
-[submodule "compiler+runtime/third-party/bpptree"]
-	path = compiler+runtime/third-party/bpptree
-	url = https://github.com/jank-lang/BppTree.git
 [submodule "compiler+runtime/third-party/immer"]
 	path = compiler+runtime/third-party/immer
 	url = https://github.com/jank-lang/immer.git
-[submodule "compiler+runtime/third-party/cli11"]
-	path = compiler+runtime/third-party/cli11
-	url = https://github.com/jank-lang/CLI11.git
 [submodule "compiler+runtime/third-party/boost-preprocessor"]
 	path = compiler+runtime/third-party/boost-preprocessor
 	url = https://github.com/jank-lang/boost-preprocessor.git

compiler+runtime/CMakeLists.txt

@@ -42,12 +42,14 @@ option(jank_coverage "Enable code coverage measurement" OFF)
 option(jank_analyze "Enable static analysis" OFF)
 option(jank_test "Enable jank's test suite" OFF)
 option(jank_unity_build "Optimize translation unit compilation for the number of cores" OFF)
+option(jank_debug_gc "Enable GC debug assertions" OFF)
+option(jank_profile_gc "Enable GC profiling (via massif or heaptrack)" OFF)
+option(jank_force_phase_2 "Force the linking of core libs into the jank binary" OFF)
 set(jank_sanitize "none" CACHE STRING "The type of Clang sanitization to use (or none)")
 set(jank_resource_dir
   "../lib/jank/${CMAKE_PROJECT_VERSION}"
   CACHE STRING
   "Where jank's runtime files are installed. Relative paths are based on the runtime jank binary path")
-set(jank_clojure_core_o "${CMAKE_BINARY_DIR}/core-libs/clojure/core.o")
 
 find_package(Git REQUIRED)
 execute_process(
@@ -81,7 +83,7 @@ set(
   jank_aot_compiler_flags
   -Wall -Wextra -Wpedantic
   -Wfloat-equal -Wuninitialized -Wswitch-enum -Wnon-virtual-dtor
-  -Wold-style-cast -Wno-gnu-case-range -Wno-c99-designator
+  -Wold-style-cast -Wno-gnu-case-range -Wno-c99-designator -Wno-c2y-extensions
   -Wno-gnu-conditional-omitted-operand
   -Wno-implicit-fallthrough
   -Wno-covered-switch-default
@@ -99,6 +101,8 @@ set(
 set(
   jank_common_compiler_flags
   -std=gnu++20
+  # https://groups.google.com/g/llvm-dev/c/W-OweiAjDcU?pli=1
+  -femulated-tls
   -DIMMER_HAS_LIBGC=1 -DIMMER_TAGGED_NODE=0 -DHAVE_CXX14=1
   -DCPPINTEROP_USE_REPL
   -DFOLLY_HAVE_JEMALLOC=0 -DFOLLY_HAVE_TCMALLOC=0 -DFOLLY_ASSUME_NO_JEMALLOC=1 -DFOLLY_ASSUME_NO_TCMALLOC=1
@@ -132,8 +136,18 @@ elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
   list(APPEND jank_common_compiler_flags -Og -g -DNDEBUG)
 endif()
 
-if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug" OR jank_force_phase_2)
   set(jank_enable_phase_2 ON)
+else()
+  set(jank_enable_phase_2 OFF)
+endif()
+
+if(jank_enable_phase_2)
+  set(jank_clojure_core_output "${CMAKE_BINARY_DIR}/core-libs/clojure/core.cpp")
+  set(jank_clojure_core_output_artifact "${CMAKE_BINARY_DIR}/CMakeFiles/jank_exe_phase_2.dir/core-libs/clojure/core.cpp.o")
+else()
+  set(jank_clojure_core_output "${CMAKE_BINARY_DIR}/core-libs/clojure/core.o")
+  set(jank_clojure_core_output_artifact "${jank_clojure_core_output}")
 endif()
 
 if(jank_test)
@@ -349,6 +363,17 @@ set(jank_lib_standalone_deps
   folly_lib
   clangCppInterOp
 )
+
+# jank compiles in two phases. The first phase builds the compiler+runtime into a binary
+# called jank-phase-1. We then we this compiler to build the core libs (like clojure.core)
+# into cpp files. We then compile those and link them into a new binary which is just
+# called jank. This bakes in the core libraries so they're very fast to load, but of
+# course it means that updating a core library means updating the whole jank binary.
+#
+# For debug builds, we don't bother building the second phase and we just create a symlink
+# called jank which points at jank-phase-1. This makes iteration a little faster, at the
+# cost of jank's startup being a little slower, since it's loading the core libs via their
+# object files, rather than having them baked in.
 add_custom_command(
   DEPENDS ${jank_lib_standalone_deps}
   OUTPUT ${CMAKE_BINARY_DIR}/libjank-standalone-phase-1.a
@@ -363,8 +388,7 @@ add_custom_target(
 )
 
 add_custom_command(
-  DEPENDS ${jank_lib_standalone_deps}
-          jank_core_libraries
+  DEPENDS ${jank_lib_standalone_deps} jank_exe_phase_2
   OUTPUT ${CMAKE_BINARY_DIR}/libjank-standalone.a
   COMMAND_EXPAND_LISTS
   VERBATIM
@@ -569,10 +593,8 @@ target_include_directories(
   PUBLIC
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/nanobench/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/folly>"
-  "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/bpptree/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/bdwgc/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/immer>"
-  "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/cli11/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/ftxui/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/zip/src>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/cpptrace/include>"
@@ -753,11 +775,16 @@ if(jank_enable_phase_2)
   add_executable(
     jank_exe_phase_2
     "$<TARGET_PROPERTY:jank_exe_phase_1,SOURCES>"
+    ${jank_clojure_core_output}
   )
   add_executable(jank::exe ALIAS jank_exe_phase_2)
-
   set_property(TARGET jank_exe_phase_2 PROPERTY OUTPUT_NAME jank)
 
+  # Our generated code is not as precisely written as hand-written C++, so we ignore various
+  # warnings and skip linting.
+  set_source_files_properties(${jank_clojure_core_output} PROPERTIES COMPILE_FLAGS "-w -Wno-c++11-narrowing")
+  set_source_files_properties(${jank_clojure_core_output} PROPERTIES SKIP_LINTING ON)
+
   # Symbol exporting for JIT.
   set_target_properties(jank_exe_phase_2 PROPERTIES ENABLE_EXPORTS 1)
 
@@ -770,17 +797,16 @@ if(jank_enable_phase_2)
   target_link_libraries(
     jank_exe_phase_2 PUBLIC
     "$<TARGET_PROPERTY:jank_exe_phase_1,LINK_LIBRARIES>"
-    ${jank_clojure_core_o}
   )
 
   jank_hook_llvm(jank_exe_phase_2)
 
-  add_dependencies(jank_exe_phase_2 jank_lib_standalone_phase_2 jank_core_libraries)
+  add_dependencies(jank_exe_phase_2 jank_core_libraries)
 
   #set_target_properties(jank_exe_phase_2 PROPERTIES LINK_FLAGS_RELEASE "-s")
 else()
   add_custom_command(
-    DEPENDS ${CMAKE_BINARY_DIR}/jank-phase-1 ${jank_clojure_core_o} ${jank_incremental_pch_flag}
+    DEPENDS ${CMAKE_BINARY_DIR}/jank-phase-1 ${jank_clojure_core_output} ${jank_incremental_pch_flag}
     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
     OUTPUT ${CMAKE_BINARY_DIR}/jank
     COMMAND ln -sf jank-phase-1 jank
@@ -825,6 +851,11 @@ if(jank_test)
   add_executable(jank::test_exe ALIAS jank_test_exe)
   add_dependencies(jank_test_exe jank_exe_phase_1 jank_core_libraries)
 
+  if(jank_enable_phase_2)
+    target_sources(jank_test_exe PUBLIC ${jank_clojure_core_output})
+    target_compile_options(jank_test_exe PUBLIC -DJANK_PHASE_2)
+  endif()
+
   set_property(TARGET jank_test_exe PROPERTY OUTPUT_NAME jank-test)
 
   target_compile_features(jank_test_exe PRIVATE ${jank_cxx_standard})
@@ -904,8 +935,8 @@ set(jank_core_libraries_flag "${CMAKE_BINARY_DIR}/classes/core-libraries")
 add_custom_command(
   DEPENDS ${CMAKE_BINARY_DIR}/jank-phase-1 ${CMAKE_SOURCE_DIR}/src/jank/clojure/core.jank ${jank_incremental_pch_flag}
   OUTPUT ${jank_core_libraries_flag}
-  BYPRODUCTS ${jank_clojure_core_o}
-  COMMAND ${CMAKE_BINARY_DIR}/jank-phase-1 compile-module -o ${jank_clojure_core_o} clojure.core
+  BYPRODUCTS ${jank_clojure_core_output}
+  COMMAND ${CMAKE_BINARY_DIR}/jank-phase-1 -O3 compile-module -o ${jank_clojure_core_output} clojure.core
   COMMAND touch ${jank_core_libraries_flag}
 )
 add_custom_target(

compiler+runtime/bin/ar-merge

@@ -32,7 +32,7 @@ case "${command}" in
   # This follows merge, but also bundles in the Clojure core library object files, following
   # jank's phase 2 building.
   merge-phase-2)
-    object_files="$(cat "${cache_dir}"/*) ${cmake_binary_dir}/core-libs/clojure/core.o"
+    object_files="$(cat "${cache_dir}"/*) @jank_clojure_core_output_artifact@"
     merge_output="${cmake_binary_dir}/libjank-standalone.a"
     rm -f "${merge_output}"
     # shellcheck disable=SC2086

compiler+runtime/bin/build-clang

@@ -52,6 +52,7 @@ srcdir="${PWD}"
 llvm_url="https://github.com/jank-lang/llvm-project.git"
 llvm_version=22
 llvm_branch="jank-snapshot/llvm${llvm_version}"
+# 4e5928689f23..a8f19259e708  jank-snapshot/llvm22 -> jank-snapshot/llvm22
 
 function prepare()
 {

compiler+runtime/bin/clean

@@ -13,4 +13,5 @@ pushd "${here}/../build"
     make clean
   fi
   rm -rf ar-cache
+  rm -rf core-libs
 popd

compiler+runtime/bin/jank/compiler+runtime/build+test.clj

@@ -31,7 +31,10 @@
                            (str "-DCMAKE_BUILD_TYPE=" build-type)
                            (str "-Djank_analyze=" analyze)
                            (str "-Djank_sanitize=" sanitize)
-                           (str "-Djank_coverage=" coverage)]
+                           (str "-Djank_coverage=" coverage)
+                           ; We force phase 2 building to use less memory in CI by
+                           ; compiling to cpp files instead of object files.
+                           "-Djank_force_phase_2=on"]
           configure-flags (cond-> configure-flags
                             (not= "on" analyze)
                             (conj "-DCMAKE_C_COMPILER_LAUNCHER=ccache"
@@ -55,7 +58,16 @@
       (util/with-elapsed-time duration
         (util/quiet-shell {:dir compiler+runtime-dir
                            :extra-env exports}
-                          "./bin/compile -v")
+                          "./bin/clean")
+        (util/log-info-with-time duration "Cleaned"))
+
+      (util/with-elapsed-time duration
+        (util/quiet-shell {:dir compiler+runtime-dir
+                           :extra-env exports}
+                          (str "./bin/compile -v"
+                               ; This uses more memory in CI, so we limit parallelism.
+                               (when (= "Release" build-type)
+                                 " -j1")))
         (util/log-info-with-time duration "Compiled"))
 
       (util/quiet-shell {:dir compiler+runtime-dir

compiler+runtime/cmake/dependency/bdwgc.cmake

@@ -4,18 +4,50 @@ set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS})
 set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY})
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
-  set(BUILD_SHARED_LIBS OFF)
+
   set(CMAKE_CXX_CLANG_TIDY "")
+  set(GC_BUILD_SHARED_LIBS OFF)
+
+  # Malloc redirection is causing crashes on macOS. However, without it, we get
+  # premature collections. Because of this, collection is disabled on macOS entirely.
+  #
+  # https://github.com/bdwgc/bdwgc/issues/829
+  if(NOT APPLE)
+    set(enable_redirect_malloc ON CACHE BOOL "Redirect malloc and friends to collector routines")
+    set(enable_uncollectable_redirection ON CACHE BOOL "Redirect to uncollectible malloc instead of garbage-collected one")
+  endif()
+
   set(enable_cplusplus ON CACHE BOOL "Enable C++")
   set(build_cord OFF CACHE BOOL "Build cord")
   set(enable_docs OFF CACHE BOOL "Enable docs")
+  set(enable_threads ON CACHE BOOL "Enable multi-threading support")
   set(enable_large_config ON CACHE BOOL "Optimize for large heap or root set")
   set(enable_throw_bad_alloc_library ON CACHE BOOL "Enable C++ gctba library build")
+  set(enable_gc_debug OFF CACHE BOOL "Support for pointer back-tracing")
+
+  if(jank_profile_gc)
+    set(enable_valgrind_tracking ON CACHE BOOL "Support tracking GC_malloc and friends for heap profiling tools")
+  endif()
+
+  if(jank_debug_gc)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DGC_ASSERTIONS")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGC_ASSERTIONS")
+    set(enable_gc_debug ON CACHE BOOL "Support for pointer back-tracing")
+  endif()
+
   add_subdirectory(third-party/bdwgc EXCLUDE_FROM_ALL)
 
+  unset(GC_BUILD_SHARED_LIBS)
+  unset(enable_redirect_malloc)
+  unset(enable_uncollectable_redirection)
   unset(enable_cplusplus)
   unset(build_cord)
   unset(enable_docs)
+  unset(enable_threads)
+  unset(enable_large_config)
+  unset(enable_throw_bad_alloc_library)
+  unset(enable_valgrind_tracking)
+  unset(enable_gc_debug)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_OLD}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_OLD}")
 set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_OLD})