Feature/namespaces v3

SOUL.md

@@ -1,10 +1,15 @@
-We're building a clojure JIT compiler in LLVM/C++ API. We use reference counting with consuming semantics and NaN boxing.
+We're building a clojure JIT compiler in LLVM/C++ API. 
 
-There's two components here - one is the frontend - a clojure program that creates AST, computes memory guidance (dropMemory for happy path and unwindMemory for landing pads)
-and builds the binary proto.
+There's two components here - one is the frontend - a clojure program that creates AST, computes memory guidance (dropMemory for happy path and unwindMemory for landing pads) and builds the binary proto.
+
+
+In the backend we use reference counting with consuming semantics and NaN boxing.
+
+The entire memory architecture relies on consuming semantics. The only exceptions are runtime functions explicitly annotated with 'outside ref count', which operate on borrowing semantics. There is also a special case regarding the reference cycle between a namespace and a var—since a var is designed to hold a weak reference to its namespace, an additional release is required during its construction to properly satisfy the consuming semantics.
 
 The second - the backend which reads proto and compiles/executes.
 
 Our JIT currently compiles the proto forms using LLVMs ORC2 and links them, on IR level with runtime written in C - which allows for huge inlining optimisations.
 
-When working with tests - always stop and ask for permission if you want to change non-test code.
+When working with tests - always stop and ask for permission if you want to change non-test code.
+

backend/codegen/CodeGen.cpp

@@ -696,12 +696,7 @@ ObjectTypeSet CodeGen::getType(const Node &node,
 }
 
 Var *CodeGen::getOrCreateVar(std::string_view name) {
-  return compilerState.varRegistry.getOrCreate(
-      std::string(name).c_str(), [name]() {
-        RTValue keyword =
-            Keyword_create(String_createDynamicStr(std::string(name).c_str()));
-        return Var_create(keyword);
-      });
+  return compilerState.getOrCreateVar(std::string(name).c_str());
 }
 
 bool CodeGen::canThrow(const Node &node) {

backend/codegen/ops/ConstNode.cpp

@@ -93,7 +93,7 @@ TypedValue CodeGen::codegen(const Node &node, const ConstNode &subnode,
     break;
   case varType: {
     const std::string name = subnode.val();
-    ScopedRef<Var> var(compilerState.varRegistry.getCurrent(name.c_str()));
+    ScopedRef<Var> var(compilerState.getCurrentVar(name.c_str()));
     if (!var) {
       throwCodeGenerationException(
           string("Unable to resolve var: ") + name + " in this context", node);

backend/codegen/ops/TheVarNode.cpp

@@ -10,7 +10,7 @@ namespace rt {
 TypedValue CodeGen::codegen(const Node &node, const TheVarNode &subnode,
                             const ObjectTypeSet &typeRestrictions) {
   auto varName = subnode.var().substr(2);
-  ScopedRef<Var> var(compilerState.varRegistry.getCurrent(varName.c_str()));
+  ScopedRef<Var> var(compilerState.getOrCreateVar(varName.c_str()));
   if (!var)
     throwCodeGenerationException(
         "Unable to resolve var: " + varName + " in this context", node);

backend/codegen/ops/VarNode.cpp

@@ -10,7 +10,7 @@ namespace rt {
 TypedValue CodeGen::codegen(const Node &node, const VarNode &subnode,
                             const ObjectTypeSet &typeRestrictions) {
   auto varName = subnode.var().substr(2);
-  ScopedRef<Var> var(compilerState.varRegistry.getCurrent(varName.c_str()));
+  ScopedRef<Var> var(compilerState.getCurrentVar(varName.c_str()));
 
   if (!var)
     throwCodeGenerationException(

backend/jit/JITEngine.cpp

@@ -76,6 +76,7 @@ JITEngine::JITEngine(llvm::OptimizationLevel defaultOptLevel,
   // Initialise runtime.
   RuntimeInterface_initialise();
   Ebr_enter_critical();
+  threadsafeState.initializeDefaultNamespaces();
 
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();

backend/main.cpp

@@ -27,7 +27,11 @@
 #include "jit/JITEngine.h"
 #include "runtime/Ebr.h"
 #include "runtime/ExecutionContext.h"
+#include "runtime/Keyword.h"
+#include "runtime/Namespace.h"
+#include "runtime/RTValue.h"
 #include "runtime/String.h"
+#include "runtime/Symbol.h"
 #include "runtime/Var.h"
 #include "tools/RTValueWrapper.h"
 #include <chrono>
@@ -130,47 +134,15 @@ int main(int argc, char *argv[]) {
     rt::ScopedRef<ExecutionContext> ctx(
         ExecutionContext_create(RT_boxPtr(PersistentArrayMap_empty())));
 
-    // Setup default Clojure namespaces and *ns* Var
-    {
-      // 1. Create "clojure.core" namespace
-      Symbol *coreSym = Symbol_create(String_create("clojure.core"));
-      Namespace *coreNs = Namespace_findOrCreate(coreSym); // coreNs has +1 refcount
-
-      // 2. Intern "*ns*" Var in "clojure.core"
-      Symbol *nsSym = Symbol_create(String_create("*ns*"));
-      Ptr_retain(coreNs); // Retain coreNs because Namespace_intern consumes self
-      Var *nsVar = Namespace_intern(coreNs, nsSym); // nsVar has +1 refcount. nsSym is consumed.
-      Var_setDynamic(nsVar, true);
-
-      // 3. Create "user" namespace
-      Symbol *userSym = Symbol_create(String_create("user"));
-      Namespace *userNs = Namespace_findOrCreate(userSym); // userNs has +1 refcount
-      RTValue userNsVal = RT_boxPtr(userNs); // userNsVal has +1 refcount
-      retain(userNsVal); // userNsVal has +2 refcount
-
-      // 4. Bind clojure.core/*ns* Var root to the user namespace object
-      Ptr_retain(nsVar); // Retain nsVar because Var_bindRoot consumes self
-      Var_bindRoot(nsVar, userNsVal); // Consumes userNsVal (+2 -> +1) and nsVar
-
-      // 5. Build clojure.core/*ns* keyword for ExecutionContext dynamic bindings map
-      RTValue nsVarKw = Keyword_create(String_create("clojure.core/*ns*"));
-      retain(nsVarKw); // nsVarKw has +2 refcount
-
-      // Assoc clojure.core/*ns* keyword with userNsVal in ExecutionContext bindings map
-      ctx->bindingsMap = RT_boxPtr(PersistentArrayMap_assoc(
-          (PersistentArrayMap *)RT_unboxPtr(ctx->bindingsMap), nsVarKw,
-          userNsVal)); // assoc consumes nsVarKw (+2 -> +1) and userNsVal (+1 -> +0)
-
-      // 6. Register in compiler state so JIT finds this exact Var object
-      engine.threadsafeState.varRegistry.registerObject("clojure.core/*ns*",
-                                                        nsVar); // Registry takes ownership of nsVar (+1)
-
-      // Release remaining local reference to clojure.core namespace object
-      Ptr_release(coreNs);
-
-      // Release remaining keyword reference
-      release(nsVarKw);
-    }
+    Var *nsVar = engine.threadsafeState.getCurrentVar("clojure.core/*ns*");
+    assert(nsVar != nullptr && "clojure.core/*ns* Var not found");
+
+    // Assoc clojure.core/*ns* Var pointer with user namespace in ExecutionContext
+    // bindings map (as we now perform dynamic bindings lookup by pointer)
+    ctx->bindingsMap = RT_boxPtr(PersistentArrayMap_assoc(
+        (PersistentArrayMap *)RT_unboxPtr(ctx->bindingsMap),
+        RT_boxPtr(nsVar),
+        RT_boxPtr(Namespace_findOrCreate(Symbol_create(String_create("user"))))));
 
     {
       ExecutionTimer t("Compiling and storing interfaces");

backend/runtime/ExecutionContext.c

@@ -49,7 +49,7 @@ RT_bind(__attribute__((swift_context)) struct ExecutionContext *ctx, RTValue v,
   }
 
   RTValue nextMap =
-      RT_boxPtr(PersistentArrayMap_assoc(currentMap, var->keyword, val));
+      RT_boxPtr(PersistentArrayMap_assoc(currentMap, v, val));
 
   ExecutionContext *nextCtx = ExecutionContext_create(nextMap);
   return nextCtx;
@@ -92,9 +92,9 @@ RT_bind_map(__attribute__((swift_context)) struct ExecutionContext *ctx,
     }
 
     // assoc consumes its arguments, so we must retain them from the source map
-    retain(var->keyword);
+    retain(v);
     retain(val);
-    currentMap = PersistentArrayMap_assoc(currentMap, var->keyword, val);
+    currentMap = PersistentArrayMap_assoc(currentMap, v, val);
   }
 
   release(bindingsMap);