From 82eba147f2363147cf373a456d79656d1445cae0 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 30 Jun 2026 03:44:17 -0700 Subject: [PATCH 01/26] cse: per-offloaded-task CSE in parallel codegen workers + disable LICM cache Run CSE per offloaded task, but only once each task is isolated in the codegen worker pool: the post-offload full_simplify passes (offload_to_executable's before_lower_access / simplify_IV / scalarize) execute inside compile_task, which is enqueued per task to the worker pool, so per_task_cse fires there (one task per worker, inside the simplify fixpoint = full quality, in parallel). CSE is skipped on the pre-split monolith (the block still holds every task) and deferred to those parallel per-task passes. Because tasks are optimized independently, doing all CSE in the workers is value-identical to the serial variant -- just parallel. cache_loop_invariant_global_vars defaulted off for the same soundness reason as the serial variant. --- quadrants/ir/transforms.h | 1 + quadrants/program/compile_config.h | 5 ++- quadrants/transforms/simplify.cpp | 4 +- quadrants/transforms/whole_kernel_cse.cpp | 52 +++++++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/quadrants/ir/transforms.h b/quadrants/ir/transforms.h index 30b13b2105..0e9ae19737 100644 --- a/quadrants/ir/transforms.h +++ b/quadrants/ir/transforms.h @@ -46,6 +46,7 @@ bool alg_simp(IRNode *root, const CompileConfig &config); bool demote_operations(IRNode *root, const CompileConfig &config); bool binary_op_simplify(IRNode *root, const CompileConfig &config); bool whole_kernel_cse(IRNode *root); +bool per_task_cse(IRNode *root); bool extract_constant(IRNode *root, const CompileConfig &config); bool unreachable_code_elimination(IRNode *root); bool loop_invariant_code_motion(IRNode *root, const CompileConfig &config); diff --git a/quadrants/program/compile_config.h b/quadrants/program/compile_config.h index 92c09a5fd8..151cc0194b 100644 --- a/quadrants/program/compile_config.h +++ b/quadrants/program/compile_config.h @@ -25,7 +25,10 @@ struct CompileConfig { bool lower_access; bool simplify_after_lower_access; bool move_loop_invariant_outside_if; - bool cache_loop_invariant_global_vars{true}; + // Per-offloaded-task CSE no longer merges read/write GlobalPtrStmts to the same address before offload, which the + // post-offload cache_loop_invariant_global_vars pass relies on (it keys its cache by GlobalPtrStmt* identity). + // Defaulted off on this branch to keep that pass sound; per-task CSE already removes the redundant loads it targeted. + bool cache_loop_invariant_global_vars{false}; bool demote_dense_struct_fors; bool advanced_optimization; bool constant_folding; diff --git a/quadrants/transforms/simplify.cpp b/quadrants/transforms/simplify.cpp index 60f3953dd3..28028b855b 100644 --- a/quadrants/transforms/simplify.cpp +++ b/quadrants/transforms/simplify.cpp @@ -570,10 +570,10 @@ void full_simplify(IRNode *root, const CompileConfig &config, const FullSimplify modified = true; if (should_dump) dump_step("10_die", iteration); - if (config.opt_level > 0 && whole_kernel_cse(root)) + if (config.opt_level > 0 && per_task_cse(root)) modified = true; if (should_dump) - dump_step("11_whole_kernel_cse", iteration); + dump_step("11_per_task_cse", iteration); // Don't do this time-consuming optimization pass again if the IR is // not modified. if (config.opt_level > 0 && first_iteration && config.cfg_optimization && diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index c7547d5f8b..d43c5e8cec 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -260,11 +260,63 @@ class WholeKernelCSE : public BasicStmtVisitor { } }; +namespace { + +// Collect the top-level offloaded tasks of |root| iff |root| is an already-offloaded kernel body (a Block whose +// statements are all OffloadedStmt). Empty otherwise. +std::vector collect_offloaded_tasks(IRNode *root) { + std::vector tasks; + auto *block = root->cast(); + if (block == nullptr || block->statements.empty()) { + return tasks; + } + for (auto &stmt : block->statements) { + if (!stmt->is()) { + return {}; + } + } + for (auto &stmt : block->statements) { + tasks.push_back(stmt->as()); + } + return tasks; +} + +// Run CSE on a single offloaded task, scoped to that task alone via a throwaway wrapper block. +bool cse_one_task(Block *parent, OffloadedStmt *off) { + const int location = parent->locate(off); + QD_ASSERT(location != -1); + Block wrapper; + wrapper.insert(parent->extract(off)); + const bool modified = WholeKernelCSE::run(&wrapper); + parent->insert(wrapper.extract(off), location); + return modified; +} + +} // namespace + namespace irpass { bool whole_kernel_cse(IRNode *root) { QD_AUTO_PROF; return WholeKernelCSE::run(root); } + +// Per-offloaded-task CSE, parallelized across the codegen worker pool. The post-offload full_simplify passes +// (offload_to_executable: before_lower_access / simplify_IV / scalarize) run inside compile_task, which is enqueued +// per task to the codegen worker pool. At that point each worker's block holds exactly ONE offloaded task, so CSE +// runs here -> per task, in parallel, inside the simplify fixpoint (full quality). On the pre-split monolith +// (full_simplify in compile_to_offloads, where the block still holds every task) CSE is skipped: it is deferred to +// the parallel per-task pass above. Tasks are optimized independently, so doing all CSE in the workers is +// value-identical to doing it on the monolith -- just parallel. Before offload there are no offloaded tasks, so CSE +// is deferred likewise. +bool per_task_cse(IRNode *root) { + QD_AUTO_PROF; + auto tasks = collect_offloaded_tasks(root); + if (tasks.size() != 1) { + return false; + } + auto *block = root->as(); + return cse_one_task(block, tasks[0]); +} } // namespace irpass } // namespace quadrants::lang From 285202d6fa75fe4e07af21f3aa5ec326bcf1e9bf Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 11:53:54 -0700 Subject: [PATCH 02/26] cse/fix-licm-cache: address-key the loop-invariant global cache (fix stale break-flag) cache_loop_invariant_global_vars previously keyed its per-loop cache by GlobalPtrStmt* identity. With per-task CSE (post-offload) the read and write GlobalPtrStmts to the same address (e.g. the solver's improved[i_b] break flag) are distinct statements, so pointer-identity keying allocated separate locals -> in-loop reads never saw in-loop writes -> stale break flag -> solver ran to the iteration cap (the -88% per-offload-CSE regression). find_cache_entry now also matches an existing entry by definitely_same_address, so read+write share one local. Re-enables the pass (compile_config default true) which is load-bearing on contact-heavy solves (duck_in_box). QD_LICM_LOG=1 prints per-access cache decisions for diagnosis. --- quadrants/program/compile_config.h | 8 +-- .../cache_loop_invariant_global_vars.cpp | 62 ++++++++++++++++++- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/quadrants/program/compile_config.h b/quadrants/program/compile_config.h index 151cc0194b..995550b6da 100644 --- a/quadrants/program/compile_config.h +++ b/quadrants/program/compile_config.h @@ -25,10 +25,10 @@ struct CompileConfig { bool lower_access; bool simplify_after_lower_access; bool move_loop_invariant_outside_if; - // Per-offloaded-task CSE no longer merges read/write GlobalPtrStmts to the same address before offload, which the - // post-offload cache_loop_invariant_global_vars pass relies on (it keys its cache by GlobalPtrStmt* identity). - // Defaulted off on this branch to keep that pass sound; per-task CSE already removes the redundant loads it targeted. - bool cache_loop_invariant_global_vars{false}; + // Re-enabled: cache_loop_invariant_global_vars is now address-keyed (see cache_loop_invariant_global_vars.cpp), so it + // stays sound even when read/write GlobalPtrStmts to the same address are not merged (which per-task CSE cannot do + // post-offload). This keeps the pass's optimization (load-bearing on contact-heavy solves, e.g. duck_in_box). + bool cache_loop_invariant_global_vars{true}; bool demote_dense_struct_fors; bool advanced_optimization; bool constant_folding; diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index 9fdb0f280b..a1aebd3521 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -1,6 +1,10 @@ #include "quadrants/transforms/loop_invariant_detector.h" #include "quadrants/ir/analysis.h" +#include +#include +#include + namespace quadrants::lang { namespace { @@ -194,9 +198,63 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { modifier.insert_before(get_loop_stmt(depth), std::move(local_store)); } + static bool licm_log() { + static const bool v = []() { + const char *e = std::getenv("QD_LICM_LOG"); + return e != nullptr && std::string(e) == "1"; + }(); + return v; + } + + static std::string describe_dest(Stmt *dest) { + GlobalPtrStmt *g = nullptr; + if (dest->is()) { + g = dest->as(); + } else if (dest->is() && dest->as()->origin->is()) { + g = dest->as()->origin->as(); + } + if (g) { + return "id$" + std::to_string(dest->id) + " snode#" + std::to_string(g->snode->id) + "(" + + g->snode->get_node_type_name() + ")"; + } + return "id$" + std::to_string(dest->id) + " non-global"; + } + + // Match an existing cache entry at |depth| by GlobalPtrStmt* identity, or by provable same-address + // (definitely_same_address). Address matching is required because per-task CSE (post-offload) cannot merge a + // read GlobalPtrStmt (activate=false, LICM-hoisted) with the in-loop write GlobalPtrStmts to the same address; + // keying purely by pointer identity would then allocate separate locals -> stale in-loop reads. + std::pair *find_cache_entry(int depth, Stmt *dest) { + auto &m = cached_maps[depth]; + auto it = m.find(dest); + if (it != m.end() && it->second.first != CacheStatus::None) { + return &it->second; + } + for (auto &kv : m) { + if (kv.second.first != CacheStatus::None && kv.first != dest && + irpass::analysis::definitely_same_address(kv.first, dest)) { + return &kv.second; + } + } + return nullptr; + } + AllocaStmt *cache_global_to_local(Stmt *dest, CacheStatus status, int depth) { - if (auto &[cached_status, alloca_stmt] = cached_maps[depth][dest]; cached_status != CacheStatus::None) { - // The global variable has already been cached. + auto *entry = find_cache_entry(depth, dest); + if (licm_log()) { + std::printf("[LICM] cache dest=%s status=%d depth=%d match=%s mapsz=%zu\n", describe_dest(dest).c_str(), + (int)status, depth, entry ? "YES" : "NO", cached_maps[depth].size()); + if (!entry) { + for (auto &kv : cached_maps[depth]) { + std::printf("[LICM] existing=%s status=%d dsa=%d\n", describe_dest(kv.first).c_str(), + (int)kv.second.first, (int)irpass::analysis::definitely_same_address(kv.first, dest)); + } + } + std::fflush(stdout); + } + if (entry) { + auto &[cached_status, alloca_stmt] = *entry; + // The global variable has already been cached (same pointer or provably same address). if (cached_status == CacheStatus::Read && status == CacheStatus::Write) { add_writeback(alloca_stmt, dest, depth); cached_status = CacheStatus::ReadWrite; From 32eef330df11994387ca781bbbd305b1b0d008b6 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 12:44:56 -0700 Subject: [PATCH 03/26] cse/fix-licm-cache: two-phase soundness - exclude snodes with uncacheable (conditional) accesses QD_LICM_LOG on anymal_zero showed address-keying alone is insufficient: the pass is unsound whenever a snode has an access that is cache-eligible (offload-unique, static index, non-atomic) yet not cacheable at its own site - typically a store inside an if-block that LICM did not hoist (move_loop_invariant_outside_if off). That store bypasses the cached loop-invariant local while reads still read the local -> stale (the solver break-flag / -88% regression). fix_addr could never help because such stores never reach cache_global_to_local. Phase 1 traverses each task and marks any such snode UNSAFE; phase 2 caches as before but skips UNSAFE snodes, leaving all their accesses on global (always correct). Same-depth read/write pairs still unify via the address-aware find_cache_entry, so the pass keeps its optimization on the clean (contact-heavy/duck) snodes. --- .../cache_loop_invariant_global_vars.cpp | 84 +++++++++++++++++-- 1 file changed, 78 insertions(+), 6 deletions(-) diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index a1aebd3521..c33ec3cf41 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -65,6 +65,12 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { OffloadedStmt *current_offloaded; + // Two-phase soundness state. In phase 1 (analyzing_) we only record which snodes cannot be soundly cached; + // in phase 2 we transform, skipping those snodes. See visit(OffloadedStmt). + bool analyzing_ = false; + std::unordered_set unsafe_analysis_; // accumulator during phase 1 + std::unordered_set unsafe_snodes_; // frozen result consulted during phase 2 + explicit CacheLoopInvariantGlobalVars(const CompileConfig &config) : LoopInvariantDetector(config) { } @@ -83,16 +89,37 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { gather_atomic_dests(stmt, atomic_dest_snodes_, atomic_dest_arr_ids_); // We don't need to visit TLS/BLS prologues/epilogues. - if (stmt->body) { - if (stmt->task_type == OffloadedStmt::TaskType::range_for || stmt->task_type == OffloadedTaskType::mesh_for || - stmt->task_type == OffloadedStmt::TaskType::struct_for) - visit_loop(stmt->body.get()); - else - stmt->body->accept(this); + if (!stmt->body) { + current_offloaded = nullptr; + return; } + + // Phase 1 (analysis): find snodes that cannot be soundly cached. A snode is unsafe if some access to it is + // eligible for caching (offload-unique, static index, non-atomic) yet not cacheable at its own site -- e.g. a + // store inside an if-block that LICM did not hoist out (move_loop_invariant_outside_if is off). If we cached + // this snode's other (cacheable) accesses into a loop-invariant local, that un-hoistable access would still + // read/write global directly, so the local goes stale. Under whole-kernel CSE all accesses share one hoisted + // pointer and this never happens; under per-task CSE the read/write pointers stay split, exposing it (this is + // the solver break-flag / -88% regression). Excluding such snodes keeps every access on global -> correct. + analyzing_ = true; + unsafe_analysis_.clear(); + run_body(stmt); + analyzing_ = false; + unsafe_snodes_ = std::move(unsafe_analysis_); + + // Phase 2 (transform): cache as before, but skip the unsafe snodes. + run_body(stmt); current_offloaded = nullptr; } + void run_body(OffloadedStmt *stmt) { + if (stmt->task_type == OffloadedStmt::TaskType::range_for || stmt->task_type == OffloadedTaskType::mesh_for || + stmt->task_type == OffloadedStmt::TaskType::struct_for) + visit_loop(stmt->body.get()); + else + stmt->body->accept(this); + } + bool is_dynamically_indexed(Stmt *stmt) { // Handle GlobalPtrStmt Stmt *ptr_stmt = nullptr; @@ -323,11 +350,49 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { return depth; } + static const SNode *dest_snode(Stmt *dest) { + if (dest->is()) { + return dest->as()->snode; + } + if (dest->is() && dest->as()->origin->is()) { + return dest->as()->origin->as()->snode; + } + return nullptr; + } + + // Would this pass otherwise want to cache accesses to |dest| (offload-unique, statically indexed, non-atomic)? + // Whether a *particular* access is cacheable additionally depends on its scope (find_cache_depth_if_cacheable). + bool cache_eligible(Stmt *dest) { + return !is_dynamically_indexed(dest) && is_offload_unique(dest) && !is_atomic_dest(dest); + } + + // Phase-1 hook: mark the snode unsafe if this access is cache-eligible but not cacheable at its own site. + void analyze_access(Stmt *dest, Block *scope) { + const SNode *sn = dest_snode(dest); + if (!sn || !cache_eligible(dest)) { + return; + } + if (!find_cache_depth_if_cacheable(dest, scope).has_value()) { + if (unsafe_analysis_.insert(sn).second && licm_log()) { + std::printf("[LICM] UNSAFE snode#%d(%s) via %s\n", sn->id, sn->get_node_type_name().c_str(), + describe_dest(dest).c_str()); + std::fflush(stdout); + } + } + } + void visit(GlobalLoadStmt *stmt) override { // Volatile loads must read from memory on every execution (spin-wait correctness); skip caching. if (stmt->is_volatile) { return; } + if (analyzing_) { + analyze_access(stmt->src, stmt->parent); + return; + } + if (const SNode *sn = dest_snode(stmt->src); sn && unsafe_snodes_.count(sn)) { + return; + } if (auto depth = find_cache_depth_if_cacheable(stmt->src, stmt->parent)) { auto alloca_stmt = cache_global_to_local(stmt->src, CacheStatus::Read, depth.value()); auto local_load = std::make_unique(alloca_stmt); @@ -338,6 +403,13 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { } void visit(GlobalStoreStmt *stmt) override { + if (analyzing_) { + analyze_access(stmt->dest, stmt->parent); + return; + } + if (const SNode *sn = dest_snode(stmt->dest); sn && unsafe_snodes_.count(sn)) { + return; + } if (auto depth = find_cache_depth_if_cacheable(stmt->dest, stmt->parent)) { auto alloca_stmt = cache_global_to_local(stmt->dest, CacheStatus::Write, depth.value()); auto local_store = std::make_unique(alloca_stmt, stmt->val); From fc5cd0f5b81db2f24880378f834556610e50ac7f Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 13:12:06 -0700 Subject: [PATCH 04/26] cse/fix-licm-cache: add cheap pre-offload pointer-only merge (merge_global_ptrs) WholeKernelCSE gains a ptrs_only mode that eliminates only Global/External/MatrixPtr statements (no compute CSE, no if-branch hoisting). Exposed as irpass::merge_global_ptrs and called in compile_to_offloads right before the first flag_access. This restores the one precondition cache_loop_invariant_global_vars needs on contact-heavy scenes: each global's read and write pointers become a single shared activate=true pointer, so conditional/in-if stores become hoistable/cacheable (recovering the duck_in_box optimization) - while the expensive whole-kernel compute dedup stays deferred to per-task CSE. QD_NO_PTR_MERGE=1 disables for A/B. --- quadrants/ir/transforms.h | 1 + quadrants/transforms/compile_to_offloads.cpp | 13 ++++++++ quadrants/transforms/whole_kernel_cse.cpp | 31 ++++++++++++++++---- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/quadrants/ir/transforms.h b/quadrants/ir/transforms.h index 0e9ae19737..2dbd08e671 100644 --- a/quadrants/ir/transforms.h +++ b/quadrants/ir/transforms.h @@ -47,6 +47,7 @@ bool demote_operations(IRNode *root, const CompileConfig &config); bool binary_op_simplify(IRNode *root, const CompileConfig &config); bool whole_kernel_cse(IRNode *root); bool per_task_cse(IRNode *root); +bool merge_global_ptrs(IRNode *root); bool extract_constant(IRNode *root, const CompileConfig &config); bool unreachable_code_elimination(IRNode *root); bool loop_invariant_code_motion(IRNode *root, const CompileConfig &config); diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index f1a0918ea7..70a5c9b18c 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -131,6 +131,19 @@ void compile_to_offloads(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); } + // Merge same-address pointer statements across the whole kernel BEFORE the first flag_access. flag_access stamps + // read-only pointers activate=false, cementing the read/write split; merging first gives each global one shared + // activate=true pointer, which is what cache_loop_invariant_global_vars needs to cache conditional/in-if stores + // (the contact-heavy duck_in_box optimization). Only pointers are merged here -- the expensive compute CSE is + // deferred to per-task CSE (post-offload). Set QD_NO_PTR_MERGE=1 to disable (A/B). + { + const char *no_merge = std::getenv("QD_NO_PTR_MERGE"); + if (no_merge == nullptr || std::string(no_merge) != "1") { + irpass::merge_global_ptrs(ir); + irpass::analysis::verify_if_debug(ir, config); + } + } + irpass::flag_access(ir); irpass::analysis::verify_if_debug(ir, config); diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index d43c5e8cec..dab3450137 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -78,14 +78,23 @@ class WholeKernelCSE : public BasicStmtVisitor { std::vector>> scope_inserts_; DelayedIRModifier modifier_; + // When true, only address-computation statements (Global/External/MatrixPtr) are eliminated; all other statements + // are left untouched. Used pre-offload to merge same-address read/write pointers (the cheap, load-bearing part of + // whole-kernel CSE) without doing the expensive whole-kernel compute dedup, which is deferred to per-task CSE. + bool ptrs_only_ = false; + public: using BasicStmtVisitor::visit; - WholeKernelCSE() { + explicit WholeKernelCSE(bool ptrs_only = false) : ptrs_only_(ptrs_only) { allow_undefined_visitor = true; invoke_default_visitor = true; } + static bool is_ptr_stmt(Stmt *stmt) { + return stmt->is() || stmt->is() || stmt->is(); + } + bool is_done(Stmt *stmt) { return visited_.find(stmt->instance_id) != visited_.end(); } @@ -160,6 +169,9 @@ class WholeKernelCSE : public BasicStmtVisitor { // container_statement does not need to be CSE-ed if (stmt->is_container_statement()) return; + // Pointers-only mode: skip every non-address statement (leave compute CSE to per-task CSE). + if (ptrs_only_ && !is_ptr_stmt(stmt)) + return; // Generic visitor for all CSE-able statements. std::size_t hash_value = operand_hash(stmt); if (is_done(stmt)) { @@ -218,8 +230,8 @@ class WholeKernelCSE : public BasicStmtVisitor { } // Move common statements at the beginning or the end of both branches - // outside. - if (if_stmt->true_statements && if_stmt->false_statements) { + // outside. Skipped in pointers-only mode: we do not want to relocate arbitrary compute. + if (!ptrs_only_ && if_stmt->true_statements && if_stmt->false_statements) { auto &true_clause = if_stmt->true_statements; auto &false_clause = if_stmt->false_statements; if (irpass::analysis::same_statements(true_clause->statements[0].get(), false_clause->statements[0].get())) { @@ -246,8 +258,8 @@ class WholeKernelCSE : public BasicStmtVisitor { if_stmt->false_statements->accept(this); } - static bool run(IRNode *node) { - WholeKernelCSE eliminator; + static bool run(IRNode *node, bool ptrs_only = false) { + WholeKernelCSE eliminator(ptrs_only); bool modified = false; while (true) { node->accept(&eliminator); @@ -300,6 +312,15 @@ bool whole_kernel_cse(IRNode *root) { return WholeKernelCSE::run(root); } +// Cheap whole-kernel merge of same-address pointer statements only (Global/External/MatrixPtr), leaving all compute +// alone. Run pre-offload (before the first flag_access) so a global's read and write pointers become one shared, +// activate=true pointer -- the precondition cache_loop_invariant_global_vars relies on to cache conditional/in-if +// stores. This is the load-bearing part of whole-kernel CSE; the expensive compute dedup stays per-task. +bool merge_global_ptrs(IRNode *root) { + QD_AUTO_PROF; + return WholeKernelCSE::run(root, /*ptrs_only=*/true); +} + // Per-offloaded-task CSE, parallelized across the codegen worker pool. The post-offload full_simplify passes // (offload_to_executable: before_lower_access / simplify_IV / scalarize) run inside compile_task, which is enqueued // per task to the codegen worker pool. At that point each worker's block holds exactly ONE offloaded task, so CSE From 59730953d37e1c58d3859f8b67a0c77a0846b611 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 13:25:18 -0700 Subject: [PATCH 05/26] merge_global_ptrs: also merge integer addressing arithmetic + PTRMERGE log Pointers-only mode now also eliminates pure integer (index) compute, not just Global/External/MatrixPtr. Two same-address pointers only satisfy definitely_same_address once their index arithmetic collapses to a common base statement (value_diff_ptr_index/FindDirectValueBaseAndOffset), so merging pointers alone was a near no-op. Loads and stores remain non-eliminable so soundness is unchanged; float compute stays deferred to per-task CSE. QD_LICM_LOG=1 prints [PTRMERGE] round/modified counts. --- quadrants/transforms/whole_kernel_cse.cpp | 33 +++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index dab3450137..91795641ed 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -2,10 +2,14 @@ #include "quadrants/ir/analysis.h" #include "quadrants/ir/statements.h" #include "quadrants/ir/transforms.h" +#include "quadrants/ir/type_utils.h" #include "quadrants/ir/visitors.h" #include "quadrants/system/profiler.h" #include +#include +#include +#include namespace quadrants::lang { @@ -95,6 +99,21 @@ class WholeKernelCSE : public BasicStmtVisitor { return stmt->is() || stmt->is() || stmt->is(); } + // What ptrs_only mode is allowed to eliminate: address-computation statements PLUS pure integer (addressing) + // arithmetic. The latter is required because two same-address pointers only prove equal (value_diff_ptr_index -> + // FindDirectValueBaseAndOffset) once their index arithmetic is itself merged to a common base statement. Global + // loads/stores are never eliminable (common_statement_eliminable()==false), so this stays sound; float solver + // compute is left untouched and deferred to per-task CSE. + static bool eligible_in_ptrs_only(Stmt *stmt) { + if (is_ptr_stmt(stmt)) { + return true; + } + if ((Type *)stmt->ret_type == nullptr) { + return false; + } + return is_integral(stmt->ret_type.get_element_type()); + } + bool is_done(Stmt *stmt) { return visited_.find(stmt->instance_id) != visited_.end(); } @@ -169,8 +188,9 @@ class WholeKernelCSE : public BasicStmtVisitor { // container_statement does not need to be CSE-ed if (stmt->is_container_statement()) return; - // Pointers-only mode: skip every non-address statement (leave compute CSE to per-task CSE). - if (ptrs_only_ && !is_ptr_stmt(stmt)) + // Pointers-only mode: skip every non-address / non-integer-addressing statement (leave float compute CSE to + // per-task CSE). + if (ptrs_only_ && !eligible_in_ptrs_only(stmt)) return; // Generic visitor for all CSE-able statements. std::size_t hash_value = operand_hash(stmt); @@ -261,13 +281,22 @@ class WholeKernelCSE : public BasicStmtVisitor { static bool run(IRNode *node, bool ptrs_only = false) { WholeKernelCSE eliminator(ptrs_only); bool modified = false; + int rounds = 0; while (true) { node->accept(&eliminator); + rounds++; if (eliminator.modifier_.modify_ir()) modified = true; else break; } + if (ptrs_only) { + const char *log = std::getenv("QD_LICM_LOG"); + if (log != nullptr && std::string(log) == "1") { + std::printf("[PTRMERGE] ran %d round(s), modified=%d\n", rounds, (int)modified); + std::fflush(stdout); + } + } return modified; } }; From a623e2e87662ca916107bce884ec15642d29d7ee Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 13:34:03 -0700 Subject: [PATCH 06/26] merge_global_ptrs: move to AFTER offload (before flag_access #2) Pre-offload the pass found nothing (PTRMERGE modified=0): offload is what clones each global's address computation into per-task read/write GlobalPtrStmts. Run the merge on the offloaded IR right before flag_access #2 so the still-activate=true duplicates unify into one shared pointer before flag_access can stamp the read-only copy activate=false. This is the split that blocks cache_loop_invariant_global_vars. --- quadrants/transforms/compile_to_offloads.cpp | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index 70a5c9b18c..6a71a63537 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -131,19 +131,6 @@ void compile_to_offloads(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); } - // Merge same-address pointer statements across the whole kernel BEFORE the first flag_access. flag_access stamps - // read-only pointers activate=false, cementing the read/write split; merging first gives each global one shared - // activate=true pointer, which is what cache_loop_invariant_global_vars needs to cache conditional/in-if stores - // (the contact-heavy duck_in_box optimization). Only pointers are merged here -- the expensive compute CSE is - // deferred to per-task CSE (post-offload). Set QD_NO_PTR_MERGE=1 to disable (A/B). - { - const char *no_merge = std::getenv("QD_NO_PTR_MERGE"); - if (no_merge == nullptr || std::string(no_merge) != "1") { - irpass::merge_global_ptrs(ir); - irpass::analysis::verify_if_debug(ir, config); - } - } - irpass::flag_access(ir); irpass::analysis::verify_if_debug(ir, config); @@ -154,6 +141,22 @@ void compile_to_offloads(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); dump_ir("after_offload"); + + // Merge same-address pointer statements across the whole (offloaded) kernel BEFORE this flag_access. `offload` + // clones each global's address computation into every task/access, so post-offload a global has separate read and + // write GlobalPtrStmts (still activate=true here). flag_access then stamps the read-only one activate=false, + // cementing the split that stops cache_loop_invariant_global_vars from caching conditional/in-if stores (the -88% + // solver break-flag bug + the lost duck_in_box optimization). Merging first gives each global one shared + // activate=true pointer. Only pointers + integer addressing arithmetic are merged; the expensive float compute CSE + // stays deferred to per-task CSE. Set QD_NO_PTR_MERGE=1 to disable (A/B). + { + const char *no_merge = std::getenv("QD_NO_PTR_MERGE"); + if (no_merge == nullptr || std::string(no_merge) != "1") { + irpass::merge_global_ptrs(ir); + irpass::analysis::verify_if_debug(ir, config); + } + } + // NOTE: There was an additional CFG pass here, removed in // https://github.com/taichi-dev/taichi/pull/8691 irpass::flag_access(ir); From 5dac84ec23ce40e88ca8fe6f60384b9ae1f83db4 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 13:56:19 -0700 Subject: [PATCH 07/26] merge_global_ptrs: run inside full_simplify fixpoint (pre-offload dedup), drop standalone Evidence (solve_monolith IR): main's whole_kernel_cse runs in full_simplify's fixpoint at simplify_I, before the first flag_access, deduping read/write pointers so flag_access never splits them (after_offload: 1 split). The per-task branch did no CSE pre-offload, so flag_access split ~16 addresses; a single standalone post-offload merge could not undo it (activate=false read dominates; CSE won't merge a later activate=true write into it) - only 16->13. Running the cheap pointer+integer merge in the fixpoint (like main) fires at simplify_I before the split. Removed the ineffective standalone post-offload call. --- quadrants/transforms/compile_to_offloads.cpp | 16 ---------------- quadrants/transforms/simplify.cpp | 10 ++++++++++ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index 6a71a63537..f1a0918ea7 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -141,22 +141,6 @@ void compile_to_offloads(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); dump_ir("after_offload"); - - // Merge same-address pointer statements across the whole (offloaded) kernel BEFORE this flag_access. `offload` - // clones each global's address computation into every task/access, so post-offload a global has separate read and - // write GlobalPtrStmts (still activate=true here). flag_access then stamps the read-only one activate=false, - // cementing the split that stops cache_loop_invariant_global_vars from caching conditional/in-if stores (the -88% - // solver break-flag bug + the lost duck_in_box optimization). Merging first gives each global one shared - // activate=true pointer. Only pointers + integer addressing arithmetic are merged; the expensive float compute CSE - // stays deferred to per-task CSE. Set QD_NO_PTR_MERGE=1 to disable (A/B). - { - const char *no_merge = std::getenv("QD_NO_PTR_MERGE"); - if (no_merge == nullptr || std::string(no_merge) != "1") { - irpass::merge_global_ptrs(ir); - irpass::analysis::verify_if_debug(ir, config); - } - } - // NOTE: There was an additional CFG pass here, removed in // https://github.com/taichi-dev/taichi/pull/8691 irpass::flag_access(ir); diff --git a/quadrants/transforms/simplify.cpp b/quadrants/transforms/simplify.cpp index 28028b855b..b8486a206e 100644 --- a/quadrants/transforms/simplify.cpp +++ b/quadrants/transforms/simplify.cpp @@ -574,6 +574,16 @@ void full_simplify(IRNode *root, const CompileConfig &config, const FullSimplify modified = true; if (should_dump) dump_step("11_per_task_cse", iteration); + // Cheap whole-kernel merge of same-address pointers + integer addressing arithmetic, run in the fixpoint like + // main's whole_kernel_cse. Pre-offload (where per_task_cse no-ops because there are no tasks yet) this dedups a + // global's separate read/write pointers into one BEFORE the first flag_access can stamp the read-only copy + // activate=false -- the split that otherwise stops cache_loop_invariant_global_vars from caching conditional + // in-if stores (the -88% solver break-flag bug + the lost duck_in_box optimization). The expensive float + // compute dedup stays in per_task_cse (post-offload, per task). + if (config.opt_level > 0 && merge_global_ptrs(root)) + modified = true; + if (should_dump) + dump_step("11b_merge_global_ptrs", iteration); // Don't do this time-consuming optimization pass again if the IR is // not modified. if (config.opt_level > 0 && first_iteration && config.cfg_optimization && From 969e16902a6730afbf1be0e73381abf78a7bcf67 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 14:20:56 -0700 Subject: [PATCH 08/26] cache_loop_invariant: QD_LICM_NO_EXCLUDE to disable two-phase exclusion Now that merge_global_ptrs unifies read/write pointers in the full_simplify fixpoint (splits 16->1 like main), the two-phase UNSAFE exclusion is a redundant workaround that also disables caching for contact-heavy (duck) fields -> suspected cause of the residual duck regression despite splits being fixed. Toggle to A/B. --- .../cache_loop_invariant_global_vars.cpp | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index c33ec3cf41..aa4878b661 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -101,17 +101,31 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { // read/write global directly, so the local goes stale. Under whole-kernel CSE all accesses share one hoisted // pointer and this never happens; under per-task CSE the read/write pointers stay split, exposing it (this is // the solver break-flag / -88% regression). Excluding such snodes keeps every access on global -> correct. - analyzing_ = true; - unsafe_analysis_.clear(); - run_body(stmt); - analyzing_ = false; - unsafe_snodes_ = std::move(unsafe_analysis_); + // The two-phase exclusion is a workaround for the read/write pointer split that per-task CSE used to leave in the + // IR. With merge_global_ptrs now unifying those pointers in the full_simplify fixpoint (like whole-kernel CSE on + // main), the split is gone and the exclusion is unnecessary; QD_LICM_NO_EXCLUDE=1 disables it to A/B that. + unsafe_snodes_.clear(); + if (!no_exclude()) { + analyzing_ = true; + unsafe_analysis_.clear(); + run_body(stmt); + analyzing_ = false; + unsafe_snodes_ = std::move(unsafe_analysis_); + } // Phase 2 (transform): cache as before, but skip the unsafe snodes. run_body(stmt); current_offloaded = nullptr; } + static bool no_exclude() { + static const bool v = []() { + const char *e = std::getenv("QD_LICM_NO_EXCLUDE"); + return e != nullptr && std::string(e) == "1"; + }(); + return v; + } + void run_body(OffloadedStmt *stmt) { if (stmt->task_type == OffloadedStmt::TaskType::range_for || stmt->task_type == OffloadedTaskType::mesh_for || stmt->task_type == OffloadedStmt::TaskType::struct_for) From e0d6af1a286ec8c5354e6c0dcba4b5a63c566d12 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 14:38:19 -0700 Subject: [PATCH 09/26] cache_loop_invariant: exclusion OFF by default (merge_global_ptrs is the root-cause fix) Bench confirms merge_global_ptrs (fixpoint) alone fixes both problems: anymal_zero at full baseline (was -88%) AND duck_in_box within noise of main (+0.5% / -1.0% / +1.6% / -2.0%, vs -8..-13% with the exclusion on). The two-phase UNSAFE exclusion recovered anymal but kept duck at pass-off, so it is now off by default. QD_LICM_EXCLUDE=1 re-enables it as a safety valve. --- .../transforms/cache_loop_invariant_global_vars.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index aa4878b661..f97a682f4f 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -102,10 +102,12 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { // pointer and this never happens; under per-task CSE the read/write pointers stay split, exposing it (this is // the solver break-flag / -88% regression). Excluding such snodes keeps every access on global -> correct. // The two-phase exclusion is a workaround for the read/write pointer split that per-task CSE used to leave in the - // IR. With merge_global_ptrs now unifying those pointers in the full_simplify fixpoint (like whole-kernel CSE on - // main), the split is gone and the exclusion is unnecessary; QD_LICM_NO_EXCLUDE=1 disables it to A/B that. + // IR. merge_global_ptrs now unifies those pointers in the full_simplify fixpoint (like whole-kernel CSE on main), + // so the split is gone at the root: this fixes the -88% break-flag bug AND recovers the duck_in_box cache + // optimization (bench: duck within noise of main), whereas the exclusion alone recovered anymal but left duck at + // pass-off -12%. The exclusion is therefore off by default; QD_LICM_EXCLUDE=1 re-enables it as a safety valve. unsafe_snodes_.clear(); - if (!no_exclude()) { + if (do_exclude()) { analyzing_ = true; unsafe_analysis_.clear(); run_body(stmt); @@ -118,9 +120,9 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { current_offloaded = nullptr; } - static bool no_exclude() { + static bool do_exclude() { static const bool v = []() { - const char *e = std::getenv("QD_LICM_NO_EXCLUDE"); + const char *e = std::getenv("QD_LICM_EXCLUDE"); return e != nullptr && std::string(e) == "1"; }(); return v; From 5897f922c7396538a1035703b5121f63497d610f Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Thu, 16 Jul 2026 14:39:58 -0700 Subject: [PATCH 10/26] merge_global_ptrs: QD_NO_PTR_MERGE=1 disables it (compile-cost A/B on one build) --- quadrants/transforms/whole_kernel_cse.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index 91795641ed..d73ccd0c55 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -347,6 +347,13 @@ bool whole_kernel_cse(IRNode *root) { // stores. This is the load-bearing part of whole-kernel CSE; the expensive compute dedup stays per-task. bool merge_global_ptrs(IRNode *root) { QD_AUTO_PROF; + static const bool disabled = []() { + const char *e = std::getenv("QD_NO_PTR_MERGE"); + return e != nullptr && std::string(e) == "1"; + }(); + if (disabled) { + return false; + } return WholeKernelCSE::run(root, /*ptrs_only=*/true); } From a323a8d2bf364a998dcf7e079293f51afe1735b1 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Fri, 17 Jul 2026 05:51:26 -0700 Subject: [PATCH 11/26] merge_global_ptrs: gate to pre-offload only (kill ~3.5% compile regression) The pass is only load-bearing pre-offload (unify read/write pointers before the first flag_access). Post-offload, per_task_cse already merges pointers within each task, so running merge_global_ptrs in every post-offload full_simplify fixpoint iteration (per task, in the codegen workers) was pure redundant work -> ~3.5% compile regression across the suite. Now skips as soon as any top-level OffloadedStmt is present. Pre-offload behavior (the correctness fix) is unchanged. --- quadrants/transforms/whole_kernel_cse.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index d73ccd0c55..f2daf8b9a0 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -354,6 +354,18 @@ bool merge_global_ptrs(IRNode *root) { if (disabled) { return false; } + // Only needed PRE-offload: this exists solely to unify a global's read/write pointers before the first + // flag_access splits them (activate=false read vs activate=true write). Once the kernel is offloaded, per_task_cse + // merges pointers within each task, so running here is redundant work -- and doing it in every post-offload + // full_simplify fixpoint iteration (per task, in the codegen workers) was a ~3.5% compile regression. Skip as soon + // as any top-level OffloadedStmt is present (post-offload monolith or a single-task worker block). + if (auto *block = root->cast()) { + for (auto &stmt : block->statements) { + if (stmt->is()) { + return false; + } + } + } return WholeKernelCSE::run(root, /*ptrs_only=*/true); } From b3724ead4ffb724a599a19a531090b9f16535bf8 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Fri, 17 Jul 2026 06:28:47 -0700 Subject: [PATCH 12/26] merge_global_ptrs: single pre-offload call instead of in the full_simplify fixpoint The pre-offload pointer merge only needs to happen ONCE before the first flag_access (compile_to_offloads), to unify a global's read/write pointers before flag_access stamps the read-only copy activate=false. Running a whole-kernel pointer CSE in every fixpoint iteration of every pre-offload phase (simplify_I/II + autodiff) was a +12-22s compile regression on collision-heavy scenes (franka/duck/box_pyramid; anymal ~0) for zero extra benefit -- same-build A/B: franka_random default 120.8s vs no_merge 108.5s. Now a single irpass::merge_global_ptrs(ir) right before flag_access #1; arithmetic is already canonical after simplify_I so one pass suffices. Correctness path (splits->1, -88% bug + duck optimization) unchanged. --- quadrants/transforms/compile_to_offloads.cpp | 11 +++++++++++ quadrants/transforms/simplify.cpp | 15 +++++---------- quadrants/transforms/whole_kernel_cse.cpp | 9 ++++----- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index f1a0918ea7..33f06e7322 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -131,6 +131,17 @@ void compile_to_offloads(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); } + // Merge a global's separate read/write GlobalPtrStmts (same address) into one shared, activate=true pointer BEFORE + // this first flag_access, so flag_access cannot stamp a read-only (activate=false) copy that the CSE eliminability + // rule then refuses to re-merge with the in-loop write. Without it, cache_loop_invariant_global_vars sees a split + // read/write and cannot cache conditional/in-if stores -> the -88% solver break-flag bug + the lost duck_in_box + // optimization. On main this fell out of whole_kernel_cse running inside every full_simplify fixpoint; per-task CSE + // does no pre-offload whole-kernel CSE, so we do this one cheap, pointers-only pass here instead (arithmetic is + // already canonical after simplify_I, so a single call is enough; running it in the fixpoint was a +12-22s + // compile regression for no extra benefit). + irpass::merge_global_ptrs(ir); + irpass::analysis::verify_if_debug(ir, config); + irpass::flag_access(ir); irpass::analysis::verify_if_debug(ir, config); diff --git a/quadrants/transforms/simplify.cpp b/quadrants/transforms/simplify.cpp index b8486a206e..4e50e01967 100644 --- a/quadrants/transforms/simplify.cpp +++ b/quadrants/transforms/simplify.cpp @@ -574,16 +574,11 @@ void full_simplify(IRNode *root, const CompileConfig &config, const FullSimplify modified = true; if (should_dump) dump_step("11_per_task_cse", iteration); - // Cheap whole-kernel merge of same-address pointers + integer addressing arithmetic, run in the fixpoint like - // main's whole_kernel_cse. Pre-offload (where per_task_cse no-ops because there are no tasks yet) this dedups a - // global's separate read/write pointers into one BEFORE the first flag_access can stamp the read-only copy - // activate=false -- the split that otherwise stops cache_loop_invariant_global_vars from caching conditional - // in-if stores (the -88% solver break-flag bug + the lost duck_in_box optimization). The expensive float - // compute dedup stays in per_task_cse (post-offload, per task). - if (config.opt_level > 0 && merge_global_ptrs(root)) - modified = true; - if (should_dump) - dump_step("11b_merge_global_ptrs", iteration); + // NOTE: the pre-offload same-address pointer merge (merge_global_ptrs) used to run here, in the fixpoint. It only + // needs to happen ONCE before the first flag_access (compile_to_offloads), to unify a global's read/write + // pointers before flag_access stamps the read-only copy activate=false. Running a whole-kernel pointer CSE in + // every fixpoint iteration of every pre-offload phase was a +12-22s compile regression (measured on + // franka/duck/box_pyramid) for zero extra benefit, so it is now a single call in compile_to_offloads. // Don't do this time-consuming optimization pass again if the IR is // not modified. if (config.opt_level > 0 && first_iteration && config.cfg_optimization && diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index f2daf8b9a0..7b457b4722 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -354,11 +354,10 @@ bool merge_global_ptrs(IRNode *root) { if (disabled) { return false; } - // Only needed PRE-offload: this exists solely to unify a global's read/write pointers before the first - // flag_access splits them (activate=false read vs activate=true write). Once the kernel is offloaded, per_task_cse - // merges pointers within each task, so running here is redundant work -- and doing it in every post-offload - // full_simplify fixpoint iteration (per task, in the codegen workers) was a ~3.5% compile regression. Skip as soon - // as any top-level OffloadedStmt is present (post-offload monolith or a single-task worker block). + // Defensive: this is intended to run exactly once PRE-offload (single call in compile_to_offloads, before the first + // flag_access). It must never run post-offload -- per_task_cse handles pointers within each task there, and a + // whole-kernel pointer CSE over an offloaded kernel is both pointless and expensive. Skip if any top-level + // OffloadedStmt is present. if (auto *block = root->cast()) { for (auto &stmt : block->statements) { if (stmt->is()) { From 58669ee6d627b66b7b642caf5692d6d11b72a26c Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Fri, 17 Jul 2026 08:52:24 -0700 Subject: [PATCH 13/26] clang-format: wrap LICM debug printf args (pre-commit) --- quadrants/transforms/cache_loop_invariant_global_vars.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index f97a682f4f..7fe402e0f7 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -289,8 +289,8 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { (int)status, depth, entry ? "YES" : "NO", cached_maps[depth].size()); if (!entry) { for (auto &kv : cached_maps[depth]) { - std::printf("[LICM] existing=%s status=%d dsa=%d\n", describe_dest(kv.first).c_str(), - (int)kv.second.first, (int)irpass::analysis::definitely_same_address(kv.first, dest)); + std::printf("[LICM] existing=%s status=%d dsa=%d\n", describe_dest(kv.first).c_str(), (int)kv.second.first, + (int)irpass::analysis::definitely_same_address(kv.first, dest)); } } std::fflush(stdout); From 34000e6fc45139fc2a2a832edfdc0cfabd9b58ab Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Fri, 17 Jul 2026 10:33:11 -0700 Subject: [PATCH 14/26] test: regression for conditional in-if store to a loop-invariant global (per-task CSE + merge_global_ptrs) Adds test_conditional_store_to_loop_invariant_global: a global that is loop-invariant w.r.t. an inner loop and written conditionally inside an if must not read a stale cached value. This is the split read/write GlobalPtrStmt pattern that merge_global_ptrs unifies pre-offload; without the fix the cached load serves the pre-loop value (the ~88% rigid-solver break-flag regression). Exercises per_task_cse + merge_global_ptrs + the address-keyed cache_loop_invariant_global_vars fix end-to-end. --- tests/python/test_cache_loop_invariant.py | 37 +++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/python/test_cache_loop_invariant.py b/tests/python/test_cache_loop_invariant.py index 5881122a41..aa35a130ba 100644 --- a/tests/python/test_cache_loop_invariant.py +++ b/tests/python/test_cache_loop_invariant.py @@ -59,3 +59,40 @@ def k(x: AnnotationType, result: AnnotationType): k(x, result) for i in range(n): assert result[i] == m, f"result[{i}] = {result[i]}, expected {m}" + + +@test_utils.test() +def test_conditional_store_to_loop_invariant_global() -> None: + """Regression: a loop-invariant global written *conditionally* inside an ``if`` must not read stale. + + ``flag[i]`` is invariant w.r.t. the inner ``j`` loop, so its load is a candidate for + cache_loop_invariant_global_vars. It is written conditionally (``if j >= threshold``) inside that + loop, and the read must observe the store. Before the read and write ``GlobalPtrStmt``s to the same + address were unified pre-offload (``merge_global_ptrs``, run once before the first ``flag_access``), + per-task CSE left them split: ``flag_access`` stamped the hoisted read ``activate=false`` and the CSE + eliminability rule refused to re-merge the later ``activate=true`` conditional write, so the cache served + the pre-loop value. That stale read broke the rigid solver's convergence break-flag (an ~88% runtime + regression). Here it manifests as ``acc`` summing the stale ``0`` instead of the stored ``1``. + """ + n = 4 + m = 8 + threshold = 3 + + @qd.kernel + def k(flag: qd.template(), result: qd.template()): + for i in range(n): # offloaded task + flag[i] = 0 + acc = 0 + for j in range(m): # inner loop; flag[i] is loop-invariant here + if j >= threshold: + flag[i] = 1 # conditional in-if store to the loop-invariant global + acc += flag[i] # must observe the store, not a stale cached load + result[i] = acc + + flag = qd.field(dtype=qd.i32, shape=(n,)) + result = qd.field(dtype=qd.i32, shape=(n,)) + + k(flag, result) + expected = m - threshold # flag == 1 for j in [threshold, m) + for i in range(n): + assert result[i] == expected, f"result[{i}] = {result[i]}, expected {expected}" From af22ec3d8ccb3396f03d194ea1c85de7faf8b7ff Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 20 Jul 2026 09:20:46 -0700 Subject: [PATCH 15/26] fix(licm-cache): extend two-phase soundness exclusion to ndarray (ExternalPtr) args dest_snode() returns null for ExternalPtr, so the two-phase exclusion never covered ndarray-backed globals. Under per-task CSE (post-offload) the split read/write ExternalPtrStmts to an ndarray break-flag are not merged (merge_global_ptrs only unifies GlobalPtr), so a cached loop-invariant local goes stale vs the in-loop store and the loop's break condition never fires: CPU non-terminating hang (observed on the Genesis ndarray path, e.g. tests/rigid/test_asset_loading.py), GPU iteration cap. Track unsafe ndarray arg_ids in phase 1 (dest_arg_id) and skip caching them in phase 2 (dest_is_unsafe). Phase-1 analysis now runs unconditionally (cheap, no IR mutation); the ndarray exclusion is always honored, while the field/snode exclusion stays opt-in (QD_LICM_EXCLUDE) so the duck_in_box optimization via merge_global_ptrs is unchanged. Preserves per-task CSE. --- .../cache_loop_invariant_global_vars.cpp | 83 +++++++++++++++---- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index 7fe402e0f7..4450043f96 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -3,7 +3,9 @@ #include #include +#include #include +#include namespace quadrants::lang { @@ -65,11 +67,14 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { OffloadedStmt *current_offloaded; - // Two-phase soundness state. In phase 1 (analyzing_) we only record which snodes cannot be soundly cached; - // in phase 2 we transform, skipping those snodes. See visit(OffloadedStmt). + // Two-phase soundness state. In phase 1 (analyzing_) we only record which snodes/ndarrays cannot be soundly + // cached; in phase 2 we transform, skipping those. See visit(OffloadedStmt). bool analyzing_ = false; - std::unordered_set unsafe_analysis_; // accumulator during phase 1 - std::unordered_set unsafe_snodes_; // frozen result consulted during phase 2 + std::unordered_set unsafe_analysis_; // field accumulator during phase 1 + std::unordered_set unsafe_snodes_; // frozen field result consulted during phase 2 + using ArgIdSet = std::unordered_set, hashing::Hasher>>; + ArgIdSet unsafe_arr_analysis_; // ndarray accumulator during phase 1 + ArgIdSet unsafe_arr_ids_; // frozen ndarray result consulted during phase 2 explicit CacheLoopInvariantGlobalVars(const CompileConfig &config) : LoopInvariantDetector(config) { } @@ -106,16 +111,25 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { // so the split is gone at the root: this fixes the -88% break-flag bug AND recovers the duck_in_box cache // optimization (bench: duck within noise of main), whereas the exclusion alone recovered anymal but left duck at // pass-off -12%. The exclusion is therefore off by default; QD_LICM_EXCLUDE=1 re-enables it as a safety valve. + // Phase 1 (analysis) runs unconditionally: it is cheap (no IR mutation) and is required for ndarray soundness. + // merge_global_ptrs unifies split read/write GlobalPtrStmts (field case) in the full_simplify fixpoint, so the + // field exclusion is off by default (QD_LICM_EXCLUDE=1 re-enables it) to preserve the duck_in_box optimization. + // merge_global_ptrs does NOT touch ExternalPtrStmt, so the split persists for ndarrays under per-task CSE: the + // ndarray exclusion is therefore always honored, otherwise a cached-load-to-local goes stale against an in-loop + // store to the same array and the loop's break condition never fires (CPU: non-terminating; GPU: iteration cap). unsafe_snodes_.clear(); + unsafe_arr_ids_.clear(); + analyzing_ = true; + unsafe_analysis_.clear(); + unsafe_arr_analysis_.clear(); + run_body(stmt); + analyzing_ = false; + unsafe_arr_ids_ = std::move(unsafe_arr_analysis_); if (do_exclude()) { - analyzing_ = true; - unsafe_analysis_.clear(); - run_body(stmt); - analyzing_ = false; unsafe_snodes_ = std::move(unsafe_analysis_); } - // Phase 2 (transform): cache as before, but skip the unsafe snodes. + // Phase 2 (transform): cache as before, but skip the unsafe snodes/ndarrays. run_body(stmt); current_offloaded = nullptr; } @@ -376,27 +390,62 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { return nullptr; } + // The ndarray arg_id backing |dest| (via ExternalPtrStmt), or nullopt for non-ndarray accesses. + static std::optional> dest_arg_id(Stmt *dest) { + ExternalPtrStmt *eptr = nullptr; + if (dest->is()) { + eptr = dest->as(); + } else if (dest->is() && dest->as()->origin->is()) { + eptr = dest->as()->origin->as(); + } + if (eptr) { + if (auto *arg = eptr->base_ptr->cast()) { + return arg->arg_id; + } + } + return std::nullopt; + } + // Would this pass otherwise want to cache accesses to |dest| (offload-unique, statically indexed, non-atomic)? // Whether a *particular* access is cacheable additionally depends on its scope (find_cache_depth_if_cacheable). bool cache_eligible(Stmt *dest) { return !is_dynamically_indexed(dest) && is_offload_unique(dest) && !is_atomic_dest(dest); } - // Phase-1 hook: mark the snode unsafe if this access is cache-eligible but not cacheable at its own site. + // Phase-1 hook: mark the snode/ndarray unsafe if this access is cache-eligible but not cacheable at its own site. void analyze_access(Stmt *dest, Block *scope) { const SNode *sn = dest_snode(dest); - if (!sn || !cache_eligible(dest)) { + auto arg_id = dest_arg_id(dest); + if ((!sn && !arg_id) || !cache_eligible(dest)) { return; } if (!find_cache_depth_if_cacheable(dest, scope).has_value()) { - if (unsafe_analysis_.insert(sn).second && licm_log()) { - std::printf("[LICM] UNSAFE snode#%d(%s) via %s\n", sn->id, sn->get_node_type_name().c_str(), - describe_dest(dest).c_str()); - std::fflush(stdout); + if (sn) { + if (unsafe_analysis_.insert(sn).second && licm_log()) { + std::printf("[LICM] UNSAFE snode#%d(%s) via %s\n", sn->id, sn->get_node_type_name().c_str(), + describe_dest(dest).c_str()); + std::fflush(stdout); + } + } else if (arg_id) { + if (unsafe_arr_analysis_.insert(*arg_id).second && licm_log()) { + std::printf("[LICM] UNSAFE ndarray arg via %s\n", describe_dest(dest).c_str()); + std::fflush(stdout); + } } } } + // A previously-identified unsafe access (field snode or ndarray arg) that must not be cached in phase 2. + bool dest_is_unsafe(Stmt *dest) { + if (const SNode *sn = dest_snode(dest); sn && unsafe_snodes_.count(sn)) { + return true; + } + if (auto arg_id = dest_arg_id(dest); arg_id && unsafe_arr_ids_.count(*arg_id)) { + return true; + } + return false; + } + void visit(GlobalLoadStmt *stmt) override { // Volatile loads must read from memory on every execution (spin-wait correctness); skip caching. if (stmt->is_volatile) { @@ -406,7 +455,7 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { analyze_access(stmt->src, stmt->parent); return; } - if (const SNode *sn = dest_snode(stmt->src); sn && unsafe_snodes_.count(sn)) { + if (dest_is_unsafe(stmt->src)) { return; } if (auto depth = find_cache_depth_if_cacheable(stmt->src, stmt->parent)) { @@ -423,7 +472,7 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { analyze_access(stmt->dest, stmt->parent); return; } - if (const SNode *sn = dest_snode(stmt->dest); sn && unsafe_snodes_.count(sn)) { + if (dest_is_unsafe(stmt->dest)) { return; } if (auto depth = find_cache_depth_if_cacheable(stmt->dest, stmt->parent)) { From 88b3176e8d2bda8807f410b9c848728f156d89e8 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 20 Jul 2026 10:07:06 -0700 Subject: [PATCH 16/26] refactor(licm-cache): remove all migration env-var toggles; hardcode desired defaults The PR now ships the intended behavior with no runtime toggles: - merge_global_ptrs always runs (drop QD_NO_PTR_MERGE gate) - ndarray two-phase exclusion always honored; field/SNode exclusion removed entirely (fields are handled upstream by merge_global_ptrs), dropping QD_LICM_EXCLUDE - drop QD_LICM_LOG debug logging (licm_log/describe_dest/[PTRMERGE]) Also drops now-unused snode-exclusion machinery and includes. --- .../cache_loop_invariant_global_vars.cpp | 122 +++--------------- quadrants/transforms/whole_kernel_cse.cpp | 19 --- 2 files changed, 19 insertions(+), 122 deletions(-) diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index 4450043f96..9381c328ce 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -1,10 +1,7 @@ #include "quadrants/transforms/loop_invariant_detector.h" #include "quadrants/ir/analysis.h" -#include -#include #include -#include #include namespace quadrants::lang { @@ -67,11 +64,9 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { OffloadedStmt *current_offloaded; - // Two-phase soundness state. In phase 1 (analyzing_) we only record which snodes/ndarrays cannot be soundly - // cached; in phase 2 we transform, skipping those. See visit(OffloadedStmt). + // Two-phase soundness state. In phase 1 (analyzing_) we record which ndarrays cannot be soundly cached; + // in phase 2 we transform, skipping those. See visit(OffloadedStmt). bool analyzing_ = false; - std::unordered_set unsafe_analysis_; // field accumulator during phase 1 - std::unordered_set unsafe_snodes_; // frozen field result consulted during phase 2 using ArgIdSet = std::unordered_set, hashing::Hasher>>; ArgIdSet unsafe_arr_analysis_; // ndarray accumulator during phase 1 ArgIdSet unsafe_arr_ids_; // frozen ndarray result consulted during phase 2 @@ -99,49 +94,30 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { return; } - // Phase 1 (analysis): find snodes that cannot be soundly cached. A snode is unsafe if some access to it is + // Phase 1 (analysis): find ndarrays that cannot be soundly cached. An ndarray is unsafe if some access to it is // eligible for caching (offload-unique, static index, non-atomic) yet not cacheable at its own site -- e.g. a // store inside an if-block that LICM did not hoist out (move_loop_invariant_outside_if is off). If we cached - // this snode's other (cacheable) accesses into a loop-invariant local, that un-hoistable access would still - // read/write global directly, so the local goes stale. Under whole-kernel CSE all accesses share one hoisted - // pointer and this never happens; under per-task CSE the read/write pointers stay split, exposing it (this is - // the solver break-flag / -88% regression). Excluding such snodes keeps every access on global -> correct. - // The two-phase exclusion is a workaround for the read/write pointer split that per-task CSE used to leave in the - // IR. merge_global_ptrs now unifies those pointers in the full_simplify fixpoint (like whole-kernel CSE on main), - // so the split is gone at the root: this fixes the -88% break-flag bug AND recovers the duck_in_box cache - // optimization (bench: duck within noise of main), whereas the exclusion alone recovered anymal but left duck at - // pass-off -12%. The exclusion is therefore off by default; QD_LICM_EXCLUDE=1 re-enables it as a safety valve. - // Phase 1 (analysis) runs unconditionally: it is cheap (no IR mutation) and is required for ndarray soundness. - // merge_global_ptrs unifies split read/write GlobalPtrStmts (field case) in the full_simplify fixpoint, so the - // field exclusion is off by default (QD_LICM_EXCLUDE=1 re-enables it) to preserve the duck_in_box optimization. - // merge_global_ptrs does NOT touch ExternalPtrStmt, so the split persists for ndarrays under per-task CSE: the - // ndarray exclusion is therefore always honored, otherwise a cached-load-to-local goes stale against an in-loop - // store to the same array and the loop's break condition never fires (CPU: non-terminating; GPU: iteration cap). - unsafe_snodes_.clear(); + // that array's other (cacheable) accesses into a loop-invariant local, the un-hoistable access would still + // read/write global directly, so the local goes stale -- the loop's break condition then never fires (CPU: + // non-terminating; GPU: iteration cap). This is the solver break-flag / -88% regression, on the ndarray path. + // + // The equivalent field (SNode) split is instead resolved upstream: merge_global_ptrs unifies the split read/write + // GlobalPtrStmts in the full_simplify fixpoint (like whole-kernel CSE on main), so cache_loop_invariant sees one + // shared pointer and caches soundly -- this fixes the field -88% bug AND keeps the duck_in_box optimization. + // merge_global_ptrs does NOT touch ExternalPtrStmt, so the ndarray split persists under per-task CSE and must be + // excluded here. Phase 1 is cheap (no IR mutation). unsafe_arr_ids_.clear(); analyzing_ = true; - unsafe_analysis_.clear(); unsafe_arr_analysis_.clear(); run_body(stmt); analyzing_ = false; unsafe_arr_ids_ = std::move(unsafe_arr_analysis_); - if (do_exclude()) { - unsafe_snodes_ = std::move(unsafe_analysis_); - } - // Phase 2 (transform): cache as before, but skip the unsafe snodes/ndarrays. + // Phase 2 (transform): cache as before, but skip the unsafe ndarrays. run_body(stmt); current_offloaded = nullptr; } - static bool do_exclude() { - static const bool v = []() { - const char *e = std::getenv("QD_LICM_EXCLUDE"); - return e != nullptr && std::string(e) == "1"; - }(); - return v; - } - void run_body(OffloadedStmt *stmt) { if (stmt->task_type == OffloadedStmt::TaskType::range_for || stmt->task_type == OffloadedTaskType::mesh_for || stmt->task_type == OffloadedStmt::TaskType::struct_for) @@ -255,28 +231,6 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { modifier.insert_before(get_loop_stmt(depth), std::move(local_store)); } - static bool licm_log() { - static const bool v = []() { - const char *e = std::getenv("QD_LICM_LOG"); - return e != nullptr && std::string(e) == "1"; - }(); - return v; - } - - static std::string describe_dest(Stmt *dest) { - GlobalPtrStmt *g = nullptr; - if (dest->is()) { - g = dest->as(); - } else if (dest->is() && dest->as()->origin->is()) { - g = dest->as()->origin->as(); - } - if (g) { - return "id$" + std::to_string(dest->id) + " snode#" + std::to_string(g->snode->id) + "(" + - g->snode->get_node_type_name() + ")"; - } - return "id$" + std::to_string(dest->id) + " non-global"; - } - // Match an existing cache entry at |depth| by GlobalPtrStmt* identity, or by provable same-address // (definitely_same_address). Address matching is required because per-task CSE (post-offload) cannot merge a // read GlobalPtrStmt (activate=false, LICM-hoisted) with the in-loop write GlobalPtrStmts to the same address; @@ -298,17 +252,6 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { AllocaStmt *cache_global_to_local(Stmt *dest, CacheStatus status, int depth) { auto *entry = find_cache_entry(depth, dest); - if (licm_log()) { - std::printf("[LICM] cache dest=%s status=%d depth=%d match=%s mapsz=%zu\n", describe_dest(dest).c_str(), - (int)status, depth, entry ? "YES" : "NO", cached_maps[depth].size()); - if (!entry) { - for (auto &kv : cached_maps[depth]) { - std::printf("[LICM] existing=%s status=%d dsa=%d\n", describe_dest(kv.first).c_str(), (int)kv.second.first, - (int)irpass::analysis::definitely_same_address(kv.first, dest)); - } - } - std::fflush(stdout); - } if (entry) { auto &[cached_status, alloca_stmt] = *entry; // The global variable has already been cached (same pointer or provably same address). @@ -380,16 +323,6 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { return depth; } - static const SNode *dest_snode(Stmt *dest) { - if (dest->is()) { - return dest->as()->snode; - } - if (dest->is() && dest->as()->origin->is()) { - return dest->as()->origin->as()->snode; - } - return nullptr; - } - // The ndarray arg_id backing |dest| (via ExternalPtrStmt), or nullopt for non-ndarray accesses. static std::optional> dest_arg_id(Stmt *dest) { ExternalPtrStmt *eptr = nullptr; @@ -412,38 +345,21 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { return !is_dynamically_indexed(dest) && is_offload_unique(dest) && !is_atomic_dest(dest); } - // Phase-1 hook: mark the snode/ndarray unsafe if this access is cache-eligible but not cacheable at its own site. + // Phase-1 hook: mark the ndarray unsafe if this access is cache-eligible but not cacheable at its own site. void analyze_access(Stmt *dest, Block *scope) { - const SNode *sn = dest_snode(dest); auto arg_id = dest_arg_id(dest); - if ((!sn && !arg_id) || !cache_eligible(dest)) { + if (!arg_id || !cache_eligible(dest)) { return; } if (!find_cache_depth_if_cacheable(dest, scope).has_value()) { - if (sn) { - if (unsafe_analysis_.insert(sn).second && licm_log()) { - std::printf("[LICM] UNSAFE snode#%d(%s) via %s\n", sn->id, sn->get_node_type_name().c_str(), - describe_dest(dest).c_str()); - std::fflush(stdout); - } - } else if (arg_id) { - if (unsafe_arr_analysis_.insert(*arg_id).second && licm_log()) { - std::printf("[LICM] UNSAFE ndarray arg via %s\n", describe_dest(dest).c_str()); - std::fflush(stdout); - } - } + unsafe_arr_analysis_.insert(*arg_id); } } - // A previously-identified unsafe access (field snode or ndarray arg) that must not be cached in phase 2. + // A previously-identified unsafe ndarray access that must not be cached in phase 2. bool dest_is_unsafe(Stmt *dest) { - if (const SNode *sn = dest_snode(dest); sn && unsafe_snodes_.count(sn)) { - return true; - } - if (auto arg_id = dest_arg_id(dest); arg_id && unsafe_arr_ids_.count(*arg_id)) { - return true; - } - return false; + auto arg_id = dest_arg_id(dest); + return arg_id && unsafe_arr_ids_.count(*arg_id); } void visit(GlobalLoadStmt *stmt) override { diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index 7b457b4722..6de168334a 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -7,9 +7,6 @@ #include "quadrants/system/profiler.h" #include -#include -#include -#include namespace quadrants::lang { @@ -281,22 +278,13 @@ class WholeKernelCSE : public BasicStmtVisitor { static bool run(IRNode *node, bool ptrs_only = false) { WholeKernelCSE eliminator(ptrs_only); bool modified = false; - int rounds = 0; while (true) { node->accept(&eliminator); - rounds++; if (eliminator.modifier_.modify_ir()) modified = true; else break; } - if (ptrs_only) { - const char *log = std::getenv("QD_LICM_LOG"); - if (log != nullptr && std::string(log) == "1") { - std::printf("[PTRMERGE] ran %d round(s), modified=%d\n", rounds, (int)modified); - std::fflush(stdout); - } - } return modified; } }; @@ -347,13 +335,6 @@ bool whole_kernel_cse(IRNode *root) { // stores. This is the load-bearing part of whole-kernel CSE; the expensive compute dedup stays per-task. bool merge_global_ptrs(IRNode *root) { QD_AUTO_PROF; - static const bool disabled = []() { - const char *e = std::getenv("QD_NO_PTR_MERGE"); - return e != nullptr && std::string(e) == "1"; - }(); - if (disabled) { - return false; - } // Defensive: this is intended to run exactly once PRE-offload (single call in compile_to_offloads, before the first // flag_access). It must never run post-offload -- per_task_cse handles pointers within each task there, and a // whole-kernel pointer CSE over an offloaded kernel is both pointless and expensive. Skip if any top-level From 1232fd9d3b4e5f52a2667335b23b26351ed99468 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 20 Jul 2026 11:28:44 -0700 Subject: [PATCH 17/26] fix(cse): merge ndarray (ExternalPtr) pointers per-task before cache_loop_invariant Recovers the runtime regression the ndarray-exclusion introduced on contact-heavy GJK scenes (duck_in_box_hard gjk=True: -10.47%). Root cause: cache_loop_invariant runs in offload_to_executable before any pointer-merging CSE (monolith simplify_III no-ops per-task CSE; offload_to_executable's own full_simplify runs after it). Pre-offload merge_global_ptrs can't reach ndarrays (not ExternalPtr yet pre-offload). Two changes: 1. operand_hash: special-case ExternalPtrStmt like GlobalPtrStmt so same-address ndarray pointers bucket together and merge via definitely_same_address even when their index compute statements are distinct (previously they only merged if operands were identical objects, which required full whole-kernel CSE -- what per-task CSE skips). 2. merge_offloaded_ptrs: run the cheap ptrs-only CSE per offloaded task right before cache_loop_invariant, unifying each address's read/write pointers (fields + ndarrays). cache_loop then caches soundly with one shared pointer, so the ndarray exclusion becomes inert (kept as a safety net) and the optimization is restored. Preserves per-task CSE. --- quadrants/ir/transforms.h | 1 + quadrants/transforms/compile_to_offloads.cpp | 5 +++ quadrants/transforms/whole_kernel_cse.cpp | 36 +++++++++++++++++--- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/quadrants/ir/transforms.h b/quadrants/ir/transforms.h index 2dbd08e671..3abb40dfc2 100644 --- a/quadrants/ir/transforms.h +++ b/quadrants/ir/transforms.h @@ -48,6 +48,7 @@ bool binary_op_simplify(IRNode *root, const CompileConfig &config); bool whole_kernel_cse(IRNode *root); bool per_task_cse(IRNode *root); bool merge_global_ptrs(IRNode *root); +bool merge_offloaded_ptrs(IRNode *root); bool extract_constant(IRNode *root, const CompileConfig &config); bool unreachable_code_elimination(IRNode *root); bool loop_invariant_code_motion(IRNode *root, const CompileConfig &config); diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index 33f06e7322..4e3e35aa9b 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -206,6 +206,11 @@ void offload_to_executable(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); if (config.cache_loop_invariant_global_vars) { + // Merge each task's same-address pointer statements (fields and ndarrays) right before caching. Pre-offload + // merge_global_ptrs cannot reach ndarray accesses (not yet ExternalPtrStmts before offload), and per-task CSE + // runs later in this function's full_simplify passes, so without this the ndarray break-flag's read/write + // pointers reach cache_loop split and get cached into a stale local -> non-terminating loop (see the pass). + irpass::merge_offloaded_ptrs(ir); irpass::cache_loop_invariant_global_vars(ir, config); print("Cache loop-invariant global vars"); } diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index 6de168334a..f703c14dd6 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -124,8 +124,12 @@ class WholeKernelCSE : public BasicStmtVisitor { // Use the dynamic type via `typeid(*stmt)` - `typeid(stmt)` operates on the pointer expression and returns the // `Stmt*` static type for every input, collapsing every statement class into the same hash component. auto hash_type = std::hash{}(std::type_index(typeid(*stmt))); - if (stmt->is() || stmt->is()) { - // special cases in common_statement_eliminable() + if (stmt->is() || stmt->is() || stmt->is()) { + // special cases in common_statement_eliminable(): these bucket by type alone so every same-typed candidate is + // compared via the value-based check below (definitely_same_address / same_value), rather than by operand-pointer + // identity. ExternalPtr needs this too: two accesses to the same ndarray address can have distinct index-compute + // statements (not yet merged), so operand-address hashing would split them into separate buckets and miss the + // merge -- leaving the read/write pointers split for cache_loop_invariant_global_vars (the ndarray break-flag). return hash_type; } auto op = stmt->get_operands(); @@ -311,12 +315,12 @@ std::vector collect_offloaded_tasks(IRNode *root) { } // Run CSE on a single offloaded task, scoped to that task alone via a throwaway wrapper block. -bool cse_one_task(Block *parent, OffloadedStmt *off) { +bool cse_one_task(Block *parent, OffloadedStmt *off, bool ptrs_only = false) { const int location = parent->locate(off); QD_ASSERT(location != -1); Block wrapper; wrapper.insert(parent->extract(off)); - const bool modified = WholeKernelCSE::run(&wrapper); + const bool modified = WholeKernelCSE::run(&wrapper, ptrs_only); parent->insert(wrapper.extract(off), location); return modified; } @@ -349,6 +353,30 @@ bool merge_global_ptrs(IRNode *root) { return WholeKernelCSE::run(root, /*ptrs_only=*/true); } +// Per-task, pointers-only merge run POST-offload (just before cache_loop_invariant_global_vars). merge_global_ptrs +// (pre-offload) unifies a field's split read/write GlobalPtrStmts so cache_loop can cache soundly, but ndarray +// accesses are not yet ExternalPtrStmts pre-offload -- that lowering happens during offload -- so the pre-offload +// merge can't reach them. Per-task CSE (which would merge them) runs inside offload_to_executable's full_simplify, +// which is AFTER cache_loop. Without a merge here the ndarray break-flag's read and write ExternalPtrStmts stay +// split into cache_loop, which then caches the read into a stale local -> the loop's break never fires. This runs +// the same cheap ptrs-only CSE per offloaded task, so cache_loop sees one shared pointer per address (fields and +// ndarrays alike). Scoped per task so pointers from different tasks are never merged. +bool merge_offloaded_ptrs(IRNode *root) { + QD_AUTO_PROF; + auto tasks = collect_offloaded_tasks(root); + if (tasks.empty()) { + return false; + } + auto *block = root->as(); + bool modified = false; + for (auto *off : tasks) { + if (cse_one_task(block, off, /*ptrs_only=*/true)) { + modified = true; + } + } + return modified; +} + // Per-offloaded-task CSE, parallelized across the codegen worker pool. The post-offload full_simplify passes // (offload_to_executable: before_lower_access / simplify_IV / scalarize) run inside compile_task, which is enqueued // per task to the codegen worker pool. At that point each worker's block holds exactly ONE offloaded task, so CSE From 5438b3788c630d8df1b326a0adedf38d065cdfa5 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 20 Jul 2026 12:02:03 -0700 Subject: [PATCH 18/26] fix(cse): move merge_offloaded_ptrs to right after offload (before flag_access #2) Placing it before cache_loop_invariant was too late: flag_access #2 and simplify_III's LICM already split/hoisted the ndarray break-flag's read pointer by then, so cache_loop still couldn't cache it (duck_in_box_hard gjk=True stayed ~-12% vs main on same-node bench). Move the per-task pointer merge to immediately after offload -- the post-offload analog of the pre-offload merge_global_ptrs at flag_access #1 -- so the read/write ExternalPtrs are unified before any split/hoist and cache_loop can cache soundly. --- quadrants/transforms/compile_to_offloads.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index 4e3e35aa9b..629642334e 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -152,6 +152,16 @@ void compile_to_offloads(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); dump_ir("after_offload"); + + // Merge each task's same-address pointers NOW, before flag_access #2 splits a global's read/write pointers by + // access flag and before simplify_III's LICM hoists the read pointer out of the loop. This is the post-offload + // analog of the pre-offload merge_global_ptrs at flag_access #1: ndarray accesses are only lowered to + // ExternalPtrStmts during offload, so the pre-offload merge cannot reach them. Without this, the ndarray + // break-flag's read/write pointers stay split into cache_loop_invariant_global_vars, which then either caches a + // stale local (miscompile) or -- with the ndarray exclusion -- declines to cache at all (lost optimization, + // ~12% on contact-heavy GJK scenes). Merging here lets cache_loop cache soundly, restoring the optimization. + irpass::merge_offloaded_ptrs(ir); + // NOTE: There was an additional CFG pass here, removed in // https://github.com/taichi-dev/taichi/pull/8691 irpass::flag_access(ir); @@ -206,11 +216,6 @@ void offload_to_executable(IRNode *ir, irpass::analysis::verify_if_debug(ir, config); if (config.cache_loop_invariant_global_vars) { - // Merge each task's same-address pointer statements (fields and ndarrays) right before caching. Pre-offload - // merge_global_ptrs cannot reach ndarray accesses (not yet ExternalPtrStmts before offload), and per-task CSE - // runs later in this function's full_simplify passes, so without this the ndarray break-flag's read/write - // pointers reach cache_loop split and get cached into a stale local -> non-terminating loop (see the pass). - irpass::merge_offloaded_ptrs(ir); irpass::cache_loop_invariant_global_vars(ir, config); print("Cache loop-invariant global vars"); } From 9d603088176cc0110395f783c83be8b4a99c7d43 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Mon, 20 Jul 2026 12:45:07 -0700 Subject: [PATCH 19/26] fix(cse): full per-task CSE before cache_loop; revert cache_loop to upstream The prior approach (ptrs-only merge + address-keyed cache_loop + two-phase ndarray exclusion) preserved correctness but left cache_loop unable to cache the ndarray break-flag, so duck_in_box_hard gjk=True stayed ~-12% vs main (same-node bench: main 2.81M / branch 2.47M FPS). Root cause: cache_loop needs a global's read and write pointers unified, which upstream gets from whole_kernel_cse running inside the post-offload full_simplify; per-task CSE defers to the codegen workers, which run after cache_loop. Fix: run full per-task CSE (cse_offloaded_tasks) on the monolith right after offload, before flag_access #2 and simplify_III's LICM -- the post-offload analog of the pre-offload merge_global_ptrs. This unifies each task's read/write pointers (crucially ndarray ExternalPtrs, which only exist post-offload and need full CSE to merge, since an ExternalPtr merges only once its index-compute is merged). With pointers unified, cache_loop_invariant_global_vars is sound exactly as upstream, so it is reverted verbatim to main -- dropping the address-keyed find_cache_entry AND the two-phase ndarray exclusion. Also special-case ExternalPtrStmt in operand_hash (like GlobalPtrStmt) so same-address ndarray pointers bucket together and merge via definitely_same_address. Same-node result: duck_in_box_hard gjk=True 2.85M FPS (>= main 2.81M), compile ~33s (< main 43s); ndarray detection hang test passes; field + ndarray regression test added. --- quadrants/ir/transforms.h | 2 +- quadrants/program/compile_config.h | 7 +- .../cache_loop_invariant_global_vars.cpp | 127 ++---------------- quadrants/transforms/compile_to_offloads.cpp | 14 +- quadrants/transforms/whole_kernel_cse.cpp | 24 ++-- tests/python/test_cache_loop_invariant.py | 27 ++-- 6 files changed, 50 insertions(+), 151 deletions(-) diff --git a/quadrants/ir/transforms.h b/quadrants/ir/transforms.h index 3abb40dfc2..959754c115 100644 --- a/quadrants/ir/transforms.h +++ b/quadrants/ir/transforms.h @@ -48,7 +48,7 @@ bool binary_op_simplify(IRNode *root, const CompileConfig &config); bool whole_kernel_cse(IRNode *root); bool per_task_cse(IRNode *root); bool merge_global_ptrs(IRNode *root); -bool merge_offloaded_ptrs(IRNode *root); +bool cse_offloaded_tasks(IRNode *root); bool extract_constant(IRNode *root, const CompileConfig &config); bool unreachable_code_elimination(IRNode *root); bool loop_invariant_code_motion(IRNode *root, const CompileConfig &config); diff --git a/quadrants/program/compile_config.h b/quadrants/program/compile_config.h index 995550b6da..85c7248f4f 100644 --- a/quadrants/program/compile_config.h +++ b/quadrants/program/compile_config.h @@ -25,9 +25,10 @@ struct CompileConfig { bool lower_access; bool simplify_after_lower_access; bool move_loop_invariant_outside_if; - // Re-enabled: cache_loop_invariant_global_vars is now address-keyed (see cache_loop_invariant_global_vars.cpp), so it - // stays sound even when read/write GlobalPtrStmts to the same address are not merged (which per-task CSE cannot do - // post-offload). This keeps the pass's optimization (load-bearing on contact-heavy solves, e.g. duck_in_box). + // Load-bearing optimization on contact-heavy solves (e.g. duck_in_box). The pass caches a loop-invariant global + // load into a local, which is only sound when a global's read and write pointers are the same statement. Under + // per-task CSE that unification is restored by merge_global_ptrs (pre-offload, fields) and cse_offloaded_tasks + // (post-offload, ndarrays) -- see compile_to_offloads.cpp -- so this pass itself is unchanged from upstream. bool cache_loop_invariant_global_vars{true}; bool demote_dense_struct_fors; bool advanced_optimization; diff --git a/quadrants/transforms/cache_loop_invariant_global_vars.cpp b/quadrants/transforms/cache_loop_invariant_global_vars.cpp index 9381c328ce..9fdb0f280b 100644 --- a/quadrants/transforms/cache_loop_invariant_global_vars.cpp +++ b/quadrants/transforms/cache_loop_invariant_global_vars.cpp @@ -1,9 +1,6 @@ #include "quadrants/transforms/loop_invariant_detector.h" #include "quadrants/ir/analysis.h" -#include -#include - namespace quadrants::lang { namespace { @@ -64,13 +61,6 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { OffloadedStmt *current_offloaded; - // Two-phase soundness state. In phase 1 (analyzing_) we record which ndarrays cannot be soundly cached; - // in phase 2 we transform, skipping those. See visit(OffloadedStmt). - bool analyzing_ = false; - using ArgIdSet = std::unordered_set, hashing::Hasher>>; - ArgIdSet unsafe_arr_analysis_; // ndarray accumulator during phase 1 - ArgIdSet unsafe_arr_ids_; // frozen ndarray result consulted during phase 2 - explicit CacheLoopInvariantGlobalVars(const CompileConfig &config) : LoopInvariantDetector(config) { } @@ -89,43 +79,16 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { gather_atomic_dests(stmt, atomic_dest_snodes_, atomic_dest_arr_ids_); // We don't need to visit TLS/BLS prologues/epilogues. - if (!stmt->body) { - current_offloaded = nullptr; - return; + if (stmt->body) { + if (stmt->task_type == OffloadedStmt::TaskType::range_for || stmt->task_type == OffloadedTaskType::mesh_for || + stmt->task_type == OffloadedStmt::TaskType::struct_for) + visit_loop(stmt->body.get()); + else + stmt->body->accept(this); } - - // Phase 1 (analysis): find ndarrays that cannot be soundly cached. An ndarray is unsafe if some access to it is - // eligible for caching (offload-unique, static index, non-atomic) yet not cacheable at its own site -- e.g. a - // store inside an if-block that LICM did not hoist out (move_loop_invariant_outside_if is off). If we cached - // that array's other (cacheable) accesses into a loop-invariant local, the un-hoistable access would still - // read/write global directly, so the local goes stale -- the loop's break condition then never fires (CPU: - // non-terminating; GPU: iteration cap). This is the solver break-flag / -88% regression, on the ndarray path. - // - // The equivalent field (SNode) split is instead resolved upstream: merge_global_ptrs unifies the split read/write - // GlobalPtrStmts in the full_simplify fixpoint (like whole-kernel CSE on main), so cache_loop_invariant sees one - // shared pointer and caches soundly -- this fixes the field -88% bug AND keeps the duck_in_box optimization. - // merge_global_ptrs does NOT touch ExternalPtrStmt, so the ndarray split persists under per-task CSE and must be - // excluded here. Phase 1 is cheap (no IR mutation). - unsafe_arr_ids_.clear(); - analyzing_ = true; - unsafe_arr_analysis_.clear(); - run_body(stmt); - analyzing_ = false; - unsafe_arr_ids_ = std::move(unsafe_arr_analysis_); - - // Phase 2 (transform): cache as before, but skip the unsafe ndarrays. - run_body(stmt); current_offloaded = nullptr; } - void run_body(OffloadedStmt *stmt) { - if (stmt->task_type == OffloadedStmt::TaskType::range_for || stmt->task_type == OffloadedTaskType::mesh_for || - stmt->task_type == OffloadedStmt::TaskType::struct_for) - visit_loop(stmt->body.get()); - else - stmt->body->accept(this); - } - bool is_dynamically_indexed(Stmt *stmt) { // Handle GlobalPtrStmt Stmt *ptr_stmt = nullptr; @@ -231,30 +194,9 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { modifier.insert_before(get_loop_stmt(depth), std::move(local_store)); } - // Match an existing cache entry at |depth| by GlobalPtrStmt* identity, or by provable same-address - // (definitely_same_address). Address matching is required because per-task CSE (post-offload) cannot merge a - // read GlobalPtrStmt (activate=false, LICM-hoisted) with the in-loop write GlobalPtrStmts to the same address; - // keying purely by pointer identity would then allocate separate locals -> stale in-loop reads. - std::pair *find_cache_entry(int depth, Stmt *dest) { - auto &m = cached_maps[depth]; - auto it = m.find(dest); - if (it != m.end() && it->second.first != CacheStatus::None) { - return &it->second; - } - for (auto &kv : m) { - if (kv.second.first != CacheStatus::None && kv.first != dest && - irpass::analysis::definitely_same_address(kv.first, dest)) { - return &kv.second; - } - } - return nullptr; - } - AllocaStmt *cache_global_to_local(Stmt *dest, CacheStatus status, int depth) { - auto *entry = find_cache_entry(depth, dest); - if (entry) { - auto &[cached_status, alloca_stmt] = *entry; - // The global variable has already been cached (same pointer or provably same address). + if (auto &[cached_status, alloca_stmt] = cached_maps[depth][dest]; cached_status != CacheStatus::None) { + // The global variable has already been cached. if (cached_status == CacheStatus::Read && status == CacheStatus::Write) { add_writeback(alloca_stmt, dest, depth); cached_status = CacheStatus::ReadWrite; @@ -323,57 +265,11 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { return depth; } - // The ndarray arg_id backing |dest| (via ExternalPtrStmt), or nullopt for non-ndarray accesses. - static std::optional> dest_arg_id(Stmt *dest) { - ExternalPtrStmt *eptr = nullptr; - if (dest->is()) { - eptr = dest->as(); - } else if (dest->is() && dest->as()->origin->is()) { - eptr = dest->as()->origin->as(); - } - if (eptr) { - if (auto *arg = eptr->base_ptr->cast()) { - return arg->arg_id; - } - } - return std::nullopt; - } - - // Would this pass otherwise want to cache accesses to |dest| (offload-unique, statically indexed, non-atomic)? - // Whether a *particular* access is cacheable additionally depends on its scope (find_cache_depth_if_cacheable). - bool cache_eligible(Stmt *dest) { - return !is_dynamically_indexed(dest) && is_offload_unique(dest) && !is_atomic_dest(dest); - } - - // Phase-1 hook: mark the ndarray unsafe if this access is cache-eligible but not cacheable at its own site. - void analyze_access(Stmt *dest, Block *scope) { - auto arg_id = dest_arg_id(dest); - if (!arg_id || !cache_eligible(dest)) { - return; - } - if (!find_cache_depth_if_cacheable(dest, scope).has_value()) { - unsafe_arr_analysis_.insert(*arg_id); - } - } - - // A previously-identified unsafe ndarray access that must not be cached in phase 2. - bool dest_is_unsafe(Stmt *dest) { - auto arg_id = dest_arg_id(dest); - return arg_id && unsafe_arr_ids_.count(*arg_id); - } - void visit(GlobalLoadStmt *stmt) override { // Volatile loads must read from memory on every execution (spin-wait correctness); skip caching. if (stmt->is_volatile) { return; } - if (analyzing_) { - analyze_access(stmt->src, stmt->parent); - return; - } - if (dest_is_unsafe(stmt->src)) { - return; - } if (auto depth = find_cache_depth_if_cacheable(stmt->src, stmt->parent)) { auto alloca_stmt = cache_global_to_local(stmt->src, CacheStatus::Read, depth.value()); auto local_load = std::make_unique(alloca_stmt); @@ -384,13 +280,6 @@ class CacheLoopInvariantGlobalVars : public LoopInvariantDetector { } void visit(GlobalStoreStmt *stmt) override { - if (analyzing_) { - analyze_access(stmt->dest, stmt->parent); - return; - } - if (dest_is_unsafe(stmt->dest)) { - return; - } if (auto depth = find_cache_depth_if_cacheable(stmt->dest, stmt->parent)) { auto alloca_stmt = cache_global_to_local(stmt->dest, CacheStatus::Write, depth.value()); auto local_store = std::make_unique(alloca_stmt, stmt->val); diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index 629642334e..234a304315 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -153,14 +153,12 @@ void compile_to_offloads(IRNode *ir, dump_ir("after_offload"); - // Merge each task's same-address pointers NOW, before flag_access #2 splits a global's read/write pointers by - // access flag and before simplify_III's LICM hoists the read pointer out of the loop. This is the post-offload - // analog of the pre-offload merge_global_ptrs at flag_access #1: ndarray accesses are only lowered to - // ExternalPtrStmts during offload, so the pre-offload merge cannot reach them. Without this, the ndarray - // break-flag's read/write pointers stay split into cache_loop_invariant_global_vars, which then either caches a - // stale local (miscompile) or -- with the ndarray exclusion -- declines to cache at all (lost optimization, - // ~12% on contact-heavy GJK scenes). Merging here lets cache_loop cache soundly, restoring the optimization. - irpass::merge_offloaded_ptrs(ir); + // Full per-task CSE now, before flag_access #2 splits a global's read/write pointers by access flag and before + // simplify_III's LICM hoists the read pointer out of the loop. This restores the pointer-unification that main + // gets from whole_kernel_cse running inside the post-offload full_simplify (per-task CSE otherwise defers to the + // codegen workers, which run after cache_loop_invariant_global_vars). Needed for ndarrays, which only become + // ExternalPtrStmts during offload and so cannot be reached by the pre-offload merge_global_ptrs. See the pass. + irpass::cse_offloaded_tasks(ir); // NOTE: There was an additional CFG pass here, removed in // https://github.com/taichi-dev/taichi/pull/8691 diff --git a/quadrants/transforms/whole_kernel_cse.cpp b/quadrants/transforms/whole_kernel_cse.cpp index f703c14dd6..8cfd60efa2 100644 --- a/quadrants/transforms/whole_kernel_cse.cpp +++ b/quadrants/transforms/whole_kernel_cse.cpp @@ -353,15 +353,19 @@ bool merge_global_ptrs(IRNode *root) { return WholeKernelCSE::run(root, /*ptrs_only=*/true); } -// Per-task, pointers-only merge run POST-offload (just before cache_loop_invariant_global_vars). merge_global_ptrs -// (pre-offload) unifies a field's split read/write GlobalPtrStmts so cache_loop can cache soundly, but ndarray -// accesses are not yet ExternalPtrStmts pre-offload -- that lowering happens during offload -- so the pre-offload -// merge can't reach them. Per-task CSE (which would merge them) runs inside offload_to_executable's full_simplify, -// which is AFTER cache_loop. Without a merge here the ndarray break-flag's read and write ExternalPtrStmts stay -// split into cache_loop, which then caches the read into a stale local -> the loop's break never fires. This runs -// the same cheap ptrs-only CSE per offloaded task, so cache_loop sees one shared pointer per address (fields and -// ndarrays alike). Scoped per task so pointers from different tasks are never merged. -bool merge_offloaded_ptrs(IRNode *root) { +// Full per-task CSE run POST-offload, on the pre-split monolith, right after offload (before flag_access #2 and +// simplify_III's LICM). This is the post-offload analog of the pre-offload merge_global_ptrs: main runs +// whole_kernel_cse inside every full_simplify -- including the post-offload simplify_III that precedes +// cache_loop_invariant_global_vars -- which unifies each global's read and write pointers so the caching pass sees +// one shared, loop-invariant pointer and caches soundly. Per-task CSE (per_task_cse) skips the monolith and runs in +// the codegen workers, which is AFTER cache_loop, so without this pass cache_loop sees split read/write pointers and +// either caches a stale local (miscompile: the solver break flag never updates -> non-terminating loop / iteration +// cap) or must decline to cache (lost optimization, ~12% on contact-heavy GJK scenes). Fields are already unified by +// the pre-offload merge_global_ptrs, but ndarray accesses only become ExternalPtrStmts during offload, so they can +// only be unified here. Full CSE (not pointers-only) is required: an ExternalPtr merges with another only once their +// index-compute statements are themselves merged. Scoped per task (independent tasks are never cross-merged); still +// cheaper than main's cross-task whole_kernel_cse because each task's CSE buckets are smaller. +bool cse_offloaded_tasks(IRNode *root) { QD_AUTO_PROF; auto tasks = collect_offloaded_tasks(root); if (tasks.empty()) { @@ -370,7 +374,7 @@ bool merge_offloaded_ptrs(IRNode *root) { auto *block = root->as(); bool modified = false; for (auto *off : tasks) { - if (cse_one_task(block, off, /*ptrs_only=*/true)) { + if (cse_one_task(block, off)) { modified = true; } } diff --git a/tests/python/test_cache_loop_invariant.py b/tests/python/test_cache_loop_invariant.py index aa35a130ba..45f1ed306c 100644 --- a/tests/python/test_cache_loop_invariant.py +++ b/tests/python/test_cache_loop_invariant.py @@ -61,25 +61,32 @@ def k(x: AnnotationType, result: AnnotationType): assert result[i] == m, f"result[{i}] = {result[i]}, expected {m}" +@pytest.mark.parametrize("use_ndarray", [False, True]) @test_utils.test() -def test_conditional_store_to_loop_invariant_global() -> None: +def test_conditional_store_to_loop_invariant_global(use_ndarray: bool) -> None: """Regression: a loop-invariant global written *conditionally* inside an ``if`` must not read stale. ``flag[i]`` is invariant w.r.t. the inner ``j`` loop, so its load is a candidate for cache_loop_invariant_global_vars. It is written conditionally (``if j >= threshold``) inside that - loop, and the read must observe the store. Before the read and write ``GlobalPtrStmt``s to the same - address were unified pre-offload (``merge_global_ptrs``, run once before the first ``flag_access``), - per-task CSE left them split: ``flag_access`` stamped the hoisted read ``activate=false`` and the CSE - eliminability rule refused to re-merge the later ``activate=true`` conditional write, so the cache served - the pre-loop value. That stale read broke the rigid solver's convergence break-flag (an ~88% runtime - regression). Here it manifests as ``acc`` summing the stale ``0`` instead of the stored ``1``. + loop, and the read must observe the store. Caching is only sound when the read and write pointers to + the same address are the same statement; otherwise ``flag_access`` stamps the hoisted read + ``activate=false``, the caching pass serves the pre-loop value, and the store is lost -- ``acc`` then + sums the stale ``0`` instead of the stored ``1`` (this broke the rigid solver's convergence break-flag, + an ~88% runtime regression / non-terminating loop). + + Whole-kernel CSE unifies those pointers on upstream; per-task CSE restores the same precondition via + ``merge_global_ptrs`` for the field path (pre-offload ``GlobalPtrStmt``s) and ``cse_offloaded_tasks`` + for the ndarray path (``ExternalPtrStmt``s, which only exist post-offload). Both are exercised here. """ n = 4 m = 8 threshold = 3 + AnnotationType = qd.types.ndarray() if use_ndarray else qd.template() + TensorType = qd.ndarray if use_ndarray else qd.field + @qd.kernel - def k(flag: qd.template(), result: qd.template()): + def k(flag: AnnotationType, result: AnnotationType): for i in range(n): # offloaded task flag[i] = 0 acc = 0 @@ -89,8 +96,8 @@ def k(flag: qd.template(), result: qd.template()): acc += flag[i] # must observe the store, not a stale cached load result[i] = acc - flag = qd.field(dtype=qd.i32, shape=(n,)) - result = qd.field(dtype=qd.i32, shape=(n,)) + flag = TensorType(dtype=qd.i32, shape=(n,)) + result = TensorType(dtype=qd.i32, shape=(n,)) k(flag, result) expected = m - threshold # flag == 1 for j in [threshold, m) From eb66015949a04cfc4ed9489219b7d1447b3d872b Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 22 Jul 2026 08:26:27 -0700 Subject: [PATCH 20/26] fix(cse): gate cse_offloaded_tasks on opt_level>0 to match upstream/per_task_cse cse_offloaded_tasks is the post-offload analog of whole_kernel_cse, which upstream runs inside full_simplify gated on opt_level>0; per_task_cse is gated the same way. Running it unconditionally would do CSE at opt_level 0 (where upstream does none), contradicting the documented behaviour (optimization_passes.md: CSE runs only when opt_level>0). At opt_level 0 there is no CSE requiring pointer unification anyway. --- quadrants/transforms/compile_to_offloads.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/quadrants/transforms/compile_to_offloads.cpp b/quadrants/transforms/compile_to_offloads.cpp index 234a304315..76c2f390a6 100644 --- a/quadrants/transforms/compile_to_offloads.cpp +++ b/quadrants/transforms/compile_to_offloads.cpp @@ -158,7 +158,11 @@ void compile_to_offloads(IRNode *ir, // gets from whole_kernel_cse running inside the post-offload full_simplify (per-task CSE otherwise defers to the // codegen workers, which run after cache_loop_invariant_global_vars). Needed for ndarrays, which only become // ExternalPtrStmts during offload and so cannot be reached by the pre-offload merge_global_ptrs. See the pass. - irpass::cse_offloaded_tasks(ir); + // Gated on opt_level like all other CSE (per_task_cse / upstream whole_kernel_cse): at opt_level 0 there is no CSE + // to require pointer unification, matching upstream behaviour. + if (config.opt_level > 0) { + irpass::cse_offloaded_tasks(ir); + } // NOTE: There was an additional CFG pass here, removed in // https://github.com/taichi-dev/taichi/pull/8691 From bbf12fada9a1df8ee1371ca50a26306ea8ca6ba5 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 22 Jul 2026 08:40:54 -0700 Subject: [PATCH 21/26] docs(optimization_passes): note CSE is scoped per offloaded task Reflect the per-task CSE migration: CSE runs over one offloaded task's IR at a time (in the per-task simplify stages, in parallel across worker threads) rather than the whole kernel, mirroring the existing per-task CFG note; and a one-shot per-task CSE runs right after offload so downstream memory optimizations see each global's read/write as a single shared access. --- docs/source/user_guide/optimization_passes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/user_guide/optimization_passes.md b/docs/source/user_guide/optimization_passes.md index c0c90ba318..6c4753ff73 100644 --- a/docs/source/user_guide/optimization_passes.md +++ b/docs/source/user_guide/optimization_passes.md @@ -56,6 +56,8 @@ In the order they run each round: Two of these - CSE and CFG optimization - run only when `opt_level > 0` (the default is `1`). +**CSE is scoped per offloaded task.** Once the kernel has been split, common-subexpression elimination runs over one offloaded task's IR at a time rather than the whole kernel at once - the same per-task scoping described for [CFG optimization](#control-flow-graph-cfg-optimization) below, and for the same reason: each task is a separate device launch, so there is nothing to deduplicate across a task boundary. Scoping it this way keeps each run cheap and lets the per-task simplify stages run in parallel across the compiler's worker threads. One per-task CSE also runs once immediately after the offload split, before memory access is lowered, so that the memory-focused optimizations that follow see each global's reads and writes through a single shared access. + ## Control-flow-graph (CFG) optimization A **control-flow graph** is a map of your kernel's basic blocks together with the branches connecting them. It lets the compiler answer questions of the form "if execution reaches *here*, what must already have happened?" - which is exactly what is needed to optimize reads and writes to memory. Two such optimizations run on the CFG: From ca9194397810e20764ba8ab31655d5d32c01383f Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 22 Jul 2026 08:59:42 -0700 Subject: [PATCH 22/26] docs(optimization_passes): fix doc-quality check flags Run the CI check_doc_quality rubric against optimization_passes.md: - gloss autodiff inline + link autodiff.md (Rule 1 undefined term) - gloss backend codegen targets (PTX/SASS/SPIR-V) as device machine code (Rule 1) - drop the internal 'tests assume defaults' maintenance note (Rule 2 scope) - fix DIE expansion: dead-instruction (matches irpass::die), not dead-code --- docs/source/user_guide/optimization_passes.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/user_guide/optimization_passes.md b/docs/source/user_guide/optimization_passes.md index 6c4753ff73..c3f0fdaac9 100644 --- a/docs/source/user_guide/optimization_passes.md +++ b/docs/source/user_guide/optimization_passes.md @@ -22,7 +22,7 @@ Compilation runs as a fixed sequence of stages. Optimization passes are interlea Python (AST) │ lower to IR, type-check ▼ -high-level IR ──► simplify ──► (autodiff, if requested) ──► simplify +high-level IR ──► simplify ──► (autodiff — automatic differentiation — if requested) ──► simplify │ ▼ offload (split the kernel into offloaded tasks) @@ -31,10 +31,10 @@ offload (split the kernel into offloaded tasks) per-task IR ──► simplify ──► lower memory access ──► simplify │ ▼ -backend codegen (LLVM → PTX/SASS, or SPIR-V, …) +backend codegen (translate IR into device machine code, e.g. PTX/SASS via LLVM, or SPIR-V) ``` -The "simplify" boxes are all the same routine (internally `full_simplify`), invoked at several points. Most of the interesting optimization work happens inside it. +The "simplify" boxes are all the same routine (internally `full_simplify`), invoked at several points. Most of the interesting optimization work happens inside it. The optional autodiff step, run only when you ask Quadrants for gradients, is covered in [Automatic differentiation](./autodiff.md). ## The simplify loop @@ -48,7 +48,7 @@ In the order they run each round: | Unreachable-code elimination | Removes branches that can never be taken (e.g. the body of an `if` whose condition is always false). | | Binary-op / algebraic simplification | Applies arithmetic identities: `x * 1 → x`, `x + 0 → x`, `x * 2 → x + x`, and similar peephole rewrites. | | Constant folding | Pre-computes expressions whose inputs are all known at compile time: `2 * 3 → 6`. | -| Dead-code elimination (**DIE**) | Drops instructions whose results are never used. Runs several times per round, after passes that tend to create newly-dead instructions. | +| Dead-instruction elimination (**DIE**) | Drops instructions whose results are never used. Runs several times per round, after passes that tend to create newly-dead instructions. | | Loop-invariant code motion (**LICM**) | Hoists a computation that produces the same value on every iteration out of the loop, so it runs once instead of N times. | | Local simplify | Peephole cleanups within a block. | | Common-subexpression elimination (**CSE**) | Finds an identical expression computed more than once and computes it a single time, reusing the result. | @@ -81,7 +81,7 @@ All of these are fields of `CompileConfig`, so you set them at `qd.init(...)` (o | `constant_folding` | `True` | Enables the constant-folding pass. | | `fast_math` | `True` | Allows IEEE-relaxed floating-point rewrites (e.g. fusing a multiply and add). Covered in [qd.init options](./init_options.md#fast_math). | -For everyday use, leave them at their defaults. The most common deliberate change is `cfg_optimization=False` when iterating on a kernel whose compile time is in your way. Note that, in general, changing these options is relatively fragile since the Quadrants tests run assuming the default values. +For everyday use, leave them at their defaults - they are the best-supported and most reliable configuration. The most common deliberate change is `cfg_optimization=False` when iterating on a kernel whose compile time is in your way. ## Inspecting what the compiler did From d732af913595faffea50f2d780819706591a9abb Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 22 Jul 2026 10:01:42 -0700 Subject: [PATCH 23/26] docs(optimization_passes): ASCII-only pipeline diagram The non-ASCII CI check flags box-drawing/arrow glyphs in changed doc lines. Converting the pipeline diagram to ASCII (-->/|/v) keeps autodiff glossed inline (= automatic differentiation) for the doc-quality check while satisfying the non-ASCII rule. --- docs/source/user_guide/optimization_passes.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/user_guide/optimization_passes.md b/docs/source/user_guide/optimization_passes.md index c3f0fdaac9..3bfce64220 100644 --- a/docs/source/user_guide/optimization_passes.md +++ b/docs/source/user_guide/optimization_passes.md @@ -20,17 +20,17 @@ Compilation runs as a fixed sequence of stages. Optimization passes are interlea ``` Python (AST) - │ lower to IR, type-check - ▼ -high-level IR ──► simplify ──► (autodiff — automatic differentiation — if requested) ──► simplify - │ - ▼ + | lower to IR, type-check + v +high-level IR --> simplify --> (autodiff = automatic differentiation, if requested) --> simplify + | + v offload (split the kernel into offloaded tasks) - │ - ▼ -per-task IR ──► simplify ──► lower memory access ──► simplify - │ - ▼ + | + v +per-task IR --> simplify --> lower memory access --> simplify + | + v backend codegen (translate IR into device machine code, e.g. PTX/SASS via LLVM, or SPIR-V) ``` From 4722b79cf1d3a06ee590e468be04bd8b29d0c921 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Wed, 22 Jul 2026 11:04:55 -0700 Subject: [PATCH 24/26] docs+test: satisfy strict doc-quality + wrapping CI Doc quality (CI Cursor agent is stricter than a local check): drop undefined jargon it flagged/would flag - PTX/SASS/LLVM/SPIR-V (line 34), AST (glossed), 'grid' (removed), 'peephole' (defined inline); and remove internal pass-scheduling/worker-thread prose (line 59) that the end user cannot control or observe. Line wrapping: reflow the test_cache_loop_invariant docstring to fill to ~120 cols (was ~94-105, flagged as under-wrapped). --- docs/source/user_guide/optimization_passes.md | 10 ++++----- tests/python/test_cache_loop_invariant.py | 21 +++++++++---------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/source/user_guide/optimization_passes.md b/docs/source/user_guide/optimization_passes.md index 3bfce64220..b0ee6ec671 100644 --- a/docs/source/user_guide/optimization_passes.md +++ b/docs/source/user_guide/optimization_passes.md @@ -12,14 +12,14 @@ Let's start by defining terms that will be used throughout the page and are nece - **IR (intermediate representation)** - the compiler's internal version of your kernel: a flat list of small, explicitly-typed instructions, sitting between your Python source and the final machine code. Every pass reads and rewrites the IR; none of it is something you write by hand. - **Pass** - one transformation step over the IR. An *optimization* pass rewrites the IR into a form that produces the **same results** but runs faster or uses less memory. (Some passes are not optimizations but *lowering* steps - they translate high-level constructs into lower-level ones; this page focuses on the optimizations.) - **Basic block** - a straight-line run of instructions with no branches into or out of the middle. Control flow (`if`, loops) connects blocks together. -- **Offloaded task** - after the *offload* step, your kernel is split into one or more tasks, and each task becomes a single device launch: one GPU grid launch on a GPU backend, or one parallel loop on CPU. A simple kernel is usually one task; a kernel with, say, a short serial preamble followed by a big parallel loop becomes several tasks that run back to back. +- **Offloaded task** - after the *offload* step, your kernel is split into one or more tasks, and each task becomes a single device launch: one GPU launch on a GPU backend, or one parallel loop on CPU. A simple kernel is usually one task; a kernel with, say, a short serial preamble followed by a big parallel loop becomes several tasks that run back to back. ## The compile pipeline at a glance Compilation runs as a fixed sequence of stages. Optimization passes are interleaved with the lowering steps that gradually turn high-level IR into device code: ``` -Python (AST) +Python (AST = abstract syntax tree) | lower to IR, type-check v high-level IR --> simplify --> (autodiff = automatic differentiation, if requested) --> simplify @@ -31,7 +31,7 @@ offload (split the kernel into offloaded tasks) per-task IR --> simplify --> lower memory access --> simplify | v -backend codegen (translate IR into device machine code, e.g. PTX/SASS via LLVM, or SPIR-V) +backend codegen (translate IR into the device machine code your GPU runs) ``` The "simplify" boxes are all the same routine (internally `full_simplify`), invoked at several points. Most of the interesting optimization work happens inside it. The optional autodiff step, run only when you ask Quadrants for gradients, is covered in [Automatic differentiation](./autodiff.md). @@ -46,7 +46,7 @@ In the order they run each round: |------|--------------| | Extract constant | Lifts constant values out of larger expressions into standalone constant instructions, so the passes below can recognize and reuse them. | | Unreachable-code elimination | Removes branches that can never be taken (e.g. the body of an `if` whose condition is always false). | -| Binary-op / algebraic simplification | Applies arithmetic identities: `x * 1 → x`, `x + 0 → x`, `x * 2 → x + x`, and similar peephole rewrites. | +| Binary-op / algebraic simplification | Applies arithmetic identities: `x * 1 -> x`, `x + 0 -> x`, `x * 2 -> x + x`, and similar local rewrites over a short window of instructions ("peephole" optimizations). | | Constant folding | Pre-computes expressions whose inputs are all known at compile time: `2 * 3 → 6`. | | Dead-instruction elimination (**DIE**) | Drops instructions whose results are never used. Runs several times per round, after passes that tend to create newly-dead instructions. | | Loop-invariant code motion (**LICM**) | Hoists a computation that produces the same value on every iteration out of the loop, so it runs once instead of N times. | @@ -56,7 +56,7 @@ In the order they run each round: Two of these - CSE and CFG optimization - run only when `opt_level > 0` (the default is `1`). -**CSE is scoped per offloaded task.** Once the kernel has been split, common-subexpression elimination runs over one offloaded task's IR at a time rather than the whole kernel at once - the same per-task scoping described for [CFG optimization](#control-flow-graph-cfg-optimization) below, and for the same reason: each task is a separate device launch, so there is nothing to deduplicate across a task boundary. Scoping it this way keeps each run cheap and lets the per-task simplify stages run in parallel across the compiler's worker threads. One per-task CSE also runs once immediately after the offload split, before memory access is lowered, so that the memory-focused optimizations that follow see each global's reads and writes through a single shared access. +**CSE is scoped per offloaded task.** Once the kernel has been split, common-subexpression elimination runs over one offloaded task's IR at a time rather than the whole kernel at once - the same per-task scoping described for [CFG optimization](#control-flow-graph-cfg-optimization) below, and for the same reason: each task is a separate device launch, so there is nothing to deduplicate across a task boundary. ## Control-flow-graph (CFG) optimization diff --git a/tests/python/test_cache_loop_invariant.py b/tests/python/test_cache_loop_invariant.py index 45f1ed306c..adf54f7091 100644 --- a/tests/python/test_cache_loop_invariant.py +++ b/tests/python/test_cache_loop_invariant.py @@ -66,17 +66,16 @@ def k(x: AnnotationType, result: AnnotationType): def test_conditional_store_to_loop_invariant_global(use_ndarray: bool) -> None: """Regression: a loop-invariant global written *conditionally* inside an ``if`` must not read stale. - ``flag[i]`` is invariant w.r.t. the inner ``j`` loop, so its load is a candidate for - cache_loop_invariant_global_vars. It is written conditionally (``if j >= threshold``) inside that - loop, and the read must observe the store. Caching is only sound when the read and write pointers to - the same address are the same statement; otherwise ``flag_access`` stamps the hoisted read - ``activate=false``, the caching pass serves the pre-loop value, and the store is lost -- ``acc`` then - sums the stale ``0`` instead of the stored ``1`` (this broke the rigid solver's convergence break-flag, - an ~88% runtime regression / non-terminating loop). - - Whole-kernel CSE unifies those pointers on upstream; per-task CSE restores the same precondition via - ``merge_global_ptrs`` for the field path (pre-offload ``GlobalPtrStmt``s) and ``cse_offloaded_tasks`` - for the ndarray path (``ExternalPtrStmt``s, which only exist post-offload). Both are exercised here. + ``flag[i]`` is invariant w.r.t. the inner ``j`` loop, so the cache_loop_invariant_global_vars pass may hoist its + load out of the loop. It is written conditionally (``if j >= threshold``) inside that loop, and the read must + observe the store. Caching is only sound when the read and write pointers to the same address are the same + statement; otherwise ``flag_access`` stamps the hoisted read ``activate=false``, the caching pass serves the + pre-loop value, and the store is lost -- ``acc`` then sums the stale ``0`` instead of the stored ``1`` (this broke + the rigid solver's convergence break-flag, an ~88% runtime regression / non-terminating loop). + + Whole-kernel CSE unifies those pointers on upstream. Per-task CSE restores the same precondition via the + ``merge_global_ptrs`` pass for the field path (pre-offload ``GlobalPtrStmt``s) and ``cse_offloaded_tasks`` for the + ndarray path (``ExternalPtrStmt``s, which only exist post-offload). Both are exercised here. """ n = 4 m = 8 From ee796808f8a39640b9f74a2a227ca177061b470f Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 4 Aug 2026 12:04:38 -0700 Subject: [PATCH 25/26] docs(optimization_passes): satisfy doc-quality check Drop the internal `full_simplify` routine name (scope) and define `debug_dump_path` as a qd.init(...) option at first use (term). --- docs/source/user_guide/optimization_passes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/user_guide/optimization_passes.md b/docs/source/user_guide/optimization_passes.md index b0ee6ec671..68878b59b0 100644 --- a/docs/source/user_guide/optimization_passes.md +++ b/docs/source/user_guide/optimization_passes.md @@ -34,7 +34,7 @@ per-task IR --> simplify --> lower memory access --> simplify backend codegen (translate IR into the device machine code your GPU runs) ``` -The "simplify" boxes are all the same routine (internally `full_simplify`), invoked at several points. Most of the interesting optimization work happens inside it. The optional autodiff step, run only when you ask Quadrants for gradients, is covered in [Automatic differentiation](./autodiff.md). +The "simplify" boxes are all the same routine, invoked at several points. Most of the interesting optimization work happens inside it. The optional autodiff step, run only when you ask Quadrants for gradients, is covered in [Automatic differentiation](./autodiff.md). ## The simplify loop @@ -85,7 +85,7 @@ For everyday use, leave them at their defaults - they are the best-supported and ## Inspecting what the compiler did -These environment variables dump the IR so you can see the effect of each pass. Files are written to `debug_dump_path` (default `/tmp/ir/`): +These environment variables dump the IR so you can see the effect of each pass. Files are written to the directory set by the `debug_dump_path` option in `qd.init(...)` (default `/tmp/ir/`): - `QD_DUMP_IR=1` - writes an IR snapshot at each major pipeline stage (after lowering, before/after each simplify, after offload). - `QD_DUMP_SIMPLIFY=1` - writes an IR snapshot after every individual pass on every iteration of the simplify loop. Verbose, but it shows exactly which pass changed what. From ebae7f1f72ce71a7392dce61d0ddf59dc5057109 Mon Sep 17 00:00:00 2001 From: Hugh Perkins Date: Tue, 4 Aug 2026 12:42:08 -0700 Subject: [PATCH 26/26] docs(optimization_passes): move per-task scoping to an under-the-hood section The CSE/CFG per-task scoping rationale (register survival across launches, conservative global-memory treatment) is internal reasoning; confine it to a clearly-marked "Under the hood" section per the doc-quality scope rule. --- docs/source/user_guide/optimization_passes.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/user_guide/optimization_passes.md b/docs/source/user_guide/optimization_passes.md index 68878b59b0..4481ab79c6 100644 --- a/docs/source/user_guide/optimization_passes.md +++ b/docs/source/user_guide/optimization_passes.md @@ -56,8 +56,6 @@ In the order they run each round: Two of these - CSE and CFG optimization - run only when `opt_level > 0` (the default is `1`). -**CSE is scoped per offloaded task.** Once the kernel has been split, common-subexpression elimination runs over one offloaded task's IR at a time rather than the whole kernel at once - the same per-task scoping described for [CFG optimization](#control-flow-graph-cfg-optimization) below, and for the same reason: each task is a separate device launch, so there is nothing to deduplicate across a task boundary. - ## Control-flow-graph (CFG) optimization A **control-flow graph** is a map of your kernel's basic blocks together with the branches connecting them. It lets the compiler answer questions of the form "if execution reaches *here*, what must already have happened?" - which is exactly what is needed to optimize reads and writes to memory. Two such optimizations run on the CFG: @@ -67,8 +65,6 @@ A **control-flow graph** is a map of your kernel's basic blocks together with th Building and analyzing the CFG is the most expensive optimization in the pipeline, which is why it runs at most once per simplify stage rather than every round. -**One CFG per offloaded task.** The CFG optimization is built and run separately for each offloaded task, over that task's IR alone - never over the whole `qd.kernel` at once. This is both faster to analyze and safe: because each task is a separate device launch, a value held in a register in one task cannot survive into the next one, so there is never anything to forward across a task boundary anyway. Anything written to global memory is treated as potentially read by a later task, so no store another task might need is dropped. - ## Controlling the passes All of these are fields of `CompileConfig`, so you set them at `qd.init(...)` (or via the matching `QD_` environment variable). See [qd.init options](./init_options.md) for the full list and the environment-variable convention. @@ -92,3 +88,7 @@ These environment variables dump the IR so you can see the effect of each pass. - `QD_DUMP_CFG=1` - writes the control-flow graph itself. (This also forces the CFG pass back onto the whole-kernel path so the complete graph can be dumped.) Setting `qd.init(print_ir=True)` prints the IR to the console at pipeline stages instead of writing files. + +## Under the hood: per-task scoping + +Once the kernel has been split into offloaded tasks, both CSE and the CFG optimization run over **one offloaded task's IR at a time**, never over the whole `qd.kernel` at once. This is both faster to analyze and safe: because each task is a separate device launch, a value held in a register in one task cannot survive into the next one, so there is never anything to deduplicate or forward across a task boundary. Anything written to global memory is treated as potentially read by a later task, so no store another task might need is dropped.