From 5848ae4e0ddfa8cf47a756427c2d2ef204962fc7 Mon Sep 17 00:00:00 2001 From: Gheorghe-Teodor Bercea Date: Wed, 22 Apr 2026 16:13:11 -0400 Subject: [PATCH 1/3] Change vector interleaving to 8 (#8) Co-authored-by: Cursor --- quadrants/runtime/amdgpu/jit_amdgpu.cpp | 8 +++ quadrants/runtime/llvm/llvm_context_pass.h | 61 ++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/quadrants/runtime/amdgpu/jit_amdgpu.cpp b/quadrants/runtime/amdgpu/jit_amdgpu.cpp index 7a2dccf783..df6e35ae54 100644 --- a/quadrants/runtime/amdgpu/jit_amdgpu.cpp +++ b/quadrants/runtime/amdgpu/jit_amdgpu.cpp @@ -142,6 +142,14 @@ std::string JITSessionAMDGPU::compile_module_to_hsaco(std::unique_ptr { + unsigned count_; + explicit AMDGPUSetLoopInterleavePass(unsigned count) : count_(count) {} + + llvm::PreservedAnalyses run(llvm::Loop &L, + llvm::LoopAnalysisManager &, + llvm::LoopStandardAnalysisResults &, + llvm::LPMUpdater &) { + // Only annotate innermost loops + if (!L.getSubLoops().empty()) + return llvm::PreservedAnalyses::all(); + + llvm::LLVMContext &ctx = L.getHeader()->getContext(); + llvm::MDNode *existing_id = L.getLoopID(); + + // Skip if interleave count is already specified + if (existing_id) { + for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) { + if (auto *node = llvm::dyn_cast(existing_id->getOperand(i))) { + if (node->getNumOperands() > 0) { + if (auto *s = llvm::dyn_cast(node->getOperand(0))) { + if (s->getString() == "llvm.loop.interleave.count") + return llvm::PreservedAnalyses::all(); + } + } + } + } + } + + // Build new interleave hint node + llvm::MDNode *hint = llvm::MDNode::get(ctx, { + llvm::MDString::get(ctx, "llvm.loop.interleave.count"), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), count_)) + }); + + // Assemble updated loop metadata: [self-ref, ...existing..., hint] + llvm::SmallVector ops; + ops.push_back(nullptr); // placeholder for self-reference + if (existing_id) { + for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) + ops.push_back(existing_id->getOperand(i).get()); + } + ops.push_back(hint); + + llvm::MDNode *new_id = llvm::MDNode::get(ctx, ops); + new_id->replaceOperandWith(0, new_id); // fix self-reference + L.setLoopID(new_id); + + return llvm::PreservedAnalyses::none(); + } +}; + } // namespace lang } // namespace quadrants From ad6d53631b600a6dd2a794eaa26110c88a388f80 Mon Sep 17 00:00:00 2001 From: ptcherni Date: Mon, 13 Jul 2026 18:38:03 -0500 Subject: [PATCH 2/3] [AMDGPU] Keep loop-interleave pass private to jit_amdgpu.cpp Move AMDGPUSetLoopInterleavePass out of the shared llvm_context_pass.h and into an anonymous namespace in jit_amdgpu.cpp (its only user), under the existing QD_WITH_AMDGPU guard, with the LoopInfo/LoopPassManager includes local to that translation unit. This keeps the common LLVM header free of backend-specific code so CPU/CUDA include sites no longer compile or depend on it. Pure relocation: pass logic and pipeline registration are unchanged. Co-authored-by: Cursor --- quadrants/runtime/amdgpu/jit_amdgpu.cpp | 67 ++++++++++++++++++++++ quadrants/runtime/llvm/llvm_context_pass.h | 58 ------------------- 2 files changed, 67 insertions(+), 58 deletions(-) diff --git a/quadrants/runtime/amdgpu/jit_amdgpu.cpp b/quadrants/runtime/amdgpu/jit_amdgpu.cpp index df6e35ae54..479d404601 100644 --- a/quadrants/runtime/amdgpu/jit_amdgpu.cpp +++ b/quadrants/runtime/amdgpu/jit_amdgpu.cpp @@ -2,7 +2,12 @@ #include "quadrants/runtime/llvm/llvm_context.h" #include "quadrants/runtime/llvm/llvm_context_pass.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Utils/Cloning.h" #include @@ -11,6 +16,68 @@ namespace quadrants { namespace lang { #if defined(QD_WITH_AMDGPU) + +namespace { +// New-PM loop pass: annotates innermost loops with llvm.loop.interleave.count +// IR metadata so the loop vectorizer uses the requested interleave factor. +// Scoped to the compilation pipeline where it is registered; does not touch +// any process-wide LLVM command-line state. Kept private to this AMDGPU JIT +// translation unit to avoid expanding the shared LLVM header surface area. +struct AMDGPUSetLoopInterleavePass + : public llvm::PassInfoMixin { + unsigned count_; + explicit AMDGPUSetLoopInterleavePass(unsigned count) : count_(count) {} + + llvm::PreservedAnalyses run(llvm::Loop &L, + llvm::LoopAnalysisManager &, + llvm::LoopStandardAnalysisResults &, + llvm::LPMUpdater &) { + // Only annotate innermost loops + if (!L.getSubLoops().empty()) + return llvm::PreservedAnalyses::all(); + + llvm::LLVMContext &ctx = L.getHeader()->getContext(); + llvm::MDNode *existing_id = L.getLoopID(); + + // Skip if interleave count is already specified + if (existing_id) { + for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) { + if (auto *node = llvm::dyn_cast(existing_id->getOperand(i))) { + if (node->getNumOperands() > 0) { + if (auto *s = llvm::dyn_cast(node->getOperand(0))) { + if (s->getString() == "llvm.loop.interleave.count") + return llvm::PreservedAnalyses::all(); + } + } + } + } + } + + // Build new interleave hint node + llvm::MDNode *hint = llvm::MDNode::get(ctx, { + llvm::MDString::get(ctx, "llvm.loop.interleave.count"), + llvm::ConstantAsMetadata::get( + llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), count_)) + }); + + // Assemble updated loop metadata: [self-ref, ...existing..., hint] + llvm::SmallVector ops; + ops.push_back(nullptr); // placeholder for self-reference + if (existing_id) { + for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) + ops.push_back(existing_id->getOperand(i).get()); + } + ops.push_back(hint); + + llvm::MDNode *new_id = llvm::MDNode::get(ctx, ops); + new_id->replaceOperandWith(0, new_id); // fix self-reference + L.setLoopID(new_id); + + return llvm::PreservedAnalyses::none(); + } +}; +} // namespace + JITModule *JITSessionAMDGPU ::add_module(std::unique_ptr M, int max_reg) { auto hsaco = compile_module_to_hsaco(M); QD_TRACE("hsaco size: {:.2f}KB", hsaco.size() / 1024.0); diff --git a/quadrants/runtime/llvm/llvm_context_pass.h b/quadrants/runtime/llvm/llvm_context_pass.h index d076dbae76..ed7f9ff883 100644 --- a/quadrants/runtime/llvm/llvm_context_pass.h +++ b/quadrants/runtime/llvm/llvm_context_pass.h @@ -298,63 +298,5 @@ struct AMDGPUConvertFuncParamAddressSpacePass : public ModulePass { #endif -// New-PM loop pass: annotates innermost loops with llvm.loop.interleave.count -// IR metadata so the loop vectorizer uses the requested interleave factor. -// Scoped to the compilation pipeline where it is registered; does not touch -// any process-wide LLVM command-line state. -struct AMDGPUSetLoopInterleavePass - : public llvm::PassInfoMixin { - unsigned count_; - explicit AMDGPUSetLoopInterleavePass(unsigned count) : count_(count) {} - - llvm::PreservedAnalyses run(llvm::Loop &L, - llvm::LoopAnalysisManager &, - llvm::LoopStandardAnalysisResults &, - llvm::LPMUpdater &) { - // Only annotate innermost loops - if (!L.getSubLoops().empty()) - return llvm::PreservedAnalyses::all(); - - llvm::LLVMContext &ctx = L.getHeader()->getContext(); - llvm::MDNode *existing_id = L.getLoopID(); - - // Skip if interleave count is already specified - if (existing_id) { - for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) { - if (auto *node = llvm::dyn_cast(existing_id->getOperand(i))) { - if (node->getNumOperands() > 0) { - if (auto *s = llvm::dyn_cast(node->getOperand(0))) { - if (s->getString() == "llvm.loop.interleave.count") - return llvm::PreservedAnalyses::all(); - } - } - } - } - } - - // Build new interleave hint node - llvm::MDNode *hint = llvm::MDNode::get(ctx, { - llvm::MDString::get(ctx, "llvm.loop.interleave.count"), - llvm::ConstantAsMetadata::get( - llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), count_)) - }); - - // Assemble updated loop metadata: [self-ref, ...existing..., hint] - llvm::SmallVector ops; - ops.push_back(nullptr); // placeholder for self-reference - if (existing_id) { - for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) - ops.push_back(existing_id->getOperand(i).get()); - } - ops.push_back(hint); - - llvm::MDNode *new_id = llvm::MDNode::get(ctx, ops); - new_id->replaceOperandWith(0, new_id); // fix self-reference - L.setLoopID(new_id); - - return llvm::PreservedAnalyses::none(); - } -}; - } // namespace lang } // namespace quadrants From ff37e707d141fe3f7f503feb47feb0ca8114e2ea Mon Sep 17 00:00:00 2001 From: ptcherni Date: Thu, 16 Jul 2026 17:37:02 -0500 Subject: [PATCH 3/3] [AMDGPU] Address review: trim shared header, 120c comments, sync GCN dump - Remove AMDGPU-only includes (Metadata.h, LoopInfo.h, LoopPassManager.h) from the shared llvm_context_pass.h; they are only needed by the private pass in jit_amdgpu.cpp, which already includes them. Keeps the common LLVM header off CPU/CUDA include sites. - Rewrap the loop-interleave comments to 120 columns. - Register the interleave pass on the print_kernel_amdgcn dump pipeline via a shared register_loop_interleave() helper so the emitted .gcn matches the linked HSACO object. pre-commit run -a passes (black, clang-format, ruff, pylint, whitespace). --- quadrants/runtime/amdgpu/jit_amdgpu.cpp | 54 ++++++++++++---------- quadrants/runtime/llvm/llvm_context_pass.h | 3 -- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/quadrants/runtime/amdgpu/jit_amdgpu.cpp b/quadrants/runtime/amdgpu/jit_amdgpu.cpp index 479d404601..28ab970f4b 100644 --- a/quadrants/runtime/amdgpu/jit_amdgpu.cpp +++ b/quadrants/runtime/amdgpu/jit_amdgpu.cpp @@ -6,6 +6,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Utils/Cloning.h" @@ -18,20 +19,19 @@ namespace lang { #if defined(QD_WITH_AMDGPU) namespace { -// New-PM loop pass: annotates innermost loops with llvm.loop.interleave.count -// IR metadata so the loop vectorizer uses the requested interleave factor. -// Scoped to the compilation pipeline where it is registered; does not touch -// any process-wide LLVM command-line state. Kept private to this AMDGPU JIT -// translation unit to avoid expanding the shared LLVM header surface area. -struct AMDGPUSetLoopInterleavePass - : public llvm::PassInfoMixin { +// New-PM loop pass: annotates innermost loops with llvm.loop.interleave.count IR metadata so the loop vectorizer uses +// the requested interleave factor. Scoped to the compilation pipeline where it is registered; does not touch any +// process-wide LLVM command-line state. Kept private to this AMDGPU JIT translation unit to avoid expanding the shared +// LLVM header surface area. +struct AMDGPUSetLoopInterleavePass : public llvm::PassInfoMixin { unsigned count_; - explicit AMDGPUSetLoopInterleavePass(unsigned count) : count_(count) {} + explicit AMDGPUSetLoopInterleavePass(unsigned count) : count_(count) { + } llvm::PreservedAnalyses run(llvm::Loop &L, - llvm::LoopAnalysisManager &, - llvm::LoopStandardAnalysisResults &, - llvm::LPMUpdater &) { + llvm::LoopAnalysisManager &, + llvm::LoopStandardAnalysisResults &, + llvm::LPMUpdater &) { // Only annotate innermost loops if (!L.getSubLoops().empty()) return llvm::PreservedAnalyses::all(); @@ -54,15 +54,13 @@ struct AMDGPUSetLoopInterleavePass } // Build new interleave hint node - llvm::MDNode *hint = llvm::MDNode::get(ctx, { - llvm::MDString::get(ctx, "llvm.loop.interleave.count"), - llvm::ConstantAsMetadata::get( - llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), count_)) - }); + llvm::MDNode *hint = llvm::MDNode::get( + ctx, {llvm::MDString::get(ctx, "llvm.loop.interleave.count"), + llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(llvm::Type::getInt32Ty(ctx), count_))}); // Assemble updated loop metadata: [self-ref, ...existing..., hint] llvm::SmallVector ops; - ops.push_back(nullptr); // placeholder for self-reference + ops.push_back(nullptr); // placeholder for self-reference if (existing_id) { for (unsigned i = 1, e = existing_id->getNumOperands(); i < e; ++i) ops.push_back(existing_id->getOperand(i).get()); @@ -70,12 +68,19 @@ struct AMDGPUSetLoopInterleavePass ops.push_back(hint); llvm::MDNode *new_id = llvm::MDNode::get(ctx, ops); - new_id->replaceOperandWith(0, new_id); // fix self-reference + new_id->replaceOperandWith(0, new_id); // fix self-reference L.setLoopID(new_id); return llvm::PreservedAnalyses::none(); } }; + +// Registers AMDGPUSetLoopInterleavePass on the given PassBuilder. Shared by the HSACO code-generation pipeline and the +// print_kernel_amdgcn dump pipeline so the emitted .gcn reflects the same interleave metadata as the linked object. +void register_loop_interleave(llvm::PassBuilder &pb) { + pb.registerLoopOptimizerEndEPCallback( + [](llvm::LoopPassManager &lpm, llvm::OptimizationLevel) { lpm.addPass(AMDGPUSetLoopInterleavePass(8)); }); +} } // namespace JITModule *JITSessionAMDGPU ::add_module(std::unique_ptr M, int max_reg) { @@ -180,6 +185,9 @@ std::string JITSessionAMDGPU::compile_module_to_hsaco(std::unique_ptr