From 7806ef5e5057427e33b530e8df3ca12dfd36c5d3 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 10 May 2026 08:33:19 +0200 Subject: [PATCH 01/33] Switch to CompilerCaching for in-memory CI caching. Drop the hand-rolled CodeCache, on-disk kernel cache, and various pre-1.11 compatibility shims; route inference and CI lookup through CompilerCaching.CacheView with consumer-defined results structs attached to each CodeInstance. Co-Authored-By: Claude Opus 4.7 (1M context) --- Project.toml | 8 +- src/GPUCompiler.jl | 19 -- src/deprecated.jl | 45 --- src/driver.jl | 53 +--- src/execution.jl | 221 -------------- src/interface.jl | 63 ++-- src/irgen.jl | 10 - src/jlgen.jl | 727 ++++++++------------------------------------- src/optim.jl | 5 +- src/rtlib.jl | 51 +--- src/utils.jl | 17 +- src/validation.jl | 13 +- 12 files changed, 177 insertions(+), 1055 deletions(-) delete mode 100644 src/deprecated.jl diff --git a/Project.toml b/Project.toml index 64b5851c..30456efa 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ authors = ["Tim Besard "] projects = ["test"] [deps] +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" ExprTools = "e2ba6199-217a-4e67-a87a-7c52f15ade04" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" @@ -15,8 +16,6 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Scratch = "6c6a2e73-6563-6170-7368-637461726353" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tracy = "e689c965-62c8-4b79-b2c5-8359227902fd" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -28,6 +27,7 @@ NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" [compat] AMDGPU_LLVM_Backend_jll = "22" +CompilerCaching = "0.2.4" ExprTools = "0.1" InteractiveUtils = "1" LLVM = "9.9" @@ -38,9 +38,7 @@ NVPTX_LLVM_Backend_jll = "22" PrecompileTools = "1" Preferences = "1" REPL = "1" -Scratch = "1" -Serialization = "1" TOML = "1" Tracy = "0.1.4" UUIDs = "1" -julia = "1.10" +julia = "1.11" diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index 2e646267..098f1617 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -8,8 +8,6 @@ using ExprTools: splitdef, combinedef using Libdl -using Serialization -using Scratch: @get_scratch! using Preferences const ENABLE_TRACY = parse(Bool, @load_preference("tracy", "false")) @@ -35,9 +33,6 @@ end const CC = Core.Compiler using Core: MethodInstance, CodeInstance, CodeInfo -compile_cache = nothing # set during __init__() -const pkgver = Base.pkgversion(GPUCompiler) - include("utils.jl") include("mangling.jl") @@ -67,25 +62,11 @@ include("driver.jl") include("execution.jl") include("reflection.jl") -include("deprecated.jl") - include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) - dir = @get_scratch!("compiled") - ## add the Julia version - dir = joinpath(dir, "v$(VERSION.major).$(VERSION.minor)") - ## also add the package version - if pkgver !== nothing - # XXX: `Base.pkgversion` is buggy and sometimes returns `nothing`, see e.g. - # JuliaLang/PackageCompiler.jl#896 and JuliaGPU/GPUCompiler.jl#593 - dir = joinpath(dir, "v$(pkgver.major).$(pkgver.minor)") - end - mkpath(dir) - global compile_cache = dir - @static if ENABLE_TRACY Tracy.@register_tracepoints() end diff --git a/src/deprecated.jl b/src/deprecated.jl deleted file mode 100644 index c5848854..00000000 --- a/src/deprecated.jl +++ /dev/null @@ -1,45 +0,0 @@ -# Deprecations scheduled for removal in the next major release. - -function defs(mod::LLVM.Module) - safe_depwarn("`GPUCompiler.defs(mod)` is deprecated; inline `filter(f -> !isdeclaration(f), collect(functions(mod)))`.", - :defs) - filter(f -> !isdeclaration(f), collect(functions(mod))) -end - -function decls(mod::LLVM.Module) - safe_depwarn("`GPUCompiler.decls(mod)` is deprecated; inline `filter(f -> isdeclaration(f) && !LLVM.isintrinsic(f), collect(functions(mod)))`.", - :decls) - filter(f -> isdeclaration(f) && !LLVM.isintrinsic(f), collect(functions(mod))) -end - -link_library!(mod::LLVM.Module, lib::LLVM.Module) = link_library!(mod, [lib]) -function link_library!(mod::LLVM.Module, libs::Vector{LLVM.Module}) - safe_depwarn("`GPUCompiler.link_library!` is deprecated; call `LLVM.link!(mod, copy(lib))` directly, or `LLVM.link!(mod, lib; only_needed=true)` with a freshly-parsed library.", - :link_library!) - libs = [copy(lib) for lib in libs] - for lib in libs - link!(mod, lib) - end -end - -# no-op 3-arg fallback so downstream overrides that chain via -# `invoke(GPUCompiler.link_libraries!, Tuple{CompilerJob, Module, -# Vector{String}}, ...)` still resolve. -link_libraries!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - undefined_fns::Vector{String}) = return - -# `true` when a downstream package has defined a 3-arg `link_libraries!` -# override for `job`, i.e. the dispatched method isn't our fallback above. -# -# Uses the same `jl_gf_invoke_lookup` path as `Core._hasmethod` rather than -# `which`, so it's safe to call from generated-function-adjacent contexts -# where `Base.get_world_counter()` returns `typemax(UInt)` and reflection -# queries like `which` / `methods` fail (see JuliaLang/julia#48611). -# All this because Enzyme.jl calls GPUCompiler.jl from a generated function. -function has_legacy_link_libraries(@nospecialize(job::CompilerJob)) - tt = Tuple{typeof(link_libraries!), typeof(job), - LLVM.Module, Vector{String}} - world = ccall(:jl_get_tls_world_age, UInt, ()) - m = ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) - return m !== nothing && (m::Method).module !== @__MODULE__ -end diff --git a/src/driver.jl b/src/driver.jl index 30529c70..d25a51f1 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -50,34 +50,16 @@ const compile_hook = Ref{Union{Nothing,Function}}(nothing) compile(target::Symbol, job::CompilerJob) Compile a `job` to one of the following formats as specified by the `target` argument: -`:julia` for Julia IR, `:llvm` for LLVM IR and `:asm` for machine code. +`:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. """ -function compile(target::Symbol, @nospecialize(job::CompilerJob); kwargs...) - # XXX: remove on next major version - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `compile` API does not take keyword arguments anymore. Use CompilerConfig instead.", :compile) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - +function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing Base.invokelatest(compile_hook[], job) end - return compile_unhooked(target, job) end -# XXX: remove on next major version -function codegen(output::Symbol, @nospecialize(job::CompilerJob); kwargs...) - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `codegen` function is an internal API. Use `GPUCompiler.compile` (with any kwargs passed to `CompilerConfig`) instead.", :codegen) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - compile_unhooked(output, job) -end - -function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob); kwargs...) +function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) if context(; throw_error=false) === nothing error("No active LLVM context. Use `JuliaContext()` do-block syntax to create one.") end @@ -188,14 +170,7 @@ end const __llvm_initialized = Ref(false) -@locked function emit_llvm(@nospecialize(job::CompilerJob); kwargs...) - # XXX: remove on next major version - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `emit_llvm` function is an internal API. Use `GPUCompiler.compile` (with any kwargs passed to `CompilerConfig`) instead.", :emit_llvm) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - +@locked function emit_llvm(@nospecialize(job::CompilerJob)) if !__llvm_initialized[] InitializeAllTargets() InitializeAllTargetInfos() @@ -259,7 +234,7 @@ const __llvm_initialized = Ref(false) target = nest_target(dyn_job.config.target, job.config.target) params = nest_params(dyn_job.config.params, job.config.params) config = CompilerConfig(dyn_job.config; toplevel=false, target, params) - return codegen(:llvm, CompilerJob(dyn_job; config)) + return compile_unhooked(:llvm, CompilerJob(dyn_job; config)) end # compile and link @@ -323,23 +298,7 @@ const __llvm_initialized = Ref(false) @tracepoint "Library linking" begin # target-specific libraries - @tracepoint "target libraries" begin - if has_legacy_link_libraries(job) - safe_depwarn( - "3-arg `link_libraries!(job, mod, undefined_fns)` is deprecated; " * - "migrate your override to the 2-arg form `link_libraries!(job, mod)`. " * - "Instead of inspecting `undefined_fns` to decide what to link, " * - "parse the library lazily with `parse(LLVM.Module, bytes; lazy=true)` " * - "and link it with `LLVM.link!(mod, lib; only_needed=true)` — " * - "the linker will then materialize only the referenced symbols.", - :link_libraries!) - undefined_fns = [LLVM.name(f) for f in functions(ir) - if isdeclaration(f) && !LLVM.isintrinsic(f)] - link_libraries!(job, ir, undefined_fns) - else - link_libraries!(job, ir) - end - end + @tracepoint "target libraries" link_libraries!(job, ir) # GPU run-time library if !uses_julia_runtime(job) diff --git a/src/execution.jl b/src/execution.jl index 9b4940a7..dcdd4236 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -66,224 +66,3 @@ function assign_args!(code, _args) return vars, var_exprs end - - -## cached compilation - -### Notes on interactions with package images and disk cache. -# Julia uses package images (pkgimg) to cache both the result of inference, -# and the result of native code emissions. Up until Julia v1.11 neither the -# inferred nor the nativce code of foreign abstract interpreters was cached -# across sessions. Julia v1.11 allows for caching of inference results across -# sessions as long as those inference results are created during precompilation. -# -# Julia cache hierarchy is roughly as follows: -# Function (name of a thing) -# -> Method (particular piece of code to dispatch to with a signature) -# -> MethodInstance (A particular Method + particular signature) -# -> CodeInstance (A MethodInstance compiled for a world) -# -# In order to cache code across sessions we need to insert CodeInstance(owner=GPUCompilerCacheToken) -# into the internal cache. Once we have done so we know that a particular CodeInstance is unique in -# the system. (During pkgimg loading conflicts will be resolved). -# -# When a pkgimg is loaded we check it's validity, this means checking that all depdencies are the same, -# the pkgimg was created for the right set of compiler flags, and that all source code that was used -# to create this pkgimg is the same. When a CodeInstance is inside a pkgimg we can extend the chain of -# validity even for GPU code, we cannot verify a "runtime" CodeInstance in the same way. -# -# Therefore when we see a compilation request for a CodeInstance that is originating from a pkgimg -# we can use it as part of the hash for the on-disk cache. (see `cache_file`) - -""" - disk_cache_enabled() - -Query if caching to disk is enabled. -""" -disk_cache_enabled() = parse(Bool, @load_preference("disk_cache", "false")) - -""" - enable_disk_cache!(state::Bool=true) - -Activate the GPUCompiler disk cache in the current environment. -You will need to restart your Julia environment for it to take effect. - -!!! note - The cache functionality requires Julia 1.11 -""" -function enable_disk_cache!(state::Bool=true) - @set_preferences!("disk_cache"=>string(state)) -end - -disk_cache_path() = @get_scratch!("disk_cache") -clear_disk_cache!() = rm(disk_cache_path(); recursive=true, force=true) - -const cache_lock = ReentrantLock() - -""" - cached_compilation(cache::Dict{Any}, src::MethodInstance, cfg::CompilerConfig, - compiler, linker) - -Compile a method instance `src` with configuration `cfg`, by invoking `compiler` and -`linker` and storing the result in `cache`. - -The `cache` argument should be a dictionary that can be indexed using any value and store -whatever the `linker` function returns. The `compiler` function should take a `CompilerJob` -and return data that can be cached across sessions (e.g., LLVM IR). This data is then -forwarded, along with the `CompilerJob`, to the `linker` function which is allowed to create -session-dependent objects (e.g., a `CuModule`). -""" -function cached_compilation(cache::AbstractDict{<:Any,V}, - src::MethodInstance, cfg::CompilerConfig, - compiler::Function, linker::Function) where {V} - # NOTE: we index the cach both using (mi, world, cfg) keys, for the fast look-up, - # and using CodeInfo keys for the slow look-up. we need to cache both for - # performance, but cannot use a separate private cache for the ci->obj lookup - # (e.g. putting it next to the CodeInfo's in the CodeCache) because some clients - # expect to be able to wipe the cache (e.g. CUDA.jl's `device_reset!`) - - # fast path: index the cache directly for the *current* world + compiler config - - world = tls_world_age() - key = (objectid(src), world, cfg) - # NOTE: we store the MethodInstance's objectid to avoid an expensive allocation. - # Base does this with a multi-level lookup, first keyed on the mi, - # then a linear scan over the (typically few) entries. - - # NOTE: no use of lock(::Function)/@lock/get! to avoid try/catch and closure overhead - lock(cache_lock) - obj = get(cache, key, nothing) - unlock(cache_lock) - - if obj === nothing || compile_hook[] !== nothing - obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V - lock(cache_lock) - cache[key] = obj - unlock(cache_lock) - end - return obj::V -end - -@noinline function cache_file(ci::CodeInstance, cfg::CompilerConfig) - h = hash(Base.objectid(ci)) - @static if isdefined(Base, :object_build_id) - bid = Base.object_build_id(ci) - if bid === nothing # CI is from a runtime compilation, not worth caching on disk - return nothing - else - bid = bid % UInt64 # The upper 64bit are a checksum, unavailable during precompilation - end - h = hash(bid, h) - end - h = hash(cfg, h) - - gpucompiler_buildid = Base.module_build_id(@__MODULE__) - if (gpucompiler_buildid >> 64) % UInt64 == 0xffffffffffffffff - return nothing # Don't cache during precompilation of GPUCompiler - end - - return joinpath( - disk_cache_path(), - # bifurcate the cache by build id of GPUCompiler - string(gpucompiler_buildid), - string(h, ".jls")) -end - -struct DiskCacheEntry - src::Type # Originally MethodInstance, but upon deserialize they were not uniqued... - cfg::CompilerConfig - asm -end - -@noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, - cfg::CompilerConfig, compiler::Function, linker::Function) - job = CompilerJob(src, cfg, world) - obj = nothing - - # fast path: find an applicable CodeInstance and see if we have compiled it before - ci = ci_cache_lookup(ci_cache(job), src, world, world)::Union{Nothing,CodeInstance} - if ci !== nothing - key = (ci, cfg) - obj = get(cache, key, nothing) - end - - # slow path: compile and link - if obj === nothing || compile_hook[] !== nothing - asm = nothing - path = nothing - ondisk_hit = false - @static if VERSION >= v"1.11.0-" - # Don't try to hit the disk cache if we are for a *compile* hook - # TODO: - # - Sould we hit disk cache if Base.generating_output() - # - Should we allow backend to opt out? - if ci !== nothing && obj === nothing && disk_cache_enabled() - path = cache_file(ci, cfg) - @debug "Looking for on-disk cache" job path - if path !== nothing && isfile(path) - ondisk_hit = true - try - @debug "Loading compiled kernel" job path - # The MI we deserialize here didn't get uniqued... - entry = deserialize(path)::DiskCacheEntry - if entry.src == src.specTypes && entry.cfg == cfg - asm = entry.asm - else - @show entry.src == src.specTypes - @show entry.cfg == cfg - @warn "Cache missmatch" src.specTypes cfg entry.src entry.cfg - end - catch ex - @warn "Failed to load compiled kernel" job path exception=(ex, catch_backtrace()) - end - end - end - end - - if asm === nothing || compile_hook[] !== nothing - # Run the compiler in-case we need to hook it. - asm = compiler(job) - end - if obj !== nothing - # we got here because of a *compile* hook; don't bother linking - return obj - end - - @static if VERSION >= v"1.11.0-" - if !ondisk_hit && path !== nothing && disk_cache_enabled() - @debug "Writing out on-disk cache" job path - mkpath(dirname(path)) - entry = DiskCacheEntry(src.specTypes, cfg, asm) - - # atomic write to disk - tmppath, io = mktemp(dirname(path); cleanup=false) - serialize(io, entry) - close(io) - @static if VERSION >= v"1.12.0-DEV.1023" - mv(tmppath, path; force=true) - else - Base.rename(tmppath, path, force=true) - end - end - end - - obj = linker(job, asm) - - if ci === nothing - ci = ci_cache_lookup(ci_cache(job), src, world, world) - if ci === nothing - error("""Did not find CodeInstance for $job. - - Pleaase make sure that the `compiler` function passed to `cached_compilation` - invokes GPUCompiler with exactly the same configuration as passed to the API. - - Note that you should do this by calling `GPUCompiler.compile`, and not by - using reflection functions (which alter the compiler configuration).""") - end - key = (ci, cfg) - end - cache[key] = obj - end - - return obj -end diff --git a/src/interface.jl b/src/interface.jl index 6aec9f1e..87f9733f 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -261,15 +261,11 @@ runtime_module(@nospecialize(job::CompilerJob)) = error("Not implemented") isintrinsic(@nospecialize(job::CompilerJob), fn::String) = false # provide a specific interpreter to use. -if VERSION >= v"1.11.0-DEV.1552" -get_interpreter(@nospecialize(job::CompilerJob)) = - GPUInterpreter(job.world; method_table_view=maybe_cached(method_table_view(job)), - token=ci_cache_token(job), inf_params=inference_params(job), - opt_params=optimization_params(job)) -else -get_interpreter(@nospecialize(job::CompilerJob)) = - GPUInterpreter(job.world; method_table_view=maybe_cached(method_table_view(job)), - code_cache=ci_cache(job), inf_params=inference_params(job), +function get_interpreter(@nospecialize(job::CompilerJob)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + cache=cache_view(job), + inf_params=inference_params(job), opt_params=optimization_params(job)) end @@ -305,36 +301,47 @@ pass_by_ref(@nospecialize(job::CompilerJob)) = false # whether pointer is a valid call target valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false +# Cache partitioning. The owner is stored on every CodeInstance and compared via `jl_egal`, +# so it must be immutable for cross-session matches (e.g. via package precompilation). # Care is required for anything that impacts: # - method_table # - inference_params # - optimization_params -# By default that is just always_inline -# the cache token is compared with jl_egal +# Default covers `always_inline` (which feeds optimization_params) and the method table. struct GPUCompilerCacheToken target_type::Type always_inline::Bool method_table::Core.MethodTable end -ci_cache_token(@nospecialize(job::CompilerJob)) = +cache_owner(@nospecialize(job::CompilerJob)) = GPUCompilerCacheToken(typeof(job.config.target), job.config.always_inline, method_table(job)) -# the codeinstance cache to use -- should only be used for the constructor -if VERSION >= v"1.11.0-DEV.1552" - # Soft deprecated user should use `CC.code_cache(get_interpreter(job))` - ci_cache(@nospecialize(job::CompilerJob)) = CC.code_cache(get_interpreter(job)) -else -function ci_cache(@nospecialize(job::CompilerJob)) - lock(GLOBAL_CI_CACHES_LOCK) do - cache = get!(GLOBAL_CI_CACHES, job.config) do - CodeCache() - end - return cache - end -end +""" + GPUCompiler.NoResults() + +Default results type carried on each cached `CodeInstance` when the consumer hasn't +overridden [`results_type`](@ref). Carries no fields; useful for compiler jobs that +don't need to memoize compiled artifacts (e.g., reflection, precompile workloads). +""" +mutable struct NoResults end + +# The consumer's results struct type, stored on each CodeInstance via `CompilerCaching`. +# Override to attach session-portable artifacts (IR, object bytes) and session-local handles +# (e.g., `CuModule`, `MTLComputePipelineState`) to compiled CIs. The struct must be a +# `mutable struct` with a zero-arg constructor. +results_type(@nospecialize(job::CompilerJob)) = NoResults + +# Construct a `CompilerCaching.CacheView` partitioned by `cache_owner(job)` and parametrized +# by `results_type(job)`. +function cache_view(@nospecialize(job::CompilerJob)) + K = typeof(cache_owner(job)) + V = results_type(job) + CompilerCaching.CacheView{K, V}(cache_owner(job), job.world) end +public GPUCompilerCacheToken, cache_owner, NoResults, results_type, cache_view + # the method table to use # deprecate method_table on next-breaking release method_table(@nospecialize(job::CompilerJob)) = GLOBAL_METHOD_TABLE @@ -401,9 +408,3 @@ finish_ir!(@nospecialize(job::CompilerJob), mod::LLVM.Module, entry::LLVM.Functi # whether an LLVM function is valid for this back-end validate_ir(@nospecialize(job::CompilerJob), mod::LLVM.Module) = IRError[] - -# deprecated -struct DeprecationMarker end -process_module!(@nospecialize(job::CompilerJob), mod::LLVM.Module) = DeprecationMarker() -process_entry!(@nospecialize(job::CompilerJob), mod::LLVM.Module, entry::LLVM.Function) = - DeprecationMarker() diff --git a/src/irgen.jl b/src/irgen.jl index 3df1d659..9129e20c 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -43,11 +43,6 @@ function irgen(@nospecialize(job::CompilerJob)) end end - deprecation_marker = process_module!(job, mod) - if deprecation_marker != DeprecationMarker() - safe_depwarn("GPUCompiler.process_module! is deprecated; implement GPUCompiler.finish_module! instead", :process_module) - end - # sanitize global values (Julia doesn't when using the external codegen policy) for val in [collect(globals(mod)); collect(functions(mod))] isdeclaration(val) && continue @@ -69,11 +64,6 @@ function irgen(@nospecialize(job::CompilerJob)) elseif job.config.kernel LLVM.name!(entry, mangle_sig(job.source.specTypes)) end - deprecation_marker = process_entry!(job, mod, entry) - if deprecation_marker != DeprecationMarker() - safe_depwarn("GPUCompiler.process_entry! is deprecated; implement GPUCompiler.finish_module! instead", :process_entry) - entry = deprecation_marker - end if job.config.entry_abi === :specfunc func = compiled[job.source].func specfunc = LLVM.name(entry) diff --git a/src/jlgen.jl b/src/jlgen.jl index 8db65c89..23e0ce6d 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -1,16 +1,13 @@ # Julia compiler integration +import CompilerCaching +using CompilerCaching: CacheView, @setup_caching + ## world age lookups -# `tls_world_age` should be used to look up the current world age. in most cases, this is -# what you should use to invoke the compiler with. +import Base: tls_world_age -if isdefined(Base, :tls_world_age) - import Base: tls_world_age -else - tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) -end ## looking up method instances @@ -38,11 +35,6 @@ function MethodError(ft::Type{<:Function}, tt::Type, world::Integer=typemax(UInt end MethodError(ft, tt, world=typemax(UInt)) = Base.MethodError(ft, tt, world) -# generate a LineInfoNode for the current source code location -macro LineInfoNode(method) - Core.LineInfoNode(__module__, method, __source__.file, Int32(__source__.line), Int32(0)) -end - """ methodinstance(ft::Type, tt::Type, [world::UInt]) @@ -56,31 +48,11 @@ This function is highly optimized, and results do not need to be cached addition Only use this function with concrete signatures, i.e., using the types of values you would pass at run time. For non-concrete signatures, use `generic_methodinstance` instead. - """ -methodinstance - -function generic_methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), - world::Integer=tls_world_age()) - sig = signature_type_by_tt(ft, tt) - - match, _ = CC._findsup(sig, nothing, world) - match === nothing && throw(MethodError(ft, tt, world)) - - mi = CC.specialize_method(match) - - return mi::MethodInstance -end - -# on 1.11 (JuliaLang/julia#52572, merged as part of JuliaLang/julia#52233) we can use -# Julia's cached method lookup to simply look up method instances at run time. -@static if VERSION >= v"1.11.0-DEV.1552" - -# XXX: version of Base.method_instance that uses a function type -@inline function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), - world::Integer=tls_world_age()) +function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), + world::Integer=tls_world_age()) sig = signature_type_by_tt(ft, tt) - @assert Base.isdispatchtuple(sig) # JuliaLang/julia#52233 + @assert Base.isdispatchtuple(sig) mi = ccall(:jl_method_lookup_by_tt, Any, (Any, Csize_t, Any), @@ -96,201 +68,17 @@ end return mi end -# on older versions of Julia, we always need to use the generic lookup -else - -const methodinstance = generic_methodinstance - -function methodinstance_generator(world::UInt, source, self, ft::Type, tt::Type) - @nospecialize - @assert CC.isType(ft) && CC.isType(tt) - ft = ft.parameters[1] - tt = tt.parameters[1] - - stub = Core.GeneratedFunctionStub(identity, Core.svec(:methodinstance, :ft, :tt), Core.svec()) - - # look up the method match - method_error = :(throw(MethodError(ft, tt, $world))) - sig = Tuple{ft, tt.parameters...} - min_world = Ref{UInt}(typemin(UInt)) - max_world = Ref{UInt}(typemax(UInt)) - match = ccall(:jl_gf_invoke_lookup_worlds, Any, - (Any, Any, Csize_t, Ref{Csize_t}, Ref{Csize_t}), - sig, #=mt=# nothing, world, min_world, max_world) - match === nothing && return stub(world, source, method_error) - - # look up the method and code instance - mi = ccall(:jl_specializations_get_linfo, Ref{MethodInstance}, - (Any, Any, Any), match.method, match.spec_types, match.sparams) - ci = CC.retrieve_code_info(mi, world) - - # prepare a new code info - new_ci = copy(ci) - empty!(new_ci.code) - empty!(new_ci.codelocs) - empty!(new_ci.linetable) - empty!(new_ci.ssaflags) - new_ci.ssavaluetypes = 0 - - # propagate edge metadata - new_ci.min_world = min_world[] - new_ci.max_world = max_world[] - new_ci.edges = Any[mi] - - # prepare the slots - new_ci.slotnames = Symbol[Symbol("#self#"), :ft, :tt] - new_ci.slotflags = UInt8[0x00 for i = 1:3] - - # return the method instance - push!(new_ci.code, CC.ReturnNode(mi)) - push!(new_ci.ssaflags, 0x00) - push!(new_ci.linetable, @LineInfoNode(methodinstance)) - push!(new_ci.codelocs, 1) - new_ci.ssavaluetypes += 1 - - return new_ci -end - -@eval function methodinstance(ft, tt) - $(Expr(:meta, :generated_only)) - $(Expr(:meta, :generated, methodinstance_generator)) -end - -end - - -## code instance cache -const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" - -if !HAS_INTEGRATED_CACHE -struct CodeCache - dict::IdDict{MethodInstance,Vector{CodeInstance}} - - CodeCache() = new(IdDict{MethodInstance,Vector{CodeInstance}}()) -end - -function Base.show(io::IO, ::MIME"text/plain", cc::CodeCache) - print(io, "CodeCache with $(mapreduce(length, +, values(cc.dict); init=0)) entries") - if !isempty(cc.dict) - print(io, ": ") - for (mi, cis) in cc.dict - println(io) - print(io, " ") - show(io, mi) - - function worldstr(min_world, max_world) - if min_world == typemax(UInt) - "empty world range" - elseif max_world == typemax(UInt) - "worlds $(Int(min_world))+" - else - "worlds $(Int(min_world)) to $(Int(max_world))" - end - end - - for (i,ci) in enumerate(cis) - println(io) - print(io, " CodeInstance for ", worldstr(ci.min_world, ci.max_world)) - end - end - end -end - -Base.empty!(cc::CodeCache) = empty!(cc.dict) - -const GLOBAL_CI_CACHES = Dict{CompilerConfig, CodeCache}() -const GLOBAL_CI_CACHES_LOCK = ReentrantLock() - - -## method invalidations - -function CC.setindex!(cache::CodeCache, ci::CodeInstance, mi::MethodInstance) - # make sure the invalidation callback is attached to the method instance - add_codecache_callback!(cache, mi) - cis = get!(cache.dict, mi, CodeInstance[]) - push!(cis, ci) -end - -# invalidation (like invalidate_method_instance, but for our cache) -struct CodeCacheCallback - cache::CodeCache -end - -@static if VERSION ≥ v"1.11.0-DEV.798" - -function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) - callback = CodeCacheCallback(cache) - CC.add_invalidation_callback!(callback, mi) -end -function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32) - cis = get(callback.cache.dict, replaced, nothing) - if cis === nothing - return - end - for ci in cis - if ci.max_world == ~0 % Csize_t - @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" -@static if VERSION >= v"1.11.0-DEV.1390" - @atomic ci.max_world = max_world -else - ci.max_world = max_world -end - end - @assert ci.max_world <= max_world - end -end - -else - -function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) - callback = CodeCacheCallback(cache) - if !isdefined(mi, :callbacks) - mi.callbacks = Any[callback] - elseif !in(callback, mi.callbacks) - push!(mi.callbacks, callback) - end -end -function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32, - seen::Set{MethodInstance}=Set{MethodInstance}()) - push!(seen, replaced) - - cis = get(callback.cache.dict, replaced, nothing) - if cis === nothing - return - end - for ci in cis - if ci.max_world == ~0 % Csize_t - @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" - ci.max_world = max_world - end - @assert ci.max_world <= max_world - end - - # recurse to all backedges to update their valid range also - if isdefined(replaced, :backedges) - backedges = filter(replaced.backedges) do @nospecialize(mi) - if mi isa MethodInstance - mi ∉ seen - elseif mi isa Type - # an `invoke` call, which is a `(sig, MethodInstance)` pair. - # let's ignore the `sig` and process the `MethodInstance` next. - false - else - error("invalid backedge") - end - end +function generic_methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), + world::Integer=tls_world_age()) + sig = signature_type_by_tt(ft, tt) - # Don't touch/empty backedges `invalidate_method_instance` in C will do that later - # replaced.backedges = Any[] + match, _ = CC._findsup(sig, nothing, world) + match === nothing && throw(MethodError(ft, tt, world)) - for mi in backedges - callback(mi::MethodInstance, max_world, seen) - end - end -end + mi = CC.specialize_method(match) + return mi::MethodInstance end -end # !HAS_INTEGRATED_CACHE ## method overrides @@ -298,8 +86,6 @@ end # !HAS_INTEGRATED_CACHE Base.Experimental.@MethodTable(GLOBAL_METHOD_TABLE) # Implements a priority lookup for method tables, where the first match in the stack get's returned. -# An alternative to this would be to use a "Union" where we would query the parent method table and -# do a most-specific match. struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView world::UInt mt::Core.MethodTable @@ -310,91 +96,45 @@ StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) CC.isoverlayed(::StackedMethodTable) = true -@static if VERSION >= v"1.11.0-DEV.363" - # https://github.com/JuliaLang/julia/pull/51078 - # same API as before but without returning isoverlayed flag - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return result - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} - parent_result === nothing && return nothing #too many matches - - # merge the parent match results with the internal method table - return CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig) +function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) + result = CC._findall(sig, table.mt, table.world, limit) + result === nothing && return nothing # to many matches + nr = CC.length(result) + if nr ≥ 1 && CC.getindex(result, nr).fully_covers + # no need to fall back to the parent method view + return result end - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds - parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)) - ) - end -else - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return CC.MethodMatchResult(result, true) - end + parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} + parent_result === nothing && return nothing #too many matches - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} - parent_result === nothing && return nothing #too many matches - - overlayed = parent_result.overlayed | !CC.isempty(result) - parent_result = parent_result.matches::CC.MethodLookupResult - - # merge the parent match results with the internal method table - return CC.MethodMatchResult( - CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig), - overlayed) - end + # merge the parent match results with the internal method table + return CC.MethodLookupResult( + CC.vcat(result.matches, parent_result.matches), + CC.WorldRange( + CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), + CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), + result.ambig | parent_result.ambig) +end - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds, true - parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)), - overlayed) - end +function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) + match, valid_worlds = CC._findsup(sig, table.mt, table.world) + match !== nothing && return match, valid_worlds + parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) + return ( + parent_match, + CC.WorldRange( + max(valid_worlds.min_world, parent_valid_worlds.min_world), + min(valid_worlds.max_world, parent_valid_worlds.max_world)) + ) end + ## interpreter -@static if VERSION >= v"1.11.0-DEV.1498" - import Core.Compiler: get_inference_world - using Base: get_world_counter -else - import Core.Compiler: get_world_counter, get_world_counter as get_inference_world -end +import Core.Compiler: get_inference_world +using Base: get_world_counter -const MTType = Core.MethodTable if isdefined(Core.Compiler, :CachedMethodTable) using Core.Compiler: CachedMethodTable maybe_cached(mtv::CC.MethodTableView) = CachedMethodTable(mtv) @@ -407,86 +147,49 @@ get_method_table_view(world::UInt, mt::CC.MethodTable) = CC.OverlayMethodTable(w # VERSION >= v"1.14.0-DEV.1691" const INFERENCE_CACHE_TYPE = isdefined(CC, :InferenceCache) ? CC.InferenceCache : Vector{CC.InferenceResult} -struct GPUInterpreter{MTV<:CC.MethodTableView} <: CC.AbstractInterpreter +""" + GPUInterpreter{MTV, K, V} + +Foreign abstract interpreter that drives Julia inference for GPU compilation. Parametric +on the method-table view (`MTV`), the cache owner type (`K`), and the consumer's results +struct type (`V`). The cache is a `CompilerCaching.CacheView{K, V}` — `@setup_caching` +wires `cache_owner` and `finish!` so each `CodeInstance` carries a `V()` results struct +on its `analysis_results` chain, partitioned in the global CI cache by the owner token. +""" +struct GPUInterpreter{MTV<:CC.MethodTableView, K, V} <: CC.AbstractInterpreter world::UInt method_table_view::MTV - -@static if HAS_INTEGRATED_CACHE - token::Any -else - code_cache::CodeCache -end + cache::CacheView{K, V} inf_cache::INFERENCE_CACHE_TYPE - inf_params::CC.InferenceParams opt_params::CC.OptimizationParams end -@static if HAS_INTEGRATED_CACHE -function GPUInterpreter(world::UInt=Base.get_world_counter(); - method_table_view::CC.MethodTableView, - token::Any, +function GPUInterpreter(world::UInt; method_table_view::CC.MethodTableView, + cache::CacheView, inf_params::CC.InferenceParams, opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - - inf_cache = INFERENCE_CACHE_TYPE() - - return GPUInterpreter(world, method_table_view, - token, inf_cache, - inf_params, opt_params) + GPUInterpreter(world, method_table_view, cache, INFERENCE_CACHE_TYPE(), + inf_params, opt_params) end function GPUInterpreter(interp::GPUInterpreter; world::UInt=interp.world, method_table_view::CC.MethodTableView=interp.method_table_view, - token::Any=interp.token, + cache::CacheView=interp.cache, inf_cache::INFERENCE_CACHE_TYPE=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, opt_params::CC.OptimizationParams=interp.opt_params) - return GPUInterpreter(world, method_table_view, - token, inf_cache, - inf_params, opt_params) -end - -else - -function GPUInterpreter(world::UInt=Base.get_world_counter(); - method_table_view::CC.MethodTableView, - code_cache::CodeCache, - inf_params::CC.InferenceParams, - opt_params::CC.OptimizationParams) - @assert world <= Base.get_world_counter() - - inf_cache = Vector{CC.InferenceResult}() - - return GPUInterpreter(world, method_table_view, - code_cache, inf_cache, - inf_params, opt_params) -end - -function GPUInterpreter(interp::GPUInterpreter; - world::UInt=interp.world, - method_table_view::CC.MethodTableView=interp.method_table_view, - code_cache::CodeCache=interp.code_cache, - inf_cache::Vector{CC.InferenceResult}=interp.inf_cache, - inf_params::CC.InferenceParams=interp.inf_params, - opt_params::CC.OptimizationParams=interp.opt_params) - return GPUInterpreter(world, method_table_view, - code_cache, inf_cache, - inf_params, opt_params) + GPUInterpreter(world, method_table_view, cache, inf_cache, inf_params, opt_params) end -end # HAS_INTEGRATED_CACHE CC.InferenceParams(interp::GPUInterpreter) = interp.inf_params CC.OptimizationParams(interp::GPUInterpreter) = interp.opt_params -#=CC.=#get_inference_world(interp::GPUInterpreter) = interp.world +get_inference_world(interp::GPUInterpreter) = interp.world CC.get_inference_cache(interp::GPUInterpreter) = interp.inf_cache -@static if HAS_INTEGRATED_CACHE - CC.cache_owner(interp::GPUInterpreter) = interp.token -else - CC.code_cache(interp::GPUInterpreter) = WorldView(interp.code_cache, interp.world) -end + +@setup_caching GPUInterpreter.cache # No need to do any locking since we're not putting our results into the runtime cache CC.lock_mi_inference(interp::GPUInterpreter, mi::MethodInstance) = nothing @@ -507,8 +210,6 @@ CC.method_table(interp::GPUInterpreter) = interp.method_table_view # semi-concrete interepretation is broken with overlays (JuliaLang/julia#47349) function CC.concrete_eval_eligible(interp::GPUInterpreter, @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) - # NOTE it's fine to skip overloading with `sv::IRInterpretationState` since we disables - # semi-concrete interpretation anyway. ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) if ret === :semi_concrete_eval @@ -516,172 +217,17 @@ function CC.concrete_eval_eligible(interp::GPUInterpreter, end return ret end -function CC.concrete_eval_eligible(interp::GPUInterpreter, - @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo) - ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, - f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo) - ret === false && return nothing - return ret -end - - -## world view of the cache -@static if VERSION < v"1.14-" - using Core.Compiler: WorldView -end - -if !HAS_INTEGRATED_CACHE - -function CC.haskey(wvc::WorldView{CodeCache}, mi::MethodInstance) - CC.get(wvc, mi, nothing) !== nothing -end -function CC.get(wvc::WorldView{CodeCache}, mi::MethodInstance, default) - # check the cache - for ci in get!(wvc.cache.dict, mi, CodeInstance[]) - if ci.min_world <= wvc.worlds.min_world && wvc.worlds.max_world <= ci.max_world - # TODO: if (code && (code == jl_nothing || jl_ir_flag_inferred((jl_array_t*)code))) - src = if ci.inferred isa Vector{UInt8} - ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), - mi.def, C_NULL, ci.inferred) - else - ci.inferred - end - return ci - end - end - - return default -end - -function CC.getindex(wvc::WorldView{CodeCache}, mi::MethodInstance) - r = CC.get(wvc, mi, nothing) - r === nothing && throw(KeyError(mi)) - return r::CodeInstance -end - -function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodInstance) - CC.setindex!(wvc.cache, ci, mi) -end - -end # HAS_INTEGRATED_CACHE ## codegen/inference integration -function ci_cache_populate(interp, cache, mi, min_world, max_world) - codeinfos = Pair{CodeInstance, CodeInfo}[] - @static if VERSION >= v"1.12.0-DEV.1434" - # see typeinfer.jl: typeinf_ext_toplevel - has_compilequeue = VERSION >= v"1.13.0-DEV.499" || v"1.12-beta3" <= VERSION < v"1.13-" - ci = CC.typeinf_ext(interp, mi, CC.SOURCE_MODE_NOT_REQUIRED) - if has_compilequeue - workqueue = CC.CompilationQueue(; interp) - push!(workqueue, ci) - else - workqueue = CodeInstance[ci] - inspected = IdSet{CodeInstance}() - end - while !isempty(workqueue) - callee = pop!(workqueue) - if has_compilequeue - CC.isinspected(workqueue, callee) && continue - CC.markinspected!(workqueue, callee) - else - callee in inspected && continue - push!(inspected, callee) - end - - # now make sure everything has source code, if desired - mi = CC.get_ci_mi(callee) - if CC.use_const_api(callee) - if VERSION >= v"1.13.0-DEV.1121" - src = CC.codeinfo_for_const(interp, mi, CC.WorldRange(callee.min_world, callee.max_world), callee.edges, callee.rettype_const) - else - src = CC.codeinfo_for_const(interp, mi, callee.rettype_const) - end - else - # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here - src = CC.typeinf_code(interp, mi, true) - end - if src isa CodeInfo - if has_compilequeue - sptypes = CC.sptypes_from_meth_instance(mi) - CC.collectinvokes!(workqueue, src, sptypes) - else - CC.collectinvokes!(workqueue, src) - end - push!(codeinfos, callee => src) - end - end - elseif VERSION >= v"1.12.0-DEV.15" - inferred_ci = CC.typeinf_ext_toplevel(interp, mi, CC.SOURCE_MODE_FORCE_SOURCE) - @assert inferred_ci !== nothing "Inference of $mi failed" - - # inference should have populated our cache - wvc = WorldView(cache, min_world, max_world) - @assert CC.haskey(wvc, mi) "GPUCompiler: Failed to compile method for $mi, between worlds $min_world and $max_world" - ci = CC.getindex(wvc, mi) - - # if ci is rettype_const, the inference result won't have been cached - # (because it is normally not supposed to be used ever again). - # to avoid the need to re-infer, set that field here. - if ci.inferred === nothing - CC.setindex!(wvc, inferred_ci, mi) - ci = CC.getindex(wvc, mi) - end - else - src = CC.typeinf_ext_toplevel(interp, mi) - - # inference should have populated our cache - wvc = WorldView(cache, min_world, max_world) - - @assert CC.haskey(wvc, mi) "GPUCompiler: Failed to compile method for $mi, between worlds $min_world and $max_world" - ci = CC.getindex(wvc, mi) - - # if ci is rettype_const, the inference result won't have been cached - # (because it is normally not supposed to be used ever again). - # to avoid the need to re-infer, set that field here. - if ci.inferred === nothing - @atomic ci.inferred = src - end - end - - return codeinfos -end - -@static if VERSION >= v"1.14-" -function ci_cache_lookup(cache, mi, min_world, max_world) - # In Julia 1.14+, WorldView was replaced by InternalCodeCache with WorldRange - # cache is OverlayCodeCache{InternalCodeCache}, extract owner from globalcache - owner = cache.globalcache.owner - wvc = CC.InternalCodeCache(owner, CC.WorldRange(min_world, max_world)) - ci = CC.get(wvc, mi, nothing) - return ci -end -else -function ci_cache_lookup(cache, mi, min_world, max_world) - wvc = WorldView(cache, min_world, max_world) - ci = CC.get(wvc, mi, nothing) - if VERSION < v"1.12.0-DEV.1434" && ci !== nothing && ci.inferred === nothing - # if for some reason we did end up with a codeinfo without inferred source, e.g., - # because of calling `Base.return_types` which only sets rettyp, pretend we didn't - # run inference so that we re-infer now and not during codegen (which is disallowed) - return nothing - end - return ci -end -end # @static if - - -## interface - -# for platforms without @cfunction-with-closure support -const _method_instances = Ref{Any}() -const _cache = Ref{Any}() -function _lookup_fun(mi, min_world, max_world) - push!(_method_instances[], mi) - ci_cache_lookup(_cache[], mi, min_world, max_world) -end +const HAS_LLVM_GET_CIS = ( + VERSION >= v"1.13.0-DEV.1120" || ( + Libdl.dlsym( + unsafe_load(cglobal(:jl_libjulia_handle, Ptr{Cvoid})), :jl_get_llvm_cis, throw_error = false + ) !== nothing + ) +) @enum CompilationPolicy::Cint begin CompilationPolicyDefault = 0 @@ -698,11 +244,8 @@ function Base.precompile(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - - # populate the cache interp = get_interpreter(job) - cache = CC.code_cache(interp) - ci_cache_populate(interp, cache, job.source, job.world, job.world) + CompilerCaching.typeinf!(interp.cache, interp, job.source) return true end @@ -755,29 +298,38 @@ function record_coverage(mi::MethodInstance, src::CodeInfo) return end -const HAS_LLVM_GET_CIS = ( - VERSION >= v"1.13.0-DEV.1120" || ( - Libdl.dlsym( - unsafe_load(cglobal(:jl_libjulia_handle, Ptr{Cvoid})), :jl_get_llvm_cis, throw_error = false - ) !== nothing - ) -) +# for platforms without @cfunction-with-closure support +const _method_instances = Ref{Any}() +const _cache = Ref{Any}() +function _lookup_fun(mi, min_world, max_world) + push!(_method_instances[], mi) + cache = _cache[]::CacheView + get(cache, mi, nothing) +end function compile_method_instance(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - # populate the cache + # populate the cache (inference) interp = get_interpreter(job) - cache = CC.code_cache(interp) - populated = ci_cache_populate(interp, cache, job.source, job.world, job.world) + cache = interp.cache + CompilerCaching.typeinf!(cache, interp, job.source) + ci = get(cache, job.source, nothing)::CodeInstance + + # collect (CI, CodeInfo) pairs for jl_emit_native + @static if VERSION >= v"1.12.0-DEV.1823" + codeinfo_pairs = CompilerCaching.get_codeinfos(ci) + else + codeinfo_pairs = nothing + end # record line coverage of all compiled code (on older versions of Julia, where - # `ci_cache_populate` does not return sources, this happens after codegen instead) - if Base.JLOptions().code_coverage != 0 - for (ci, src) in populated - record_coverage(ci.def::MethodInstance, src) + # inference does not return sources, this happens after codegen instead) + if Base.JLOptions().code_coverage != 0 && codeinfo_pairs !== nothing + for (ci′, src) in codeinfo_pairs + record_coverage(ci′.def::MethodInstance, src::CodeInfo) end end @@ -787,7 +339,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if Sys.ARCH == :x86 || Sys.ARCH == :x86_64 function lookup_fun(mi, min_world, max_world) push!(method_instances, mi) - ci_cache_lookup(cache, mi, min_world, max_world) + get(cache, mi, nothing) end lookup_cb = @cfunction($lookup_fun, Any, (Any, UInt, UInt)) else @@ -831,10 +383,8 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) native_code = if VERSION >= v"1.12.0-DEV.1823" codeinfos = Any[] - for (ci, src) in populated - # each item in the list should be a CodeInstance followed by a CodeInfo - # indicating something to compile - push!(codeinfos, ci::CodeInstance) + for (ci′, src) in codeinfo_pairs + push!(codeinfos, ci′::CodeInstance) push!(codeinfos, src::CodeInfo) end @ccall jl_emit_native(codeinfos::Vector{Any}, ts_mod::LLVM.API.LLVMOrcThreadSafeModuleRef, Ref(params)::Ptr{Base.CodegenParams}, #=extern linkage=# false::Cint)::Ptr{Cvoid} @@ -856,22 +406,15 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # XXX: this is wrong; we can't expose the underlying LLVM module, but should # instead always go through the callback in order to unlock it properly. - # rework this once we depend on Julia 1.9 or later. llvm_ts_mod = LLVM.ThreadSafeModule(llvm_mod_ref) llvm_mod = nothing llvm_ts_mod() do mod llvm_mod = mod end end - if !(Sys.ARCH == :x86 || Sys.ARCH == :x86_64) - cache_gbl = nothing - end # Since Julia 1.13, the caller is responsible for initializing global variables that # point to global values or bindings with their address in memory. - # Similarly on previous versions when imaging=true, it is also the caller's responsibility - # (see https://github.com/JuliaGPU/GPUCompiler.jl/issues/753), but we can support this on versions - # that have HAS_LLVM_GVS_GLOBALS. gvs = nothing inits = nothing @static if VERSION >= v"1.13.0-DEV.623" @@ -909,17 +452,9 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # Maintain a map from global variables to their initialized Julia values. # The objects pointed to are perma-rooted, during codegen. - # It is legal to call `Base.unsafe_pointer_to_objref` on `values(gv_to_value)`, - # but x->pointer_from_objref(Base.unsafe_pointer_to_objref(x)) is not idempotent, - # thus we store raw pointers here. - # Currently GVs are privatized, so users may have to handle embedded pointers, - # but this dictionary provides a clear indication that the embedded pointer is - # indeed avalid Julia object. gv_to_value = Dict{String, Ptr{Cvoid}}() - # On certain version of Julia we have no reliable way to match the `gvs` to their initializers `inits`. if gvs === nothing - # global variables here properly. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -945,8 +480,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init - # set the initializer - # TODO(vc): To enable full relocation we should actually strip out the initializers here. if LLVM.isnull(initializer(gv)) val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) initializer!(gv, val) @@ -958,9 +491,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if HAS_LLVM_GET_CIS # on sufficiently recent versions of Julia, we can query the CIs compiled. - # this is required after the move to `invoke(::CodeInstance)`, because our - # lookup function (used to populate method_instances) isn't always called then. - num_cis = Ref{Csize_t}(0) @ccall jl_get_llvm_cis(native_code::Ptr{Cvoid}, num_cis::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing @@ -970,7 +500,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) )::Nothing elseif VERSION >= v"1.12.0-DEV.1703" # slightly older versions of Julia used MIs directly - num_mis = Ref{Csize_t}(0) @ccall jl_get_llvm_mis(native_code::Ptr{Cvoid}, num_mis::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing @@ -981,53 +510,47 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if !HAS_LLVM_GET_CIS for mi in method_instances - ci = ci_cache_lookup(cache, mi, job.world, job.world) - ci === nothing && continue + ci′ = get(cache, mi, nothing) + ci′ === nothing && continue llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall( :jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), - native_code, ci, llvm_func_idx, llvm_specfunc_idx + native_code, ci′, llvm_func_idx, llvm_specfunc_idx ) - # Suppose we have two nested interpreters in use at the same time. - # Looking up a ci from the cache is not unique for a given mi. - # Consequently its possible we may not have compiled the ci found - # by the cache (instead having compiled the ci from the other interp). if llvm_func_idx[] == -1 continue end - push!(code_instances, ci) + push!(code_instances, ci′) end else # `jl_get_llvm_cis` can report stale CIs that no longer cover the world # this job was compiled in. The explicit cache lookup path already filters # by world; do the same here before de-duplicating by MI. - filter!(code_instances) do ci - ci.min_world <= job.world <= ci.max_world + filter!(code_instances) do ci′ + ci′.min_world <= job.world <= ci′.max_world end - # To avoid a clash in the compiled cache containing both with an interpreter token (like GPUCompiler.GPUCompilerCacheToken) and native, + # To avoid a clash in the compiled cache containing both an interpreter token and native, # prefer the non-native code-instance. - # TODO: in the future we should migrate compiled to have the ci as the key, not the mi. owned_mis = Set{MethodInstance}() - for ci in code_instances - if ci.owner !== nothing - push!(owned_mis, ci.def::MethodInstance) + for ci′ in code_instances + if ci′.owner !== nothing + push!(owned_mis, ci′.def::MethodInstance) end end - filter!(code_instances) do ci - return ci.owner !== nothing || ci.def ∉ owned_mis + filter!(code_instances) do ci′ + return ci′.owner !== nothing || ci′.def ∉ owned_mis end end - # Avoid redundant code_instances. This is necessary to avoid false positives trying to add the same key'd mi to the compiled Dict. unique!(code_instances) # record line coverage of all compiled code (on newer versions of Julia, this happens - # based on the sources returned by `ci_cache_populate` instead) - if Base.JLOptions().code_coverage != 0 && isempty(populated) + # based on the sources returned by inference instead) + if Base.JLOptions().code_coverage != 0 && codeinfo_pairs === nothing for ci in code_instances src = ci.inferred if src isa String @@ -1040,20 +563,19 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end resize!(method_instances, length(code_instances)) - for (i, ci) in enumerate(code_instances) - method_instances[i] = ci.def::MethodInstance + for (i, ci′) in enumerate(code_instances) + method_instances[i] = ci′.def::MethodInstance end # process all compiled method instances compiled = Dict() - for (ci, mi) in zip(code_instances, method_instances) - + for (ci′, mi) in zip(code_instances, method_instances) # get the function index llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall(:jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), - native_code, ci, llvm_func_idx, llvm_specfunc_idx) + native_code, ci′, llvm_func_idx, llvm_specfunc_idx) @assert llvm_func_idx[] != -1 || llvm_specfunc_idx[] != -1 "Static compilation failed" # get the function @@ -1079,7 +601,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # NOTE: it's not safe to store raw LLVM functions here, since those may get # removed or renamed during optimization, so we store their name instead. - compiled[mi] = (; ci, func=llvm_func, specfunc=llvm_specfunc) + compiled[mi] = (; ci=ci′, func=llvm_func, specfunc=llvm_specfunc) end # ensure that the requested method instance was compiled @@ -1087,18 +609,3 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end - -# partially revert JuliaLangjulia#49391 -@static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master - !(v"1.11-beta2" <= VERSION < v"1.12") # reverted on 1.11-beta2 -function CC.typeinf(interp::GPUInterpreter, frame::CC.InferenceState) - if CC.__measure_typeinf__[] - CC.Timings.enter_new_timer(frame) - v = CC._typeinf(interp, frame) - CC.Timings.exit_current_timer(frame) - return v - else - return CC._typeinf(interp, frame) - end -end -end diff --git a/src/optim.jl b/src/optim.jl index 7a07d200..5672daa9 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -358,13 +358,10 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op end add!(fpm, GCInvariantVerifierPass()) add!(fpm, LateLowerGCPass()) - if uses_julia_runtime(job) && VERSION >= v"1.11.0-DEV.208" + if uses_julia_runtime(job) add!(fpm, FinalLowerGCPass()) end end - if uses_julia_runtime(job) && VERSION < v"1.11.0-DEV.208" - add!(mpm, FinalLowerGCPass()) - end if opt_level >= 2 add!(mpm, NewPMFunctionPassManager()) do fpm diff --git a/src/rtlib.jl b/src/rtlib.jl index 85cb1249..bcd90883 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -120,12 +120,14 @@ else is_asserts() = false end -@locked function load_runtime(@nospecialize(job::CompilerJob)) - global compile_cache - if compile_cache === nothing # during precompilation - return build_runtime(job) - end +# in-memory cache of serialized runtime libraries, keyed by `runtime_slug(job)` +# (parametrized by opaque-pointer support). Survives package precompilation by virtue +# of being plain bitcode bytes; native modules are reparsed on demand because LLVM +# modules are tied to a context. +const _runtime_libs = Dict{String, Vector{UInt8}}() +const _runtime_libs_lock = ReentrantLock() +@locked function load_runtime(@nospecialize(job::CompilerJob)) slug = runtime_slug(job) if !supports_typed_pointers(context()) slug *= "-opaque" @@ -136,40 +138,15 @@ end slug *= "-asserts" end - name = "runtime_$(slug).bc" - path = joinpath(compile_cache, name) - - # the cache is shared across processes and may disappear at any point - # (e.g. `reset_runtime()` in another process), so treat it as best-effort - if ispath(path) - try - return parse(LLVM.Module, MemoryBufferFile(path); lazy=true) - catch err - @debug "Failed to load cached GPU runtime library; rebuilding" exception=(err, catch_backtrace()) - end - end - - @debug "Building the GPU runtime library at $path" - lib = build_runtime(job) - - try - # atomic write to disk - mkpath(compile_cache) - temp_path, io = mktemp(compile_cache; cleanup=false) + bytes = Base.@lock _runtime_libs_lock get!(_runtime_libs, slug) do + lib = build_runtime(job) + io = IOBuffer() write(io, lib) - close(io) - @static if VERSION >= v"1.12.0-DEV.1023" - mv(temp_path, path; force=true) - else - Base.rename(temp_path, path, force=true) - end - catch err - @warn "Failed to cache GPU runtime library" exception=(err, catch_backtrace()) maxlog=1 + take!(io) end - return lib + return parse(LLVM.Module, MemoryBuffer(bytes); lazy=true) end -# remove the existing cache -# NOTE: call this function from global scope, so any change triggers recompilation. -reset_runtime() = rm(compile_cache; recursive=true, force=true) +# clear the in-memory runtime library cache +reset_runtime() = Base.@lock _runtime_libs_lock empty!(_runtime_libs) diff --git a/src/utils.jl b/src/utils.jl index e8938a8f..1f18edf0 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -99,20 +99,9 @@ for level in [:debug, :info, :warn, :error] # using with_logger would create a closure, which is incompatible with # generated functions, so instead we reproduce its implementation here safe_logstate = Base.CoreLogging.LogState(safe_logger) - @static if VERSION < v"1.11-" - t = current_task() - old_logstate = t.logstate - try - t.logstate = safe_logstate - $(esc(macrocall)) - finally - t.logstate = old_logstate - end - else - Base.ScopedValues.@with( - Base.CoreLogging.CURRENT_LOGSTATE => safe_logstate, $(esc(macrocall)) - ) - end + Base.ScopedValues.@with( + Base.CoreLogging.CURRENT_LOGSTATE => safe_logstate, $(esc(macrocall)) + ) end end end diff --git a/src/validation.jl b/src/validation.jl index c1a86549..c08f1fd2 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -14,18 +14,7 @@ function method_matches(@nospecialize(tt::Type{<:Tuple}); world::Integer) end function typeinf_type(mi::MethodInstance; interp::CC.AbstractInterpreter) - @static if VERSION < v"1.11.0" - code = Core.Compiler.get(Core.Compiler.code_cache(interp), mi, nothing) - if code isa Core.Compiler.CodeInstance - return code.rettype - end - result = Core.Compiler.InferenceResult(mi, Core.Compiler.typeinf_lattice(interp)) - Core.Compiler.typeinf(interp, result, :global) - Core.Compiler.is_inferred(result) || return Any - Core.Compiler.widenconst(Core.Compiler.ignorelimited(result.result)) - else - something(Core.Compiler.typeinf_type(interp, mi), Any) - end + something(Core.Compiler.typeinf_type(interp, mi), Any) end function check_method(@nospecialize(job::CompilerJob)) From b5f742f64fcc3eb5d2f0090b3757f4fd3bbd5a8d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 10 May 2026 08:50:48 +0200 Subject: [PATCH 02/33] Widen cache_owner to include the target+params instances. Previously partitioned only by `typeof(target)`, which collided across target instances that produce different IR (e.g. different macOS, SM arch, or CPU features). Folding the full target and params into the token matches the spirit of `runtime_slug`, while staying within the inference-determinant scope of the docstring. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/interface.jl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 87f9733f..d90c4979 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -302,20 +302,25 @@ pass_by_ref(@nospecialize(job::CompilerJob)) = false valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false # Cache partitioning. The owner is stored on every CodeInstance and compared via `jl_egal`, -# so it must be immutable for cross-session matches (e.g. via package precompilation). +# so it (and every field) must be immutable for cross-session matches (e.g. via package +# precompilation); custom `target` / `params` types must be `struct`s, not `mutable struct`s. # Care is required for anything that impacts: # - method_table # - inference_params # - optimization_params -# Default covers `always_inline` (which feeds optimization_params) and the method table. -struct GPUCompilerCacheToken - target_type::Type +# The default covers the full target+params instances (so backends with version- or +# arch-specific knobs partition cleanly), `always_inline` (which feeds optimization_params), +# and the method table. +struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams} + target::T + params::P always_inline::Bool method_table::Core.MethodTable end cache_owner(@nospecialize(job::CompilerJob)) = - GPUCompilerCacheToken(typeof(job.config.target), job.config.always_inline, method_table(job)) + GPUCompilerCacheToken(job.config.target, job.config.params, + job.config.always_inline, method_table(job)) """ GPUCompiler.NoResults() From 13e816fc93202b89797675aa4f10d7ff0052a7e9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 10 May 2026 12:30:57 +0200 Subject: [PATCH 03/33] Cache runtime library bitcode on the per-function CodeInstance. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds optional `bitcode`/`bitcode!` hooks on the consumer's `results_type`. When opted in, `emit_function!` reads renamed per-function bitcode from the cached CI on a hit, and writes it on a miss. Cross-session persistence rides on package precompilation; a small session-local assembled-module cache (keyed by `(cache_owner, opaque_pointers)`) keeps the within-session fast path. Drops the `runtime_slug` interface — `cache_owner` now subsumes its role of identifying compatible IR. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/bpf.jl | 2 -- src/gcn.jl | 4 --- src/interface.jl | 17 +++++---- src/metal.jl | 6 ---- src/native.jl | 1 - src/ptx.jl | 6 ---- src/rtlib.jl | 90 +++++++++++++++++++++++++++++------------------- src/spirv.jl | 5 --- 8 files changed, 65 insertions(+), 66 deletions(-) diff --git a/src/bpf.jl b/src/bpf.jl index 80070d6d..ff75ea23 100644 --- a/src/bpf.jl +++ b/src/bpf.jl @@ -26,8 +26,6 @@ end ## job -runtime_slug(job::CompilerJob{BPFCompilerTarget}) = "bpf" - const bpf_intrinsics = () # TODO isintrinsic(::CompilerJob{BPFCompilerTarget}, fn::String) = in(fn, bpf_intrinsics) diff --git a/src/gcn.jl b/src/gcn.jl index be047143..dc06c8b3 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -39,10 +39,6 @@ end ## job -# TODO: encode debug build or not in the compiler job -# https://github.com/JuliaGPU/CUDAnative.jl/issues/368 -runtime_slug(job::CompilerJob{GCNCompilerTarget}) = "gcn-$(job.config.target.dev_isa)$(job.config.target.features)-$(job.config.target.backend)" - const gcn_intrinsics = () # TODO: ("vprintf", "__assertfail", "malloc", "free") isintrinsic(::CompilerJob{GCNCompilerTarget}, fn::String) = in(fn, gcn_intrinsics) diff --git a/src/interface.jl b/src/interface.jl index d90c4979..21ec01ee 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -277,11 +277,6 @@ can_throw(@nospecialize(job::CompilerJob)) = uses_julia_runtime(job) # if not, safepoints at function entry will not be emitted can_safepoint(@nospecialize(job::CompilerJob)) = uses_julia_runtime(job) -# generate a string that represents the type of compilation, for selecting a compiled -# instance of the runtime library. this slug should encode everything that affects -# the generated code of this compiler job (with exception of the function source) -runtime_slug(@nospecialize(job::CompilerJob)) = error("Not implemented") - # the type of the kernel state object, or Nothing if this back-end doesn't need one. # # the generated code will be rewritten to include an object of this type as the first @@ -345,7 +340,17 @@ function cache_view(@nospecialize(job::CompilerJob)) CompilerCaching.CacheView{K, V}(cache_owner(job), job.world) end -public GPUCompilerCacheToken, cache_owner, NoResults, results_type, cache_view +# Optional consumer hooks for caching post-codegen LLVM bitcode on the results struct. +# Override these on your `results_type(job)` to opt in to cross-session bitcode caching +# (most relevant for runtime library functions, which GPUCompiler compiles per-target and +# would otherwise rebuild every session). The `opaque_pointers` flag tracks the LLVM +# context's pointer mode at compile time — `bitcode` should reject mismatches by +# returning `nothing`, since opaque- and typed-pointer IR aren't interchangeable. +bitcode(@nospecialize(results), opaque_pointers::Bool) = nothing +bitcode!(@nospecialize(results), bytes::Vector{UInt8}, opaque_pointers::Bool) = nothing + +public GPUCompilerCacheToken, cache_owner, NoResults, results_type, cache_view, + bitcode, bitcode! # the method table to use # deprecate method_table on next-breaking release diff --git a/src/metal.jl b/src/metal.jl index 0c7dd6aa..413e4a5b 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -106,12 +106,6 @@ have_fma(@nospecialize(target::MetalCompilerTarget), T::Type) = true ## job -# the debug-info level is part of the slug (as on PTX): it changes the debug metadata emitted -# into the runtime library, and the device-exception reporters branch on it, so a library -# built at one level must not be reused at another. -runtime_slug(job::CompilerJob{MetalCompilerTarget}) = - "metal-macos$(job.config.target.macos)-debuginfo=$(Int(llvm_debug_info(job)))" - isintrinsic(@nospecialize(job::CompilerJob{MetalCompilerTarget}), fn::String) = return startswith(fn, "air.") diff --git a/src/native.jl b/src/native.jl index fdd880ec..536c31d4 100644 --- a/src/native.jl +++ b/src/native.jl @@ -33,6 +33,5 @@ end ## job -runtime_slug(job::CompilerJob{NativeCompilerTarget}) = "native_$(job.config.target.cpu)-$(hash(job.config.target.features))$(job.config.target.jlruntime ? "-jlrt" : "")" uses_julia_runtime(job::CompilerJob{NativeCompilerTarget}) = job.config.target.jlruntime can_vectorize(job::CompilerJob{NativeCompilerTarget}) = true diff --git a/src/ptx.jl b/src/ptx.jl index dfde75ad..c425b6bc 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -138,12 +138,6 @@ isintrinsic(@nospecialize(job::CompilerJob{PTXCompilerTarget}), fn::String) = # libdevice's __CUDA_ARCH dispatch, are still supported by the external back-end. startswith(fn, "llvm.nvvm.") -# XXX: the debuginfo part should be handled by GPUCompiler as it applies to all back-ends. -runtime_slug(@nospecialize(job::CompilerJob{PTXCompilerTarget})) = - "ptx$(job.config.target.ptx.major)$(job.config.target.ptx.minor)" * - "-$(cpu_name(job.config.target))" * - "-debuginfo=$(Int(llvm_debug_info(job)))" - function finish_module!(@nospecialize(job::CompilerJob{PTXCompilerTarget}), mod::LLVM.Module, entry::LLVM.Function) # tell NVVMReflect whether to flush denormals; this mirrors what Clang does diff --git a/src/rtlib.jl b/src/rtlib.jl index bcd90883..d5488944 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,9 +60,30 @@ end ## functionality to build the runtime library -function emit_function!(mod, config::CompilerConfig, f, method) +# Compile a single runtime function and link it into `mod`. If `cache` is provided and the +# consumer's results type opts in via `GPUCompiler.bitcode`/`bitcode!`, per-function bitcode +# is read from / written to the cached `CodeInstance` to avoid recompilation across sessions. +function emit_function!(mod, config::CompilerConfig, f, method, + cache::Union{Nothing, CacheView}=nothing) tt = Base.to_tuple_type(method.types) source = generic_methodinstance(f, tt) + name = method.llvm_name + opaque_pointers = !supports_typed_pointers(context()) + + # fast path: pull renamed bitcode straight from the cached CI + if cache !== nothing + hit = CompilerCaching.lookup(cache, source) + if hit !== nothing + cached = bitcode(hit[2], opaque_pointers) + if cached !== nothing + func_mod = parse(LLVM.Module, MemoryBuffer(cached)) + link!(mod, func_mod) + return + end + end + end + + # slow path: compile, rename, optionally cache, link new_mod, meta = compile_unhooked(:llvm, CompilerJob(source, config)) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) @@ -73,20 +94,26 @@ function emit_function!(mod, config::CompilerConfig, f, method) # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - temp_name = LLVM.name(meta.entry) - link!(mod, new_mod) - entry = functions(mod)[temp_name] - - # if a declaration already existed, replace it with the function to avoid aliasing - # (and getting function names like gpu_signal_exception1) - name = method.llvm_name - if haskey(functions(mod), name) - decl = functions(mod)[name] - @assert value_type(decl) == value_type(entry) - replace_uses!(decl, entry) + # rename to the final `gpu_*` name on the per-function module, so the cached bitcode is + # immediately link-ready (no per-session rename pass). + if haskey(functions(new_mod), name) && functions(new_mod)[name] !== meta.entry + decl = functions(new_mod)[name] + @assert value_type(decl) == value_type(meta.entry) + replace_uses!(decl, meta.entry) erase!(decl) end - LLVM.name!(entry, name) + LLVM.name!(meta.entry, name) + + if cache !== nothing + hit = CompilerCaching.lookup(cache, source) + if hit !== nothing + io = IOBuffer() + write(io, new_mod) + bitcode!(hit[2], take!(io), opaque_pointers) + end + end + + link!(mod, new_mod) end function build_runtime(@nospecialize(job::CompilerJob)) @@ -96,6 +123,11 @@ function build_runtime(@nospecialize(job::CompilerJob)) # derive a job that represents the runtime itself (notably with kernel=false). config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, strip=false) + # cache view shared with the runtime functions' compile_unhooked calls — owner is + # determined by target+params+always_inline+method_table, all preserved on `config`. + proto = CompilerJob(job.source, config, job.world) + cache = cache_view(proto) + for method in values(Runtime.methods) def = if isa(method.def, Symbol) isdefined(runtime_module(job), method.def) || continue @@ -103,7 +135,7 @@ function build_runtime(@nospecialize(job::CompilerJob)) else method.def end - emit_function!(mod, config, typeof(def), method) + emit_function!(mod, config, typeof(def), method, cache) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -114,31 +146,17 @@ function build_runtime(@nospecialize(job::CompilerJob)) mod end -@static if VERSION >= v"1.11.0" - import Core.Compiler: is_asserts -else - is_asserts() = false -end - -# in-memory cache of serialized runtime libraries, keyed by `runtime_slug(job)` -# (parametrized by opaque-pointer support). Survives package precompilation by virtue -# of being plain bitcode bytes; native modules are reparsed on demand because LLVM -# modules are tied to a context. -const _runtime_libs = Dict{String, Vector{UInt8}}() +# session-local cache of assembled runtime libraries, keyed by +# `(cache_owner, opaque_pointers)`. Avoids re-running `build_runtime` (which re-parses and +# re-links per-function bitcode) on every kernel compile within a session. Cross-session +# persistence happens at the per-function level via the `bitcode`/`bitcode!` hooks. +const _runtime_libs = Dict{Tuple{Any, Bool}, Vector{UInt8}}() const _runtime_libs_lock = ReentrantLock() @locked function load_runtime(@nospecialize(job::CompilerJob)) - slug = runtime_slug(job) - if !supports_typed_pointers(context()) - slug *= "-opaque" - end - - # Julia codegen changes metadata in modules when `FORCE_ASSERTIONS=1` - if is_asserts() - slug *= "-asserts" - end + key = (cache_owner(job), !supports_typed_pointers(context())) - bytes = Base.@lock _runtime_libs_lock get!(_runtime_libs, slug) do + bytes = Base.@lock _runtime_libs_lock get!(_runtime_libs, key) do lib = build_runtime(job) io = IOBuffer() write(io, lib) @@ -148,5 +166,5 @@ const _runtime_libs_lock = ReentrantLock() return parse(LLVM.Module, MemoryBuffer(bytes); lazy=true) end -# clear the in-memory runtime library cache +# clear the session-local runtime library cache reset_runtime() = Base.@lock _runtime_libs_lock empty!(_runtime_libs) diff --git a/src/spirv.jl b/src/spirv.jl index 3400fef6..8388fc1a 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -58,11 +58,6 @@ llvm_datalayout(::SPIRVCompilerTarget) = Int===Int64 ? ## job -# TODO: encode debug build or not in the compiler job -# https://github.com/JuliaGPU/CUDAnative.jl/issues/368 -runtime_slug(job::CompilerJob{SPIRVCompilerTarget}) = - "spirv-" * String(job.config.target.backend) - function finish_module!(job::CompilerJob{SPIRVCompilerTarget}, mod::LLVM.Module, entry::LLVM.Function) # update calling convention From 554702cde96883c29e8c2bcdf9c2009d463a91e4 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 12 May 2026 11:39:10 +0200 Subject: [PATCH 04/33] Remove 1.10 from CI. --- .buildkite/pipeline.yml | 2 +- .github/workflows/Test.yml | 17 +---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8a990ec2..39e8c997 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -95,7 +95,7 @@ steps: - label: "OpenCL.jl" plugins: - JuliaCI/julia#v1: - version: "1.10" + version: "1.11" - JuliaCI/julia-coverage#v1: codecov: true command: | diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index 5bd4869e..c05109ab 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -19,27 +19,12 @@ jobs: strategy: fail-fast: false matrix: - version: ['1.10', '1.11', '1.12', '1.13-nightly', 'nightly'] + version: ['1.11', '1.12', '1.13-nightly', 'nightly'] os: [ubuntu-24.04, ubuntu-24.04-arm, macOS-15, macOS-15-intel, windows-2025] llvm_args: [''] include: # starting with Julia 1.10, we can enable opaque pointers # from Julia 1.12 on, this is the default. - - version: '1.10' - os: 'ubuntu-24.04' - llvm_args: '--opaque-pointers' - - version: '1.10' - os: 'ubuntu-24.04-arm' - llvm_args: '--opaque-pointers' - - version: '1.10' - os: 'macOS-15' - llvm_args: '--opaque-pointers' - - version: '1.10' - os: 'macOS-15-intel' - llvm_args: '--opaque-pointers' - - version: '1.10' - os: 'windows-2025' - llvm_args: '--opaque-pointers' - version: '1.11' os: 'ubuntu-24.04' llvm_args: '--opaque-pointers' From 283e70e47cc30ccf3da8c54f80232e13b1aa62c0 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 12 May 2026 12:24:54 +0200 Subject: [PATCH 05/33] Fix tests. --- test/helpers/native.jl | 18 -------- test/native.jl | 92 --------------------------------------- test/native/precompile.jl | 24 +--------- test/ptx/precompile.jl | 2 +- 4 files changed, 2 insertions(+), 134 deletions(-) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 656028f4..63d8a0f3 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -69,22 +69,4 @@ function code_execution(@nospecialize(func), @nospecialize(types); kwargs...) end end -const runtime_cache = Dict{Any, Any}() - -function compiler(job) - JuliaContext() do ctx - GPUCompiler.compile(:asm, job) - end -end - -function linker(job, asm) - asm -end - -# simulates cached codegen -function cached_execution(@nospecialize(func), @nospecialize(types); kwargs...) - job, kwargs = create_job(func, types; validate=false, kwargs...) - GPUCompiler.cached_compilation(runtime_cache, job.source, job.config, compiler, linker) -end - end diff --git a/test/native.jl b/test/native.jl index 6e242de0..9574ff7e 100644 --- a/test/native.jl +++ b/test/native.jl @@ -118,98 +118,6 @@ end @check "add i64 %{{[0-9]+}}, 2" GPUCompiler.code_llvm(job) end - - # cached_compilation interface - invocations = Ref(0) - function compiler(job) - invocations[] += 1 - JuliaContext() do ctx - ir, ir_meta = GPUCompiler.compile(:llvm, job) - string(ir) - end - end - linker(job, compiled) = compiled - cache = Dict() - ft = typeof(mod.kernel) - tt = Tuple{Int64} - - # initial compilation - source = methodinstance(ft, tt, Base.get_world_counter()) - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 2" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 1 - - # cached compilation - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 2" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 1 - - # redefinition - @eval mod kernel(i) = child(i)+3 - source = methodinstance(ft, tt, Base.get_world_counter()) - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 2 - - # cached compilation - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 2 - - # redefinition of an unrelated function - @eval mod unrelated(i) = 42 - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 2 - - # redefining child functions - @eval mod @noinline child(i) = i+1 - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 3 - - # cached compilation - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 3 - - # change in configuration - config = CompilerConfig(job.config; name="foobar") - @test @filecheck begin - @check "define i64 @foobar" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, config, compiler, linker) - end - @test invocations[] == 4 - - # tasks running in the background should keep on using the old version - c1, c2 = Condition(), Condition() - function background(job) - local_source = methodinstance(ft, tt, Base.get_world_counter()) - notify(c1) - wait(c2) # wait for redefinition - GPUCompiler.cached_compilation(cache, local_source, job.config, compiler, linker) - end - t = @async Base.invokelatest(background, job) - wait(c1) # make sure the task has started - @eval mod kernel(i) = child(i)+4 - source = methodinstance(ft, tt, Base.get_world_counter()) - ir = Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test contains(ir, r"add i64 %\d+, 4") - notify(c2) # wake up the task - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - fetch(t) - end end @testset "allowed mutable types" begin diff --git a/test/native/precompile.jl b/test/native/precompile.jl index f08a561f..9917eeb7 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -70,7 +70,7 @@ precompile_test_harness("Inference caching") do load_path token = let job, _ = NativeCompiler.Native.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) + GPUCompiler.cache_owner(job) end @test !check_presence(identity_mi, token) @@ -103,27 +103,5 @@ precompile_test_harness("Inference caching") do load_path _, meta = GPUCompiler.compile(:llvm, job) @test haskey(meta.compiled, job.source) end - - GPUCompiler.clear_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == false - - GPUCompiler.enable_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == true - - job, _ = NativeCompiler.Native.create_job(NativeBackend.kernel, (Vector{Int}, Int); validate=false) - @assert job.source == kernel_mi - ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) - @assert ci !== nothing - @assert ci.inferred !== nothing - path = GPUCompiler.cache_file(ci, job.config) - @test path !== nothing - @test !ispath(path) - NativeCompiler.Native.cached_execution(NativeBackend.kernel, (Vector{Int}, Int)) - @test ispath(path) - GPUCompiler.clear_disk_cache!() - @test !ispath(path) - - GPUCompiler.enable_disk_cache!(false) - @test GPUCompiler.disk_cache_enabled() == false end end diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl index 290c60be..853dce8d 100644 --- a/test/ptx/precompile.jl +++ b/test/ptx/precompile.jl @@ -34,7 +34,7 @@ precompile_test_harness("Inference caching") do load_path token = let job, _ = PTX.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) + GPUCompiler.cache_owner(job) end ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing while ci !== nothing From b318df05e3bdd2dce00e4548318077ab06500570 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 10:28:37 +0200 Subject: [PATCH 06/33] Restore 1.10 compat. --- .github/workflows/Test.yml | 2 +- Project.toml | 12 +- ext/GPUCompilerCompilerCachingExt.jl | 38 +++ src/GPUCompiler.jl | 11 + src/deprecated.jl | 241 +++++++++++++ src/driver.jl | 5 +- src/interface.jl | 78 ++--- src/jlgen.jl | 493 ++++++++++++++++++++++----- src/optim.jl | 7 +- src/rtlib.jl | 50 +-- src/utils.jl | 39 ++- src/validation.jl | 9 +- 12 files changed, 801 insertions(+), 184 deletions(-) create mode 100644 ext/GPUCompilerCompilerCachingExt.jl create mode 100644 src/deprecated.jl diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index c05109ab..fe550da4 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - version: ['1.11', '1.12', '1.13-nightly', 'nightly'] + version: ['1.10', '1.11', '1.12', '1.13-nightly', 'nightly'] os: [ubuntu-24.04, ubuntu-24.04-arm, macOS-15, macOS-15-intel, windows-2025] llvm_args: [''] include: diff --git a/Project.toml b/Project.toml index 30456efa..35896497 100644 --- a/Project.toml +++ b/Project.toml @@ -1,13 +1,12 @@ name = "GPUCompiler" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "1.23.0" +version = "2.0.0" authors = ["Tim Besard "] [workspace] projects = ["test"] [deps] -CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" ExprTools = "e2ba6199-217a-4e67-a87a-7c52f15ade04" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" @@ -22,12 +21,17 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [weakdeps] AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444" NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" +[extensions] +GPUCompilerCompilerCachingExt = "CompilerCaching" + + [compat] AMDGPU_LLVM_Backend_jll = "22" -CompilerCaching = "0.2.4" +CompilerCaching = "0.3" ExprTools = "0.1" InteractiveUtils = "1" LLVM = "9.9" @@ -41,4 +45,4 @@ REPL = "1" TOML = "1" Tracy = "0.1.4" UUIDs = "1" -julia = "1.11" +julia = "1.10" diff --git a/ext/GPUCompilerCompilerCachingExt.jl b/ext/GPUCompilerCompilerCachingExt.jl new file mode 100644 index 00000000..e885046d --- /dev/null +++ b/ext/GPUCompilerCompilerCachingExt.jl @@ -0,0 +1,38 @@ +module GPUCompilerCompilerCachingExt + +# GPUCompiler ↔ CompilerCaching glue. +# +# CompilerCaching attaches a `CachedResult{V}` to each newly inferred `CodeInstance`'s +# `analysis_results` chain so back-ends can recover typed results on cache hits. Julia +# only allows writing `CodeInstance.analysis_results` during the `CC.finish!` step of +# inference (it's a const field afterwards on 1.12), so the typed-results attachment +# has to live on a `CC.finish!` override. +# +# We define that override here, parametrically over `GPUInterpreter{V, MTV}`'s results +# type `V`. When `V === Nothing` (the default — no consumer override of `results_type`) +# this is a no-op pass-through to the default `CC.finish!`, so loading CompilerCaching +# alongside a no-results back-end has no inference-time cost. +# +# The extension only loads on Julia ≥ 1.11 (CompilerCaching's `__init__` errors on +# 1.10), so the integrated cache and `analysis_results` are guaranteed to exist here. + +using GPUCompiler: GPUInterpreter +using CompilerCaching: CachedResult +const CC = Core.Compiler + +@static if hasmethod(CC.finish!, Tuple{CC.AbstractInterpreter, CC.InferenceState, UInt, UInt64}) + function CC.finish!(interp::GPUInterpreter{V, MTV}, caller::CC.InferenceState, + validation_world::UInt, time_before::UInt64) where {MTV, V} + V === Nothing || CC.stack_analysis_result!(caller.result, CachedResult{V}(V())) + @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState, + validation_world::UInt, time_before::UInt64) + end +else + function CC.finish!(interp::GPUInterpreter{V, MTV}, + caller::CC.InferenceState) where {MTV, V} + V === Nothing || CC.stack_analysis_result!(caller.result, CachedResult{V}(V())) + @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState) + end +end + +end # module diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index 098f1617..a4661418 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -33,6 +33,16 @@ end const CC = Core.Compiler using Core: MethodInstance, CodeInstance, CodeInfo +# `HAS_INTEGRATED_CACHE` distinguishes 1.11+ (owner-keyed `Core.Compiler.InternalCodeCache`) +# from 1.10 (per-interpreter `CodeCache` IdDict + invalidation callbacks). The two have +# disjoint shapes; everything cache-related fans on this flag. +const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" + +# Optional callback invoked from `compile(...)` / `cached_compilation(...)` before +# compilation runs. Set by `@device_code_*` reflection macros. Defined here (early) +# so the legacy `cached_compilation` in deprecated.jl can reference it. +const compile_hook = Ref{Union{Nothing,Function}}(nothing) + include("utils.jl") include("mangling.jl") @@ -49,6 +59,7 @@ include("metal.jl") include("runtime.jl") # compiler implementation +include("deprecated.jl") include("jlgen.jl") include("irgen.jl") include("optim.jl") diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 00000000..ad84b04c --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,241 @@ +# Legacy and deprecated functionality. +# +# This file houses code paths that are kept for Julia 1.10 (which lacks the integrated +# CodeInstance cache and `analysis_results`) plus older interfaces retained for the 2.0 +# breaking release. Everything here is expected to go away once 1.10 falls off the LTS +# line and downstream packages have migrated. + +# `WorldView` is used by the integrated-cache JIT lookup callback (1.11–1.13) and by the +# legacy `CodeCache` interface (1.10). Moved out of `Core.Compiler` on 1.14+. +@static if VERSION < v"1.14-" + using Core.Compiler: WorldView +end + + +## Julia 1.10: in-process `CodeCache` + WorldView + +@static if !HAS_INTEGRATED_CACHE + +""" + CodeCache + +Per-job CodeInstance store used on Julia 1.10, where there's no integrated cache and no +`cache_owner` partition on `CodeInstance`. Inference deposits CIs via a `WorldView` over +this store; invalidation callbacks attached to each `MethodInstance` clip `max_world` on +stale CIs when methods are redefined. +""" +struct CodeCache + dict::IdDict{MethodInstance, Vector{CodeInstance}} + + CodeCache() = new(IdDict{MethodInstance, Vector{CodeInstance}}()) +end + +function Base.show(io::IO, ::MIME"text/plain", cc::CodeCache) + print(io, "CodeCache with $(mapreduce(length, +, values(cc.dict); init=0)) entries") + if !isempty(cc.dict) + print(io, ": ") + for (mi, cis) in cc.dict + println(io) + print(io, " ") + show(io, mi) + for ci in cis + println(io) + worldstr = if ci.min_world == typemax(UInt) + "empty world range" + elseif ci.max_world == typemax(UInt) + "worlds $(Int(ci.min_world))+" + else + "worlds $(Int(ci.min_world)) to $(Int(ci.max_world))" + end + print(io, " CodeInstance for ", worldstr) + end + end + end +end + +Base.empty!(cc::CodeCache) = empty!(cc.dict) + +# Session-local registry of `CodeCache`s, keyed by the configuration that produced them. +# 1.10 has no per-CI partitioning, so we partition by `cache_owner(job)` ourselves. +const GLOBAL_CI_CACHES = Dict{Any, CodeCache}() +const GLOBAL_CI_CACHES_LOCK = ReentrantLock() + +function get_code_cache(@nospecialize(job::CompilerJob)) + key = cache_owner(job) + Base.@lock GLOBAL_CI_CACHES_LOCK get!(GLOBAL_CI_CACHES, key) do + CodeCache() + end +end + +# Invalidation: keep `max_world` honest when callees in a CI's edge graph change. +struct CodeCacheCallback + cache::CodeCache +end + +function CC.setindex!(cache::CodeCache, ci::CodeInstance, mi::MethodInstance) + add_codecache_callback!(cache, mi) + cis = get!(cache.dict, mi, CodeInstance[]) + push!(cis, ci) +end + +@static if VERSION ≥ v"1.11.0-DEV.798" + +function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) + callback = CodeCacheCallback(cache) + CC.add_invalidation_callback!(callback, mi) +end +function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32) + cis = get(callback.cache.dict, replaced, nothing) + cis === nothing && return + for ci in cis + if ci.max_world == ~0 % Csize_t + @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" +@static if VERSION >= v"1.11.0-DEV.1390" + @atomic ci.max_world = max_world +else + ci.max_world = max_world +end + end + @assert ci.max_world <= max_world + end +end + +else + +function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) + callback = CodeCacheCallback(cache) + if !isdefined(mi, :callbacks) + mi.callbacks = Any[callback] + elseif !in(callback, mi.callbacks) + push!(mi.callbacks, callback) + end +end +function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32, + seen::Set{MethodInstance}=Set{MethodInstance}()) + push!(seen, replaced) + + cis = get(callback.cache.dict, replaced, nothing) + if cis !== nothing + for ci in cis + if ci.max_world == ~0 % Csize_t + @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" + ci.max_world = max_world + end + @assert ci.max_world <= max_world + end + end + + # recurse into backedges to update their valid range too + if isdefined(replaced, :backedges) + backedges = filter(replaced.backedges) do @nospecialize(mi) + if mi isa MethodInstance + mi ∉ seen + elseif mi isa Type + # an `invoke` call, which is a `(sig, MethodInstance)` pair. + # let's ignore the `sig` and process the `MethodInstance` next. + false + else + error("invalid backedge") + end + end + + # Don't touch/empty backedges — `invalidate_method_instance` in C does that later. + + for mi in backedges + callback(mi::MethodInstance, max_world, seen) + end + end +end + +end + +## 1.10 cache view interface + +function CC.haskey(wvc::WorldView{CodeCache}, mi::MethodInstance) + CC.get(wvc, mi, nothing) !== nothing +end + +function CC.get(wvc::WorldView{CodeCache}, mi::MethodInstance, default) + for ci in get!(wvc.cache.dict, mi, CodeInstance[]) + if ci.min_world <= wvc.worlds.min_world && wvc.worlds.max_world <= ci.max_world + return ci + end + end + return default +end + +function CC.getindex(wvc::WorldView{CodeCache}, mi::MethodInstance) + r = CC.get(wvc, mi, nothing) + r === nothing && throw(KeyError(mi)) + return r::CodeInstance +end + +function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodInstance) + CC.setindex!(wvc.cache, ci, mi) +end + +end # !HAS_INTEGRATED_CACHE + + +## Legacy `cached_compilation` (1.10+) + +# A session-local, MI-keyed kernel cache modeled after the pre-CompilerCaching API. Used +# by back-ends that haven't migrated to the `CacheView`-based flow (and by all back-ends +# on Julia 1.10, where the new flow doesn't apply because there's no integrated cache / +# `analysis_results`). + +""" + cached_compilation(cache::AbstractDict, src::MethodInstance, cfg::CompilerConfig, + compiler, linker) + +Compile a method instance `src` with configuration `cfg`, by invoking `compiler` and +`linker` and storing the result in `cache`. + +The `cache` argument should be a dictionary that can be indexed using any value and store +whatever the `linker` function returns. The `compiler` function should take a +`CompilerJob` and return data that the `linker` function then turns into a session-local +artifact (e.g. a `CuModule`). + +This is the legacy caching API used before GPUCompiler 2.0. New code on Julia 1.11+ +should prefer `CompilerCaching.CacheView`-based caching (see the package extension). +""" +function cached_compilation(cache::AbstractDict{<:Any,V}, + src::MethodInstance, cfg::CompilerConfig, + compiler::Function, linker::Function) where {V} + world = tls_world_age() + key = (objectid(src), world, cfg) + # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive boxing + # allocation. Base does this with a multi-level lookup; we use a single-level dict. + + Base.@lock _cached_compilation_lock begin + obj = get(cache, key, nothing) + end + + if obj === nothing || compile_hook[] !== nothing + obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V + Base.@lock _cached_compilation_lock begin + cache[key] = obj + end + end + obj::V +end + +const _cached_compilation_lock = ReentrantLock() + +@noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, + cfg::CompilerConfig, compiler::Function, + linker::Function) + job = CompilerJob(src, cfg, world) + asm = nothing + + # provide a hook to use in tools / debuggers + if compile_hook[] !== nothing + Base.invokelatest(compile_hook[], job) + end + + asm = compiler(job) + obj = linker(job, asm) + obj +end + +@public cached_compilation diff --git a/src/driver.jl b/src/driver.jl index d25a51f1..86de7f10 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -42,9 +42,8 @@ end ## compiler entrypoint export compile - -# (::CompilerJob) -const compile_hook = Ref{Union{Nothing,Function}}(nothing) +# `compile_hook` is declared in `GPUCompiler.jl` so files included earlier +# (notably `deprecated.jl`'s `cached_compilation`) can reference it too. """ compile(target::Symbol, job::CompilerJob) diff --git a/src/interface.jl b/src/interface.jl index 21ec01ee..d31ce1bd 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -261,12 +261,25 @@ runtime_module(@nospecialize(job::CompilerJob)) = error("Not implemented") isintrinsic(@nospecialize(job::CompilerJob), fn::String) = false # provide a specific interpreter to use. +@static if HAS_INTEGRATED_CACHE function get_interpreter(@nospecialize(job::CompilerJob)) - GPUInterpreter(job.world; - method_table_view=maybe_cached(method_table_view(job)), - cache=cache_view(job), - inf_params=inference_params(job), - opt_params=optimization_params(job)) + V = results_type(job) + GPUInterpreter{V}(job.world; + method_table_view=maybe_cached(method_table_view(job)), + owner=cache_owner(job), + inf_params=inference_params(job), + opt_params=optimization_params(job)) +end +else +function get_interpreter(@nospecialize(job::CompilerJob)) + V = results_type(job) + cache = get_code_cache(job) + GPUInterpreter{V}(job.world; + method_table_view=maybe_cached(method_table_view(job)), + code_cache=cache, + inf_params=inference_params(job), + opt_params=optimization_params(job)) +end end # does this target support throwing Julia exceptions with jl_throw? @@ -296,9 +309,15 @@ pass_by_ref(@nospecialize(job::CompilerJob)) = false # whether pointer is a valid call target valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false -# Cache partitioning. The owner is stored on every CodeInstance and compared via `jl_egal`, -# so it (and every field) must be immutable for cross-session matches (e.g. via package -# precompilation); custom `target` / `params` types must be `struct`s, not `mutable struct`s. +# Cache partitioning. On Julia 1.11+, the owner is stored on every `CodeInstance` and +# compared via `jl_egal`, so it (and every field) must be immutable for cross-session +# matches (e.g. via package precompilation); custom `target` / `params` types must be +# `struct`s, not `mutable struct`s. +# +# On Julia 1.10, where there's no per-CI owner field, this token only identifies the +# session-local cache partition (see `cached_compilation` / `GLOBAL_CI_CACHES` in +# `deprecated.jl`); the immutability requirement is still useful for stable hashing. +# # Care is required for anything that impacts: # - method_table # - inference_params @@ -317,40 +336,17 @@ cache_owner(@nospecialize(job::CompilerJob)) = GPUCompilerCacheToken(job.config.target, job.config.params, job.config.always_inline, method_table(job)) -""" - GPUCompiler.NoResults() - -Default results type carried on each cached `CodeInstance` when the consumer hasn't -overridden [`results_type`](@ref). Carries no fields; useful for compiler jobs that -don't need to memoize compiled artifacts (e.g., reflection, precompile workloads). -""" -mutable struct NoResults end - -# The consumer's results struct type, stored on each CodeInstance via `CompilerCaching`. -# Override to attach session-portable artifacts (IR, object bytes) and session-local handles -# (e.g., `CuModule`, `MTLComputePipelineState`) to compiled CIs. The struct must be a -# `mutable struct` with a zero-arg constructor. -results_type(@nospecialize(job::CompilerJob)) = NoResults - -# Construct a `CompilerCaching.CacheView` partitioned by `cache_owner(job)` and parametrized -# by `results_type(job)`. -function cache_view(@nospecialize(job::CompilerJob)) - K = typeof(cache_owner(job)) - V = results_type(job) - CompilerCaching.CacheView{K, V}(cache_owner(job), job.world) -end +# The consumer's results struct type, stored on each `CodeInstance` (1.11+) via the +# `CompilerCaching` extension. Override to attach session-portable artifacts (e.g. IR or +# object bytes) and session-local handles (e.g. `CuModule`, `MTLComputePipelineState`). +# The struct must be a `mutable struct` with a zero-arg constructor. +# +# When the consumer hasn't overridden, the default `Nothing` opts out: no results struct +# is attached during inference, and the `analysis_results` chain is untouched. This is the +# right default for reflection paths, precompile workloads, and the legacy 1.10 flow. +results_type(@nospecialize(job::CompilerJob)) = Nothing -# Optional consumer hooks for caching post-codegen LLVM bitcode on the results struct. -# Override these on your `results_type(job)` to opt in to cross-session bitcode caching -# (most relevant for runtime library functions, which GPUCompiler compiles per-target and -# would otherwise rebuild every session). The `opaque_pointers` flag tracks the LLVM -# context's pointer mode at compile time — `bitcode` should reject mismatches by -# returning `nothing`, since opaque- and typed-pointer IR aren't interchangeable. -bitcode(@nospecialize(results), opaque_pointers::Bool) = nothing -bitcode!(@nospecialize(results), bytes::Vector{UInt8}, opaque_pointers::Bool) = nothing - -public GPUCompilerCacheToken, cache_owner, NoResults, results_type, cache_view, - bitcode, bitcode! +@public GPUCompilerCacheToken, cache_owner, results_type # the method table to use # deprecate method_table on next-breaking release diff --git a/src/jlgen.jl b/src/jlgen.jl index 23e0ce6d..23069e79 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -1,12 +1,13 @@ # Julia compiler integration -import CompilerCaching -using CompilerCaching: CacheView, @setup_caching - ## world age lookups -import Base: tls_world_age +@static if isdefined(Base, :tls_world_age) + import Base: tls_world_age +else + tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) +end ## looking up method instances @@ -49,10 +50,28 @@ This function is highly optimized, and results do not need to be cached addition Only use this function with concrete signatures, i.e., using the types of values you would pass at run time. For non-concrete signatures, use `generic_methodinstance` instead. """ -function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), - world::Integer=tls_world_age()) +methodinstance + +function generic_methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), + world::Integer=tls_world_age()) + sig = signature_type_by_tt(ft, tt) + + match, _ = CC._findsup(sig, nothing, world) + match === nothing && throw(MethodError(ft, tt, world)) + + mi = CC.specialize_method(match) + + return mi::MethodInstance +end + +# On 1.11+ (JuliaLang/julia#52572 / JuliaLang/julia#52233) `jl_method_lookup_by_tt` returns +# a usable `MethodInstance` directly, so we just call it. +@static if VERSION >= v"1.11.0-DEV.1552" + +@inline function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), + world::Integer=tls_world_age()) sig = signature_type_by_tt(ft, tt) - @assert Base.isdispatchtuple(sig) + @assert Base.isdispatchtuple(sig) # JuliaLang/julia#52233 mi = ccall(:jl_method_lookup_by_tt, Any, (Any, Csize_t, Any), @@ -68,16 +87,10 @@ function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), return mi end -function generic_methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), - world::Integer=tls_world_age()) - sig = signature_type_by_tt(ft, tt) - - match, _ = CC._findsup(sig, nothing, world) - match === nothing && throw(MethodError(ft, tt, world)) - - mi = CC.specialize_method(match) - - return mi::MethodInstance +# On 1.10 we use a generated function that performs the lookup at world+specialization +# resolution time. See deprecated.jl for `methodinstance_generator`. +else +const methodinstance = generic_methodinstance end @@ -85,7 +98,8 @@ end Base.Experimental.@MethodTable(GLOBAL_METHOD_TABLE) -# Implements a priority lookup for method tables, where the first match in the stack get's returned. +# Implements a priority lookup for method tables, where the first match in the stack get's +# returned. An alternative would be a Union with a most-specific match query. struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView world::UInt mt::Core.MethodTable @@ -96,44 +110,86 @@ StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) CC.isoverlayed(::StackedMethodTable) = true -function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return result +@static if VERSION >= v"1.11.0-DEV.363" + # https://github.com/JuliaLang/julia/pull/51078 + # same API as before but without returning isoverlayed flag + function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) + result = CC._findall(sig, table.mt, table.world, limit) + result === nothing && return nothing # too many matches + nr = CC.length(result) + if nr ≥ 1 && CC.getindex(result, nr).fully_covers + return result + end + + parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} + parent_result === nothing && return nothing # too many matches + + return CC.MethodLookupResult( + CC.vcat(result.matches, parent_result.matches), + CC.WorldRange( + CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), + CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), + result.ambig | parent_result.ambig) + end + + function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) + match, valid_worlds = CC._findsup(sig, table.mt, table.world) + match !== nothing && return match, valid_worlds + parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) + return ( + parent_match, + CC.WorldRange( + max(valid_worlds.min_world, parent_valid_worlds.min_world), + min(valid_worlds.max_world, parent_valid_worlds.max_world)) + ) end +else + function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) + result = CC._findall(sig, table.mt, table.world, limit) + result === nothing && return nothing # too many matches + nr = CC.length(result) + if nr ≥ 1 && CC.getindex(result, nr).fully_covers + return CC.MethodMatchResult(result, true) + end - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} - parent_result === nothing && return nothing #too many matches + parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} + parent_result === nothing && return nothing # too many matches - # merge the parent match results with the internal method table - return CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig) -end + overlayed = parent_result.overlayed | !CC.isempty(result) + parent_result = parent_result.matches::CC.MethodLookupResult -function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds - parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)) - ) + return CC.MethodMatchResult( + CC.MethodLookupResult( + CC.vcat(result.matches, parent_result.matches), + CC.WorldRange( + CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), + CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), + result.ambig | parent_result.ambig), + overlayed) + end + + function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) + match, valid_worlds = CC._findsup(sig, table.mt, table.world) + match !== nothing && return match, valid_worlds, true + parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) + return ( + parent_match, + CC.WorldRange( + max(valid_worlds.min_world, parent_valid_worlds.min_world), + min(valid_worlds.max_world, parent_valid_worlds.max_world)), + overlayed) + end end ## interpreter -import Core.Compiler: get_inference_world -using Base: get_world_counter +@static if VERSION >= v"1.11.0-DEV.1498" + import Core.Compiler: get_inference_world + using Base: get_world_counter +else + import Core.Compiler: get_world_counter, get_world_counter as get_inference_world +end if isdefined(Core.Compiler, :CachedMethodTable) using Core.Compiler: CachedMethodTable @@ -148,49 +204,94 @@ get_method_table_view(world::UInt, mt::CC.MethodTable) = CC.OverlayMethodTable(w const INFERENCE_CACHE_TYPE = isdefined(CC, :InferenceCache) ? CC.InferenceCache : Vector{CC.InferenceResult} """ - GPUInterpreter{MTV, K, V} + GPUInterpreter{MTV, V} + +Foreign abstract interpreter that drives Julia inference for GPU compilation. + +The `V` type parameter is the consumer's results-struct type (default `Nothing`). +On 1.11+ the `GPUCompiler` package extension `GPUCompilerCompilerCachingExt` +wires `CC.finish!` so that each newly-inferred `CodeInstance` carries a fresh +`V()` on its `analysis_results` chain; when `V === Nothing` (no consumer override +of `results_type`) or the extension isn't loaded, nothing is attached and +inference behaves like the default `NativeInterpreter`. -Foreign abstract interpreter that drives Julia inference for GPU compilation. Parametric -on the method-table view (`MTV`), the cache owner type (`K`), and the consumer's results -struct type (`V`). The cache is a `CompilerCaching.CacheView{K, V}` — `@setup_caching` -wires `cache_owner` and `finish!` so each `CodeInstance` carries a `V()` results struct -on its `analysis_results` chain, partitioned in the global CI cache by the owner token. +The interpreter is partitioned by an `owner` token (1.11+) or an in-process +`CodeCache` (1.10, see [`deprecated.jl`](deprecated.jl)). """ -struct GPUInterpreter{MTV<:CC.MethodTableView, K, V} <: CC.AbstractInterpreter +struct GPUInterpreter{V, MTV<:CC.MethodTableView} <: CC.AbstractInterpreter world::UInt method_table_view::MTV - cache::CacheView{K, V} + +@static if HAS_INTEGRATED_CACHE + owner::Any +else + code_cache::CodeCache +end inf_cache::INFERENCE_CACHE_TYPE inf_params::CC.InferenceParams opt_params::CC.OptimizationParams end -function GPUInterpreter(world::UInt; method_table_view::CC.MethodTableView, - cache::CacheView, - inf_params::CC.InferenceParams, - opt_params::CC.OptimizationParams) +@static if HAS_INTEGRATED_CACHE +function GPUInterpreter{V}(world::UInt=Base.get_world_counter(); + method_table_view::CC.MethodTableView, + owner::Any, + inf_params::CC.InferenceParams, + opt_params::CC.OptimizationParams) where V @assert world <= Base.get_world_counter() - GPUInterpreter(world, method_table_view, cache, INFERENCE_CACHE_TYPE(), - inf_params, opt_params) + return GPUInterpreter{V, typeof(method_table_view)}( + world, method_table_view, owner, INFERENCE_CACHE_TYPE(), + inf_params, opt_params) end -function GPUInterpreter(interp::GPUInterpreter; +function GPUInterpreter(interp::GPUInterpreter{V, MTV}; world::UInt=interp.world, method_table_view::CC.MethodTableView=interp.method_table_view, - cache::CacheView=interp.cache, + owner::Any=interp.owner, inf_cache::INFERENCE_CACHE_TYPE=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, - opt_params::CC.OptimizationParams=interp.opt_params) - GPUInterpreter(world, method_table_view, cache, inf_cache, inf_params, opt_params) + opt_params::CC.OptimizationParams=interp.opt_params) where {MTV, V} + return GPUInterpreter{V, typeof(method_table_view)}( + world, method_table_view, owner, inf_cache, + inf_params, opt_params) +end + +CC.cache_owner(interp::GPUInterpreter) = interp.owner + +else # 1.10: in-process CodeCache + +function GPUInterpreter{V}(world::UInt=Base.get_world_counter(); + method_table_view::CC.MethodTableView, + code_cache::CodeCache, + inf_params::CC.InferenceParams, + opt_params::CC.OptimizationParams) where V + @assert world <= Base.get_world_counter() + return GPUInterpreter{V, typeof(method_table_view)}( + world, method_table_view, code_cache, Vector{CC.InferenceResult}(), + inf_params, opt_params) end +function GPUInterpreter(interp::GPUInterpreter{V, MTV}; + world::UInt=interp.world, + method_table_view::CC.MethodTableView=interp.method_table_view, + code_cache::CodeCache=interp.code_cache, + inf_cache::Vector{CC.InferenceResult}=interp.inf_cache, + inf_params::CC.InferenceParams=interp.inf_params, + opt_params::CC.OptimizationParams=interp.opt_params) where {MTV, V} + return GPUInterpreter{V, typeof(method_table_view)}( + world, method_table_view, code_cache, inf_cache, + inf_params, opt_params) +end + +CC.code_cache(interp::GPUInterpreter) = WorldView(interp.code_cache, interp.world) + +end # HAS_INTEGRATED_CACHE + CC.InferenceParams(interp::GPUInterpreter) = interp.inf_params CC.OptimizationParams(interp::GPUInterpreter) = interp.opt_params -get_inference_world(interp::GPUInterpreter) = interp.world +#=CC.=#get_inference_world(interp::GPUInterpreter) = interp.world CC.get_inference_cache(interp::GPUInterpreter) = interp.inf_cache -@setup_caching GPUInterpreter.cache - # No need to do any locking since we're not putting our results into the runtime cache CC.lock_mi_inference(interp::GPUInterpreter, mi::MethodInstance) = nothing CC.unlock_mi_inference(interp::GPUInterpreter, mi::MethodInstance) = nothing @@ -217,6 +318,173 @@ function CC.concrete_eval_eligible(interp::GPUInterpreter, end return ret end +function CC.concrete_eval_eligible(interp::GPUInterpreter, + @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo) + ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, + f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo) + ret === false && return nothing + return ret +end + + +## driving inference and walking callees + +# Drive type inference on `mi` using `interp`. Recursively walks callees so that on 1.12+ +# their CodeInstances are populated in the integrated cache with stored source — this is +# what `collect_codeinfos` later relies on to gather `(CI, CodeInfo)` pairs for +# `jl_emit_native`. Returns the root `CodeInstance` on 1.11+ (or `nothing` if inference +# failed); on 1.10, where there's no integrated cache to return into, returns `nothing` +# and the caller fetches the CI via `code_cache(interp)`. +function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) +@static if VERSION >= v"1.12.0-DEV.1434" + ci = CC.typeinf_ext(interp, mi, CC.SOURCE_MODE_NOT_REQUIRED) + ci === nothing && return nothing + + has_compilequeue = VERSION >= v"1.13.0-DEV.499" || v"1.12-beta3" <= VERSION < v"1.13-" + if has_compilequeue + workqueue = CC.CompilationQueue(; interp) + push!(workqueue, ci) + else + workqueue = CodeInstance[ci] + inspected = IdSet{CodeInstance}() + end + + while !isempty(workqueue) + callee = pop!(workqueue) + if has_compilequeue + CC.isinspected(workqueue, callee) && continue + CC.markinspected!(workqueue, callee) + else + callee in inspected && continue + push!(inspected, callee) + end + + callee_mi = CC.get_ci_mi(callee) + if CC.use_const_api(callee) + # const-return: synthesized CodeInfo later, no need to store source + continue + end + + src = CC.typeinf_code(interp, callee_mi, true) + if src isa CodeInfo + # Make sure the inferred source is persisted on the CI so we can read it back + # in `collect_codeinfos` (separate session or not). This is what + # `CompilerCaching.typeinf!` does internally; we inline it here so GPUCompiler + # itself doesn't depend on CompilerCaching. + if (@atomic callee.inferred) === nothing + @atomic callee.inferred = src + end + if has_compilequeue + sptypes = CC.sptypes_from_meth_instance(callee_mi) + CC.collectinvokes!(workqueue, src, sptypes) + else + CC.collectinvokes!(workqueue, src) + end + end + end + + return ci +elseif VERSION >= v"1.12.0-DEV.15" + inferred_ci = CC.typeinf_ext_toplevel(interp, mi, CC.SOURCE_MODE_FORCE_SOURCE) + @assert inferred_ci !== nothing "Inference of $mi failed" + + # `typeinf_ext_toplevel` will have populated the cache; in some const-return cases + # the inference result wasn't recorded — set it from the returned CI. + wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) + if CC.haskey(wvc, mi) + ci = CC.getindex(wvc, mi) + if ci.inferred === nothing + CC.setindex!(wvc, inferred_ci, mi) + end + end + + return inferred_ci +else + src = CC.typeinf_ext_toplevel(interp, mi) + @assert src !== nothing "Inference of $mi failed" + + # On 1.11/1.10 we look the CI up via the cache afterwards (no return from + # `typeinf_ext_toplevel` here). For const-return CIs, store the source explicitly + # so callers re-using the CI don't need to re-infer. + wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) + if CC.haskey(wvc, mi) + ci = CC.getindex(wvc, mi) + if ci.inferred === nothing + @atomic ci.inferred = src + end + return ci + end + + return nothing +end +end + +# Retrieve CodeInfo for a CI, synthesizing one for const-return CIs (which never store +# inferred source). Returns `nothing` when no source is available — caller's choice +# whether that's an error. +function _ci_codeinfo(ci::CodeInstance) + raw = @atomic :monotonic ci.inferred + if raw isa CodeInfo + return raw + elseif raw isa Vector{UInt8} || raw isa String + mi = @static VERSION >= v"1.12-" ? CC.get_ci_mi(ci) : ci.def::MethodInstance + return ccall(:jl_uncompress_ir, Ref{CodeInfo}, (Any, Any, Any), + mi.def, ci, raw)::CodeInfo + elseif raw === nothing && CC.use_const_api(ci) + # const-return CIs skip source storage during inference; synthesize CodeInfo + # from the cached `rettype_const`. + mi = @static VERSION >= v"1.12-" ? CC.get_ci_mi(ci) : ci.def::MethodInstance + @static if VERSION >= v"1.13.0-DEV.1121" + src = CC.codeinfo_for_const(CC.NativeInterpreter(), mi, + CC.WorldRange(ci.min_world, ci.max_world), + ci.edges, ci.rettype_const) + elseif VERSION >= v"1.12-" + src = CC.codeinfo_for_const(CC.NativeInterpreter(), mi, ci.rettype_const) + # Work around 1.12/1.13 not setting nargs/isva in `codeinfo_for_const` + @static if v"1.12-" <= VERSION < v"1.14.0-DEV.60" + if src.nargs == 0 && mi.def isa Method + src.nargs = mi.def.nargs + src.isva = mi.def.isva + end + end + else + src = CC.codeinfo_for_const(CC.NativeInterpreter(), mi, ci.rettype_const) + end + return src + end + return nothing +end + +# Walk forward `:invoke` edges from `root` and collect `(CodeInstance, CodeInfo)` pairs +# for `jl_emit_native` (1.12+ payload format). Source is read back from the CIs that +# `drive_inference!` populated above. On 1.11 and older this is unused (those Julia +# versions take a list of `MethodInstance`s + a lookup callback). +function collect_codeinfos(root::CodeInstance) + pairs = Pair{CodeInstance, CodeInfo}[] + visited = IdSet{CodeInstance}() + queue = CodeInstance[root] + while !isempty(queue) + ci = pop!(queue) + ci in visited && continue + push!(visited, ci) + + src = _ci_codeinfo(ci) + src === nothing && continue + push!(pairs, ci => src) + + for stmt in src.code + if stmt isa Expr && stmt.head === :(=) + stmt = stmt.args[2] + end + if stmt isa Expr && (stmt.head === :invoke || + (VERSION >= v"1.12-" && stmt.head === :invoke_modify)) + callee = stmt.args[1] + callee isa CodeInstance && push!(queue, callee) + end + end + end + return pairs +end ## codegen/inference integration @@ -245,7 +513,7 @@ function Base.precompile(@nospecialize(job::CompilerJob)) error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end interp = get_interpreter(job) - CompilerCaching.typeinf!(interp.cache, interp, job.source) + drive_inference!(interp, job.source) return true end @@ -298,13 +566,36 @@ function record_coverage(mi::MethodInstance, src::CodeInfo) return end -# for platforms without @cfunction-with-closure support +# for platforms without @cfunction-with-closure support, used pre-1.12 (and on 1.12 paths +# that still require `cgparams.lookup`). const _method_instances = Ref{Any}() -const _cache = Ref{Any}() +const _lookup_cache = Ref{Any}() function _lookup_fun(mi, min_world, max_world) push!(_method_instances[], mi) - cache = _cache[]::CacheView - get(cache, mi, nothing) + lookup_ci(_lookup_cache[], mi, min_world, max_world) +end + +# Resolve a `(MI, world)` to a CodeInstance, on whatever cache shape we have: +# `WorldView{CodeCache}` on 1.10, an `owner` token on 1.11+. +@static if HAS_INTEGRATED_CACHE +function lookup_ci(owner, mi::MethodInstance, min_world::UInt, max_world::UInt) + @static if VERSION >= v"1.14-" + wvc = CC.InternalCodeCache(owner, CC.WorldRange(min_world, max_world)) + else + wvc = WorldView(CC.InternalCodeCache(owner), CC.WorldRange(min_world, max_world)) + end + return CC.get(wvc, mi, nothing) +end +else +function lookup_ci(cache::CodeCache, mi::MethodInstance, min_world::UInt, max_world::UInt) + wvc = WorldView(cache, min_world, max_world) + ci = CC.get(wvc, mi, nothing) + if ci !== nothing && ci.inferred === nothing + # rettype-only CI without source — pretend we don't have it so we re-infer + return nothing + end + return ci +end end function compile_method_instance(@nospecialize(job::CompilerJob)) @@ -312,17 +603,22 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - # populate the cache (inference) + # drive inference interp = get_interpreter(job) - cache = interp.cache - CompilerCaching.typeinf!(cache, interp, job.source) - ci = get(cache, job.source, nothing)::CodeInstance + root_ci = drive_inference!(interp, job.source) - # collect (CI, CodeInfo) pairs for jl_emit_native - @static if VERSION >= v"1.12.0-DEV.1823" - codeinfo_pairs = CompilerCaching.get_codeinfos(ci) + # the cache handle for callback lookups: an owner token on 1.11+, a CodeCache on 1.10 + cache_handle = @static if HAS_INTEGRATED_CACHE + cache_owner(job) else - codeinfo_pairs = nothing + interp.code_cache + end + + # gather (CI, CodeInfo) pairs for jl_emit_native (1.12+) + codeinfo_pairs = if VERSION >= v"1.12.0-DEV.1823" && root_ci !== nothing + collect_codeinfos(root_ci) + else + nothing end # record line coverage of all compiled code (on older versions of Julia, where @@ -339,11 +635,11 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if Sys.ARCH == :x86 || Sys.ARCH == :x86_64 function lookup_fun(mi, min_world, max_world) push!(method_instances, mi) - get(cache, mi, nothing) + lookup_ci(cache_handle, mi, min_world, max_world) end lookup_cb = @cfunction($lookup_fun, Any, (Any, UInt, UInt)) else - _cache[] = cache + _lookup_cache[] = cache_handle _method_instances[] = method_instances lookup_cb = @cfunction(_lookup_fun, Any, (Any, UInt, UInt)) end @@ -384,6 +680,8 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) native_code = if VERSION >= v"1.12.0-DEV.1823" codeinfos = Any[] for (ci′, src) in codeinfo_pairs + # each item in the list is a CodeInstance followed by a CodeInfo + # indicating something to compile push!(codeinfos, ci′::CodeInstance) push!(codeinfos, src::CodeInfo) end @@ -414,7 +712,9 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end # Since Julia 1.13, the caller is responsible for initializing global variables that - # point to global values or bindings with their address in memory. + # point to global values or bindings with their address in memory. Similarly on earlier + # versions where `HAS_LLVM_GVS_GLOBALS` is true (see + # https://github.com/JuliaGPU/GPUCompiler.jl/issues/753). gvs = nothing inits = nothing @static if VERSION >= v"1.13.0-DEV.623" @@ -455,6 +755,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) gv_to_value = Dict{String, Ptr{Cvoid}}() if gvs === nothing + # No reliable GV table on this Julia — best-effort discovery from the module. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -487,7 +788,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end end - code_instances = Core.CodeInstance[] + code_instances = CodeInstance[] if HAS_LLVM_GET_CIS # on sufficiently recent versions of Julia, we can query the CIs compiled. @@ -510,7 +811,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if !HAS_LLVM_GET_CIS for mi in method_instances - ci′ = get(cache, mi, nothing) + ci′ = lookup_ci(cache_handle, mi, job.world, job.world) ci′ === nothing && continue llvm_func_idx = Ref{Int32}(-1) @@ -533,8 +834,9 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) ci′.min_world <= job.world <= ci′.max_world end - # To avoid a clash in the compiled cache containing both an interpreter token and native, - # prefer the non-native code-instance. + # When `jl_get_llvm_cis` returns CIs the cache may contain both an + # interpreter-token-owned CI (ours) and a `nothing`-owner native CI for the same MI; + # prefer the foreign one. owned_mis = Set{MethodInstance}() for ci′ in code_instances if ci′.owner !== nothing @@ -609,3 +911,18 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end + +# partially revert JuliaLang/julia#49391 — see #527 +@static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master + !(v"1.11-beta2" <= VERSION < v"1.12") # reverted on 1.11-beta2 +function CC.typeinf(interp::GPUInterpreter, frame::CC.InferenceState) + if CC.__measure_typeinf__[] + CC.Timings.enter_new_timer(frame) + v = CC._typeinf(interp, frame) + CC.Timings.exit_current_timer(frame) + return v + else + return CC._typeinf(interp, frame) + end +end +end diff --git a/src/optim.jl b/src/optim.jl index 5672daa9..8c6d1500 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -358,10 +358,15 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op end add!(fpm, GCInvariantVerifierPass()) add!(fpm, LateLowerGCPass()) - if uses_julia_runtime(job) + # FinalLowerGCPass moved from a module pass to a function pass in + # JuliaLang/julia#51081 (1.11.0-DEV.208). + if uses_julia_runtime(job) && VERSION >= v"1.11.0-DEV.208" add!(fpm, FinalLowerGCPass()) end end + if uses_julia_runtime(job) && VERSION < v"1.11.0-DEV.208" + add!(mpm, FinalLowerGCPass()) + end if opt_level >= 2 add!(mpm, NewPMFunctionPassManager()) do fpm diff --git a/src/rtlib.jl b/src/rtlib.jl index d5488944..ef097eda 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,30 +60,12 @@ end ## functionality to build the runtime library -# Compile a single runtime function and link it into `mod`. If `cache` is provided and the -# consumer's results type opts in via `GPUCompiler.bitcode`/`bitcode!`, per-function bitcode -# is read from / written to the cached `CodeInstance` to avoid recompilation across sessions. -function emit_function!(mod, config::CompilerConfig, f, method, - cache::Union{Nothing, CacheView}=nothing) +# Compile a single runtime function and link it into `mod`. +function emit_function!(mod, config::CompilerConfig, f, method) tt = Base.to_tuple_type(method.types) source = generic_methodinstance(f, tt) name = method.llvm_name - opaque_pointers = !supports_typed_pointers(context()) - - # fast path: pull renamed bitcode straight from the cached CI - if cache !== nothing - hit = CompilerCaching.lookup(cache, source) - if hit !== nothing - cached = bitcode(hit[2], opaque_pointers) - if cached !== nothing - func_mod = parse(LLVM.Module, MemoryBuffer(cached)) - link!(mod, func_mod) - return - end - end - end - # slow path: compile, rename, optionally cache, link new_mod, meta = compile_unhooked(:llvm, CompilerJob(source, config)) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) @@ -94,8 +76,7 @@ function emit_function!(mod, config::CompilerConfig, f, method, # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - # rename to the final `gpu_*` name on the per-function module, so the cached bitcode is - # immediately link-ready (no per-session rename pass). + # rename to the final `gpu_*` name on the per-function module so the link is direct if haskey(functions(new_mod), name) && functions(new_mod)[name] !== meta.entry decl = functions(new_mod)[name] @assert value_type(decl) == value_type(meta.entry) @@ -104,30 +85,16 @@ function emit_function!(mod, config::CompilerConfig, f, method, end LLVM.name!(meta.entry, name) - if cache !== nothing - hit = CompilerCaching.lookup(cache, source) - if hit !== nothing - io = IOBuffer() - write(io, new_mod) - bitcode!(hit[2], take!(io), opaque_pointers) - end - end - link!(mod, new_mod) end function build_runtime(@nospecialize(job::CompilerJob)) mod = LLVM.Module("GPUCompiler run-time library") - # the compiler job passed into here is identifies the job that requires the runtime. + # the compiler job passed into here identifies the job that requires the runtime. # derive a job that represents the runtime itself (notably with kernel=false). config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, strip=false) - # cache view shared with the runtime functions' compile_unhooked calls — owner is - # determined by target+params+always_inline+method_table, all preserved on `config`. - proto = CompilerJob(job.source, config, job.world) - cache = cache_view(proto) - for method in values(Runtime.methods) def = if isa(method.def, Symbol) isdefined(runtime_module(job), method.def) || continue @@ -135,7 +102,7 @@ function build_runtime(@nospecialize(job::CompilerJob)) else method.def end - emit_function!(mod, config, typeof(def), method, cache) + emit_function!(mod, config, typeof(def), method) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -146,10 +113,9 @@ function build_runtime(@nospecialize(job::CompilerJob)) mod end -# session-local cache of assembled runtime libraries, keyed by -# `(cache_owner, opaque_pointers)`. Avoids re-running `build_runtime` (which re-parses and -# re-links per-function bitcode) on every kernel compile within a session. Cross-session -# persistence happens at the per-function level via the `bitcode`/`bitcode!` hooks. +# Session-local cache of assembled runtime libraries, keyed by +# `(cache_owner(job), opaque_pointers)`. Cross-session persistence is the back-end's +# concern: rebuild on first use of each session, then reuse within the session. const _runtime_libs = Dict{Tuple{Any, Bool}, Vector{UInt8}}() const _runtime_libs_lock = ReentrantLock() diff --git a/src/utils.jl b/src/utils.jl index 1f18edf0..db2fc7d7 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -6,6 +6,28 @@ else __has_internal_julia_change(version_or::VersionNumber, feature::Symbol) = false end + + +## `public` keyword compat + +""" + @public foo, bar + +Declare `foo, bar` as public API. Lowers to `public foo, bar` on 1.11+ (where `public` +is keyword syntax) and to a no-op on 1.10. +""" +macro public(symbols_expr) + syms = symbols_expr isa Symbol ? [symbols_expr] : + symbols_expr.head === :tuple ? [a isa Symbol ? a : a.args[1] for a in symbols_expr.args] : + [symbols_expr.args[1]] + if VERSION >= v"1.11.0-DEV.469" + esc(Expr(:public, syms...)) + else + nothing + end +end + + ## debug verification should_verify() = ccall(:jl_is_debugbuild, Cint, ()) == 1 || @@ -99,9 +121,20 @@ for level in [:debug, :info, :warn, :error] # using with_logger would create a closure, which is incompatible with # generated functions, so instead we reproduce its implementation here safe_logstate = Base.CoreLogging.LogState(safe_logger) - Base.ScopedValues.@with( - Base.CoreLogging.CURRENT_LOGSTATE => safe_logstate, $(esc(macrocall)) - ) + @static if VERSION < v"1.11-" + t = current_task() + old_logstate = t.logstate + try + t.logstate = safe_logstate + $(esc(macrocall)) + finally + t.logstate = old_logstate + end + else + Base.ScopedValues.@with( + Base.CoreLogging.CURRENT_LOGSTATE => safe_logstate, $(esc(macrocall)) + ) + end end end end diff --git a/src/validation.jl b/src/validation.jl index c08f1fd2..5760afc7 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -14,7 +14,14 @@ function method_matches(@nospecialize(tt::Type{<:Tuple}); world::Integer) end function typeinf_type(mi::MethodInstance; interp::CC.AbstractInterpreter) - something(Core.Compiler.typeinf_type(interp, mi), Any) + @static if hasmethod(Core.Compiler.typeinf_type, Tuple{CC.AbstractInterpreter, MethodInstance}) + rt = Core.Compiler.typeinf_type(interp, mi) + else + # Julia 1.10: only the 4-arg form exists; reconstruct it from the MI. + method = mi.def::Method + rt = Core.Compiler.typeinf_type(interp, method, mi.specTypes, mi.sparam_vals) + end + return something(rt, Any) end function check_method(@nospecialize(job::CompilerJob)) From f4954225568c44eef4332482e7f8e0147f934d7d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 12:21:30 +0200 Subject: [PATCH 07/33] Restore per-CI bitcode caching for the GPU runtime library. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `emit_function!` now memoizes each `gpu_*` runtime function's renamed, post-irgen LLVM bitcode on its own `CodeInstance`'s `analysis_results` when the back-end opts in via the new `bitcode`/`bitcode!` trait pair. Cross-session persistence rides on package precompilation; the session-local `_runtime_libs` assembled-module cache keeps repeated within-session linking cheap. Gated on `HAS_INTEGRATED_CACHE` so 1.10 falls through to plain compile + link (still serviced by `_runtime_libs`). With Metal opted in, a second kernel compile in the same session — even after `reset_runtime()` invalidates `_runtime_libs` — completes ~25× faster than a cold rebuild on 1.12, because each runtime function is now a parse-and-link instead of a full Julia → LLVM run. Restores the optimization originally landed in aa4e64d (reverted in 566811d for 1.10 compat); the new version sits behind the same infrastructure as `cached_compilation`, so the 1.10 path is no longer load-bearing on the trait being callable. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/deprecated.jl | 6 +++--- src/interface.jl | 43 ++++++++++++++++++++++++++++++++++++++++++- src/rtlib.jl | 45 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index ad84b04c..9ff6d898 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -207,20 +207,20 @@ function cached_compilation(cache::AbstractDict{<:Any,V}, # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive boxing # allocation. Base does this with a multi-level lookup; we use a single-level dict. - Base.@lock _cached_compilation_lock begin + Base.@lock cached_compilation_lock begin obj = get(cache, key, nothing) end if obj === nothing || compile_hook[] !== nothing obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V - Base.@lock _cached_compilation_lock begin + Base.@lock cached_compilation_lock begin cache[key] = obj end end obj::V end -const _cached_compilation_lock = ReentrantLock() +const cached_compilation_lock = ReentrantLock() @noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, cfg::CompilerConfig, compiler::Function, diff --git a/src/interface.jl b/src/interface.jl index d31ce1bd..338ff72a 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -346,7 +346,48 @@ cache_owner(@nospecialize(job::CompilerJob)) = # right default for reflection paths, precompile workloads, and the legacy 1.10 flow. results_type(@nospecialize(job::CompilerJob)) = Nothing -@public GPUCompilerCacheToken, cache_owner, results_type +""" + bitcode(results) -> Union{Nothing, Vector{UInt8}} + bitcode!(results, bytes::Vector{UInt8}) -> Nothing + +Optional consumer hooks for stashing post-irgen LLVM bitcode on the results struct. +Used by [`emit_function!`](@ref) to memoize each runtime function's bitcode on its +`CodeInstance` (1.11+, via `analysis_results`); back-ends can populate the slot from +any compile they want to memoize at the LLVM stage. + +Override both on your [`results_type`](@ref) to opt in. The default pair (`bitcode` → +`nothing`, `bitcode!` → no-op) means no LLVM-stage caching happens. + +The LLVM context's pointer mode (opaque vs. typed) is assumed fixed for the lifetime of +a session — and across precompile/load pairs of the same Julia version, since pkgimages +are invalidated when the toolchain changes. Back-ends don't need to gate caching on +that mode. + +On Julia 1.10 these hooks are never invoked (no integrated cache to stash bitcode on); +runtime libraries fall back to the session-local `runtime_libs` cache. +""" +bitcode(@nospecialize(results)) = nothing +bitcode!(@nospecialize(results), bytes::Vector{UInt8}) = nothing + +@static if HAS_INTEGRATED_CACHE + """ + cache_view(job::CompilerJob) -> CompilerCaching.CacheView + + Construct a `CacheView{typeof(cache_owner(job)), results_type(job)}` over Julia's + integrated `CodeInstance` cache at `job.world`. The handle back-ends pass to + `CompilerCaching.lookup` / `CompilerCaching.results` when populating their own + per-CI artifacts. + """ + function cache_view(@nospecialize(job::CompilerJob)) + owner = cache_owner(job) + CompilerCaching.CacheView{typeof(owner), results_type(job)}(owner, job.world) + end +end + +@public GPUCompilerCacheToken, cache_owner, results_type, bitcode, bitcode! +@static if HAS_INTEGRATED_CACHE + @public cache_view +end # the method table to use # deprecate method_table on next-breaking release diff --git a/src/rtlib.jl b/src/rtlib.jl index ef097eda..7e5293e2 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,12 +60,32 @@ end ## functionality to build the runtime library -# Compile a single runtime function and link it into `mod`. +# Compile a single runtime function and link it into `mod`. On Julia 1.11+ each +# runtime function's renamed bitcode is cached on its `CodeInstance`'s +# `analysis_results` when the back-end opts in via the `bitcode` / `bitcode!` +# trait pair — cross-session persistence rides on package precompilation. On +# 1.10 (no integrated cache) we fall through to plain compile-and-link; the +# session-local `runtime_libs` cache below avoids repeating that work +# within a session. function emit_function!(mod, config::CompilerConfig, f, method) tt = Base.to_tuple_type(method.types) source = generic_methodinstance(f, tt) name = method.llvm_name + @static if HAS_INTEGRATED_CACHE + rt_job = CompilerJob(source, config) + cache = cache_view(rt_job) + hit = CompilerCaching.lookup(cache, source) + if hit !== nothing + cached = bitcode(hit[2]) + if cached !== nothing + func_mod = parse(LLVM.Module, MemoryBuffer(cached)) + link!(mod, func_mod) + return + end + end + end + new_mod, meta = compile_unhooked(:llvm, CompilerJob(source, config)) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) @@ -76,7 +96,8 @@ function emit_function!(mod, config::CompilerConfig, f, method) # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - # rename to the final `gpu_*` name on the per-function module so the link is direct + # rename to the final `gpu_*` name on the per-function module, so the cached bitcode + # is immediately link-ready (no per-session rename pass on a cache hit). if haskey(functions(new_mod), name) && functions(new_mod)[name] !== meta.entry decl = functions(new_mod)[name] @assert value_type(decl) == value_type(meta.entry) @@ -85,6 +106,18 @@ function emit_function!(mod, config::CompilerConfig, f, method) end LLVM.name!(meta.entry, name) + @static if HAS_INTEGRATED_CACHE + # Re-lookup: inference for this runtime function's MI may have happened + # inside `compile_unhooked` (callee walk or `precompile`), so the CI + # exists now even if the pre-compile lookup returned nothing. + hit = CompilerCaching.lookup(cache, source) + if hit !== nothing + io = IOBuffer() + write(io, new_mod) + bitcode!(hit[2], take!(io)) + end + end + link!(mod, new_mod) end @@ -116,13 +149,13 @@ end # Session-local cache of assembled runtime libraries, keyed by # `(cache_owner(job), opaque_pointers)`. Cross-session persistence is the back-end's # concern: rebuild on first use of each session, then reuse within the session. -const _runtime_libs = Dict{Tuple{Any, Bool}, Vector{UInt8}}() -const _runtime_libs_lock = ReentrantLock() +const runtime_libs = Dict{Tuple{Any, Bool}, Vector{UInt8}}() +const runtime_libs_lock = ReentrantLock() @locked function load_runtime(@nospecialize(job::CompilerJob)) key = (cache_owner(job), !supports_typed_pointers(context())) - bytes = Base.@lock _runtime_libs_lock get!(_runtime_libs, key) do + bytes = Base.@lock runtime_libs_lock get!(runtime_libs, key) do lib = build_runtime(job) io = IOBuffer() write(io, lib) @@ -133,4 +166,4 @@ const _runtime_libs_lock = ReentrantLock() end # clear the session-local runtime library cache -reset_runtime() = Base.@lock _runtime_libs_lock empty!(_runtime_libs) +reset_runtime() = Base.@lock runtime_libs_lock empty!(runtime_libs) From b3d3898ebd80d7e69c4950384fee5c9d88c4e1f8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 13:10:15 +0200 Subject: [PATCH 08/33] Don't cache relocated GVs. --- src/driver.jl | 2 ++ src/interface.jl | 6 +++--- src/jlgen.jl | 35 ++++++++++++++++++++++++++++------- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index 86de7f10..94dd4778 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -334,6 +334,8 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) + relocate_gvs!(ir, gv_to_value) + if job.config.optimize @tracepoint "optimization" begin optimize!(job, ir; job.config.opt_level) diff --git a/src/interface.jl b/src/interface.jl index 338ff72a..0f5aa8c6 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -351,9 +351,9 @@ results_type(@nospecialize(job::CompilerJob)) = Nothing bitcode!(results, bytes::Vector{UInt8}) -> Nothing Optional consumer hooks for stashing post-irgen LLVM bitcode on the results struct. -Used by [`emit_function!`](@ref) to memoize each runtime function's bitcode on its -`CodeInstance` (1.11+, via `analysis_results`); back-ends can populate the slot from -any compile they want to memoize at the LLVM stage. +GPUCompiler currently uses them from [`emit_function!`](@ref) to memoize each runtime +function's bitcode on its `CodeInstance` (1.11+, via `analysis_results`); back-ends can +populate the slot from any compile they want to memoize at the LLVM stage. Override both on your [`results_type`](@ref) to opt in. The default pair (`bitcode` → `nothing`, `bitcode!` → no-op) means no LLVM-stage caching happens. diff --git a/src/jlgen.jl b/src/jlgen.jl index 23069e79..2f824a63 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -750,12 +750,14 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end end - # Maintain a map from global variables to their initialized Julia values. - # The objects pointed to are perma-rooted, during codegen. + # Maintain a map from global variables to their initialized Julia values. The + # objects pointed to are perma-rooted during codegen. We *don't* bake these + # addresses into the IR yet, so that we can cache it across sessions. gv_to_value = Dict{String, Ptr{Cvoid}}() - if gvs === nothing # No reliable GV table on this Julia — best-effort discovery from the module. + # On these older versions Julia's own emit may have already baked in absolute + # pointer values; we recover them by reading existing initializers. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -781,10 +783,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init - if LLVM.isnull(initializer(gv)) - val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) - initializer!(gv, val) - end end end @@ -912,6 +910,29 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end +""" + relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + +Bake absolute pointer values into the initializers of `julia.constgv`-tagged +global variables, matching them by name against `gv_to_value`. Only GVs whose +initializer is currently null are touched: Preserving existing initializers +covers the older-Julia path where Julia itself emits pointer values directly. + +GVs present in `mod` but missing from `gv_to_value` keep a null initializer. +""" +function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + mod_gvs = globals(mod) + for (name, init) in gv_to_value + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + if LLVM.isnull(initializer(gv)) + val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) + initializer!(gv, val) + end + end + return +end + # partially revert JuliaLang/julia#49391 — see #527 @static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master !(v"1.11-beta2" <= VERSION < v"1.12") # reverted on 1.11-beta2 From 2b21abea8f0d8ccbc4ea9a9ac6c820c88b7b6b17 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 13:25:32 +0200 Subject: [PATCH 09/33] Null-out GV initializers on 1.12 to mirror 1.13+ behavior. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Julia 1.13+, `jl_emit_native_impl` itself sets every `jl_sysimg_gvar` to a null initializer before returning (aotcompile.cpp:865), leaving relocation to the caller. On 1.12, `jl_emit_native_impl` instead bakes session-local pointer values into the initializer via `literal_static_pointer_val` — so without intervention, the bitcode we hand to `bitcode!` for caching carries live pointers from the current session and isn't safe to reload in a future one. After collecting `gv_to_value` from `jl_get_llvm_gvs` / `jl_get_llvm_gvs_globals`, immediately reset each tracked GV's initializer to null. `relocate_gvs!` at the toplevel link step then re-applies the session-current values regardless of which Julia we're on, so optimization still sees the resolved constants. On 1.13+ the null-out is a no-op (Julia already nulled them); on 1.12 this is what makes per-CI runtime bitcode caching genuinely cross-session-safe for back-ends that pull in `julia.constgv`-touching runtime functions (CUDA's `gc_pool_alloc`, `box_*`/`unbox_*`, …). Metal's stubs don't trip this either way. Verified: 27 `julia.constgv` GVs in Metal's cached runtime-fn bitcode on 1.12, all with null initializers post-change (was 27/27 non-null). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/jlgen.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/jlgen.jl b/src/jlgen.jl index 2f824a63..e734af7d 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -783,6 +783,13 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init + # Mirror what `jl_emit_native_impl` does on 1.13+ (aotcompile.cpp:865): + # reset the initializer to null so the IR we hand back is session-portable, + # then `relocate_gvs!` at the toplevel link step writes the session-current + # value in. On 1.13+ this is a no-op (Julia already nulled them); on 1.12, + # where `jl_emit_native_impl` bakes pointers via `literal_static_pointer_val`, + # this is what makes the bitcode we cache safe across sessions. + initializer!(gv, LLVM.null(value_type(initializer(gv)))) end end From e8a74a8cee890d650c8e02fbaf4796656f65485e Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 14:18:31 +0200 Subject: [PATCH 10/33] Inline the CompilerCaching extension; delegate inference to it on 1.11+. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CompilerCaching becomes a strong dep (it loads as an empty shell on 1.10, so there's no overhead). The `GPUCompilerCompilerCachingExt` extension — which existed just to wire the parametric `CC.finish!` override that attaches a `CachedResult{V}` to every inferred CodeInstance — moves into `jlgen.jl` directly, gated on `HAS_INTEGRATED_CACHE`. Same code, one less file. With CompilerCaching available unconditionally we can also drop the inline copies of its inference machinery from `jlgen.jl`: - `drive_inference!` on 1.11+ is now a two-line delegation to `CompilerCaching.typeinf!` + `get(cache, mi, nothing)`. The 1.10 implementation (which talks to the per-interpreter `CodeCache`) moves to `deprecated.jl`. - `collect_codeinfos` / `_ci_codeinfo` go away; the single call site in `compile_method_instance` calls `CompilerCaching.get_codeinfos` directly. - `StackedMethodTable` is re-exported from CompilerCaching on 1.11+; the 1.10 variant (with the older `MethodMatchResult`-shape `findall`) moves to `deprecated.jl`. Net result: ~200 lines deleted from `jlgen.jl`, no behavior change. All 1.10-only code is now in `deprecated.jl`, ready to disappear in one diff when 1.10 support drops. Co-Authored-By: Claude Opus 4.7 (1M context) --- Project.toml | 5 +- ext/GPUCompilerCompilerCachingExt.jl | 38 ---- src/GPUCompiler.jl | 6 + src/deprecated.jl | 76 +++++++ src/jlgen.jl | 286 +++++---------------------- 5 files changed, 133 insertions(+), 278 deletions(-) delete mode 100644 ext/GPUCompilerCompilerCachingExt.jl diff --git a/Project.toml b/Project.toml index 35896497..d4ad4bda 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ authors = ["Tim Besard "] projects = ["test"] [deps] +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" ExprTools = "e2ba6199-217a-4e67-a87a-7c52f15ade04" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" @@ -21,13 +22,9 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [weakdeps] AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" -CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444" NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" -[extensions] -GPUCompilerCompilerCachingExt = "CompilerCaching" - [compat] AMDGPU_LLVM_Backend_jll = "22" diff --git a/ext/GPUCompilerCompilerCachingExt.jl b/ext/GPUCompilerCompilerCachingExt.jl deleted file mode 100644 index e885046d..00000000 --- a/ext/GPUCompilerCompilerCachingExt.jl +++ /dev/null @@ -1,38 +0,0 @@ -module GPUCompilerCompilerCachingExt - -# GPUCompiler ↔ CompilerCaching glue. -# -# CompilerCaching attaches a `CachedResult{V}` to each newly inferred `CodeInstance`'s -# `analysis_results` chain so back-ends can recover typed results on cache hits. Julia -# only allows writing `CodeInstance.analysis_results` during the `CC.finish!` step of -# inference (it's a const field afterwards on 1.12), so the typed-results attachment -# has to live on a `CC.finish!` override. -# -# We define that override here, parametrically over `GPUInterpreter{V, MTV}`'s results -# type `V`. When `V === Nothing` (the default — no consumer override of `results_type`) -# this is a no-op pass-through to the default `CC.finish!`, so loading CompilerCaching -# alongside a no-results back-end has no inference-time cost. -# -# The extension only loads on Julia ≥ 1.11 (CompilerCaching's `__init__` errors on -# 1.10), so the integrated cache and `analysis_results` are guaranteed to exist here. - -using GPUCompiler: GPUInterpreter -using CompilerCaching: CachedResult -const CC = Core.Compiler - -@static if hasmethod(CC.finish!, Tuple{CC.AbstractInterpreter, CC.InferenceState, UInt, UInt64}) - function CC.finish!(interp::GPUInterpreter{V, MTV}, caller::CC.InferenceState, - validation_world::UInt, time_before::UInt64) where {MTV, V} - V === Nothing || CC.stack_analysis_result!(caller.result, CachedResult{V}(V())) - @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState, - validation_world::UInt, time_before::UInt64) - end -else - function CC.finish!(interp::GPUInterpreter{V, MTV}, - caller::CC.InferenceState) where {MTV, V} - V === Nothing || CC.stack_analysis_result!(caller.result, CachedResult{V}(V())) - @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState) - end -end - -end # module diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index a4661418..ebaacae6 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -38,6 +38,12 @@ using Core: MethodInstance, CodeInstance, CodeInfo # disjoint shapes; everything cache-related fans on this flag. const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" +# Loads as an empty shell on 1.10; on 1.11+ provides `CacheView`, `typeinf!`, +# `get_codeinfos`, `lookup`, `results`, etc. We `import` the module name (not its +# exports) to avoid clashing with `LLVM.lookup`; internal call sites qualify with +# `CompilerCaching.`. +import CompilerCaching + # Optional callback invoked from `compile(...)` / `cached_compilation(...)` before # compilation runs. Set by `@device_code_*` reflection macros. Defined here (early) # so the legacy `cached_compilation` in deprecated.jl can reference it. diff --git a/src/deprecated.jl b/src/deprecated.jl index 9ff6d898..bed01897 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -174,6 +174,82 @@ function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodIns CC.setindex!(wvc.cache, ci, mi) end + +## 1.10 StackedMethodTable +# +# Priority-lookup method table. The 1.11+ version lives in CompilerCaching; here we keep +# the 1.10 copy, which has to produce the older `MethodMatchResult` shape from +# `CC.findall` / `CC.findsup`. +struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView + world::UInt + mt::Core.MethodTable + parent::MTV +end +StackedMethodTable(world::UInt, mt::Core.MethodTable) = + StackedMethodTable(world, mt, CC.InternalMethodTable(world)) +StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) = + StackedMethodTable(world, mt, StackedMethodTable(world, parent)) + +CC.isoverlayed(::StackedMethodTable) = true + +function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) + result = CC._findall(sig, table.mt, table.world, limit) + result === nothing && return nothing # too many matches + nr = CC.length(result) + if nr ≥ 1 && CC.getindex(result, nr).fully_covers + return CC.MethodMatchResult(result, true) + end + + parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} + parent_result === nothing && return nothing # too many matches + + overlayed = parent_result.overlayed | !CC.isempty(result) + parent_result = parent_result.matches::CC.MethodLookupResult + + return CC.MethodMatchResult( + CC.MethodLookupResult( + CC.vcat(result.matches, parent_result.matches), + CC.WorldRange( + CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), + CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), + result.ambig | parent_result.ambig), + overlayed) +end + +function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) + match, valid_worlds = CC._findsup(sig, table.mt, table.world) + match !== nothing && return match, valid_worlds, true + parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) + return ( + parent_match, + CC.WorldRange( + max(valid_worlds.min_world, parent_valid_worlds.min_world), + min(valid_worlds.max_world, parent_valid_worlds.max_world)), + overlayed) +end + + +## 1.10 inference driver +# +# On 1.11+ this is delegated to `CompilerCaching.typeinf!`; here we keep an inline copy +# that talks to the per-interpreter `CodeCache` instead of the integrated one. Returns +# `nothing` (no integrated CI to hand back; the caller fetches via `code_cache(interp)`). +function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) + src = CC.typeinf_ext_toplevel(interp, mi) + @assert src !== nothing "Inference of $mi failed" + + # For const-return CIs the inference result wasn't recorded — set it from the + # returned source so callers re-using the CI don't need to re-infer. + wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) + if CC.haskey(wvc, mi) + ci = CC.getindex(wvc, mi) + if ci.inferred === nothing + @atomic ci.inferred = src + end + end + return nothing +end + end # !HAS_INTEGRATED_CACHE diff --git a/src/jlgen.jl b/src/jlgen.jl index e734af7d..528dd5a6 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -98,87 +98,10 @@ end Base.Experimental.@MethodTable(GLOBAL_METHOD_TABLE) -# Implements a priority lookup for method tables, where the first match in the stack get's -# returned. An alternative would be a Union with a most-specific match query. -struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView - world::UInt - mt::Core.MethodTable - parent::MTV -end -StackedMethodTable(world::UInt, mt::Core.MethodTable) = StackedMethodTable(world, mt, CC.InternalMethodTable(world)) -StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) = StackedMethodTable(world, mt, StackedMethodTable(world, parent)) - -CC.isoverlayed(::StackedMethodTable) = true - -@static if VERSION >= v"1.11.0-DEV.363" - # https://github.com/JuliaLang/julia/pull/51078 - # same API as before but without returning isoverlayed flag - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # too many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - return result - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} - parent_result === nothing && return nothing # too many matches - - return CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig) - end - - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds - parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)) - ) - end -else - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # too many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - return CC.MethodMatchResult(result, true) - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} - parent_result === nothing && return nothing # too many matches - - overlayed = parent_result.overlayed | !CC.isempty(result) - parent_result = parent_result.matches::CC.MethodLookupResult - - return CC.MethodMatchResult( - CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig), - overlayed) - end - - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds, true - parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)), - overlayed) - end +# Priority-lookup method table. On 1.11+ we re-export CompilerCaching's; the 1.10 copy +# (with the older `MethodMatchResult`-returning `findall` signature) lives in deprecated.jl. +@static if HAS_INTEGRATED_CACHE + using CompilerCaching: StackedMethodTable end @@ -300,6 +223,33 @@ function CC.add_remark!(interp::GPUInterpreter, sv::CC.InferenceState, msg) @safe_debug "Inference remark during GPU compilation of $(sv.linfo): $msg" end +# Hook `CompilerCaching.CachedResult{V}` onto every CI we finish inferring, parametrically +# over `GPUInterpreter{V}`'s consumer results-type `V`. When `V === Nothing` (default — no +# back-end override of `results_type`) this collapses to a pass-through to the default +# `CC.finish!`, so back-ends that don't opt in pay no inference-time cost. +# +# Lives here (rather than CompilerCaching's `@setup_caching`) because the interpreter +# carries its `V` as a type parameter, not as a `cache::CacheView` field. Gated on the +# integrated cache (1.11+); on 1.10 there's no `analysis_results` to attach to. +@static if HAS_INTEGRATED_CACHE + @static if hasmethod(CC.finish!, Tuple{CC.AbstractInterpreter, CC.InferenceState, UInt, UInt64}) + function CC.finish!(interp::GPUInterpreter{V, MTV}, caller::CC.InferenceState, + validation_world::UInt, time_before::UInt64) where {V, MTV} + V === Nothing || CC.stack_analysis_result!(caller.result, + CompilerCaching.CachedResult{V}(V())) + @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState, + validation_world::UInt, time_before::UInt64) + end + else + function CC.finish!(interp::GPUInterpreter{V, MTV}, + caller::CC.InferenceState) where {V, MTV} + V === Nothing || CC.stack_analysis_result!(caller.result, + CompilerCaching.CachedResult{V}(V())) + @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState) + end + end +end + CC.may_optimize(interp::GPUInterpreter) = true CC.may_compress(interp::GPUInterpreter) = true CC.may_discard_trees(interp::GPUInterpreter) = true @@ -329,161 +279,25 @@ end ## driving inference and walking callees -# Drive type inference on `mi` using `interp`. Recursively walks callees so that on 1.12+ -# their CodeInstances are populated in the integrated cache with stored source — this is -# what `collect_codeinfos` later relies on to gather `(CI, CodeInfo)` pairs for -# `jl_emit_native`. Returns the root `CodeInstance` on 1.11+ (or `nothing` if inference -# failed); on 1.10, where there's no integrated cache to return into, returns `nothing` -# and the caller fetches the CI via `code_cache(interp)`. -function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) -@static if VERSION >= v"1.12.0-DEV.1434" - ci = CC.typeinf_ext(interp, mi, CC.SOURCE_MODE_NOT_REQUIRED) - ci === nothing && return nothing - - has_compilequeue = VERSION >= v"1.13.0-DEV.499" || v"1.12-beta3" <= VERSION < v"1.13-" - if has_compilequeue - workqueue = CC.CompilationQueue(; interp) - push!(workqueue, ci) - else - workqueue = CodeInstance[ci] - inspected = IdSet{CodeInstance}() - end - - while !isempty(workqueue) - callee = pop!(workqueue) - if has_compilequeue - CC.isinspected(workqueue, callee) && continue - CC.markinspected!(workqueue, callee) - else - callee in inspected && continue - push!(inspected, callee) - end - - callee_mi = CC.get_ci_mi(callee) - if CC.use_const_api(callee) - # const-return: synthesized CodeInfo later, no need to store source - continue - end - - src = CC.typeinf_code(interp, callee_mi, true) - if src isa CodeInfo - # Make sure the inferred source is persisted on the CI so we can read it back - # in `collect_codeinfos` (separate session or not). This is what - # `CompilerCaching.typeinf!` does internally; we inline it here so GPUCompiler - # itself doesn't depend on CompilerCaching. - if (@atomic callee.inferred) === nothing - @atomic callee.inferred = src - end - if has_compilequeue - sptypes = CC.sptypes_from_meth_instance(callee_mi) - CC.collectinvokes!(workqueue, src, sptypes) - else - CC.collectinvokes!(workqueue, src) - end - end - end - - return ci -elseif VERSION >= v"1.12.0-DEV.15" - inferred_ci = CC.typeinf_ext_toplevel(interp, mi, CC.SOURCE_MODE_FORCE_SOURCE) - @assert inferred_ci !== nothing "Inference of $mi failed" - - # `typeinf_ext_toplevel` will have populated the cache; in some const-return cases - # the inference result wasn't recorded — set it from the returned CI. - wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) - if CC.haskey(wvc, mi) - ci = CC.getindex(wvc, mi) - if ci.inferred === nothing - CC.setindex!(wvc, inferred_ci, mi) - end - end - - return inferred_ci -else - src = CC.typeinf_ext_toplevel(interp, mi) - @assert src !== nothing "Inference of $mi failed" - - # On 1.11/1.10 we look the CI up via the cache afterwards (no return from - # `typeinf_ext_toplevel` here). For const-return CIs, store the source explicitly - # so callers re-using the CI don't need to re-infer. - wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) - if CC.haskey(wvc, mi) - ci = CC.getindex(wvc, mi) - if ci.inferred === nothing - @atomic ci.inferred = src - end - return ci - end +# Construct a `CompilerCaching.CacheView` matching the interpreter's owner+world. `V` +# doesn't matter for the inference machinery (typeinf! / get_codeinfos) — they only +# use the view for `MethodInstance → CodeInstance` lookups — so we pick `Nothing`. +@static if HAS_INTEGRATED_CACHE + @inline cache_view(interp::GPUInterpreter) = + CompilerCaching.CacheView{typeof(interp.owner), Nothing}(interp.owner, interp.world) +end - return nothing -end -end - -# Retrieve CodeInfo for a CI, synthesizing one for const-return CIs (which never store -# inferred source). Returns `nothing` when no source is available — caller's choice -# whether that's an error. -function _ci_codeinfo(ci::CodeInstance) - raw = @atomic :monotonic ci.inferred - if raw isa CodeInfo - return raw - elseif raw isa Vector{UInt8} || raw isa String - mi = @static VERSION >= v"1.12-" ? CC.get_ci_mi(ci) : ci.def::MethodInstance - return ccall(:jl_uncompress_ir, Ref{CodeInfo}, (Any, Any, Any), - mi.def, ci, raw)::CodeInfo - elseif raw === nothing && CC.use_const_api(ci) - # const-return CIs skip source storage during inference; synthesize CodeInfo - # from the cached `rettype_const`. - mi = @static VERSION >= v"1.12-" ? CC.get_ci_mi(ci) : ci.def::MethodInstance - @static if VERSION >= v"1.13.0-DEV.1121" - src = CC.codeinfo_for_const(CC.NativeInterpreter(), mi, - CC.WorldRange(ci.min_world, ci.max_world), - ci.edges, ci.rettype_const) - elseif VERSION >= v"1.12-" - src = CC.codeinfo_for_const(CC.NativeInterpreter(), mi, ci.rettype_const) - # Work around 1.12/1.13 not setting nargs/isva in `codeinfo_for_const` - @static if v"1.12-" <= VERSION < v"1.14.0-DEV.60" - if src.nargs == 0 && mi.def isa Method - src.nargs = mi.def.nargs - src.isva = mi.def.isva - end - end - else - src = CC.codeinfo_for_const(CC.NativeInterpreter(), mi, ci.rettype_const) - end - return src - end - return nothing -end - -# Walk forward `:invoke` edges from `root` and collect `(CodeInstance, CodeInfo)` pairs -# for `jl_emit_native` (1.12+ payload format). Source is read back from the CIs that -# `drive_inference!` populated above. On 1.11 and older this is unused (those Julia -# versions take a list of `MethodInstance`s + a lookup callback). -function collect_codeinfos(root::CodeInstance) - pairs = Pair{CodeInstance, CodeInfo}[] - visited = IdSet{CodeInstance}() - queue = CodeInstance[root] - while !isempty(queue) - ci = pop!(queue) - ci in visited && continue - push!(visited, ci) - - src = _ci_codeinfo(ci) - src === nothing && continue - push!(pairs, ci => src) - - for stmt in src.code - if stmt isa Expr && stmt.head === :(=) - stmt = stmt.args[2] - end - if stmt isa Expr && (stmt.head === :invoke || - (VERSION >= v"1.12-" && stmt.head === :invoke_modify)) - callee = stmt.args[1] - callee isa CodeInstance && push!(queue, callee) - end - end - end - return pairs +# Drive type inference on `mi` using `interp`. On 1.11+ this delegates to +# `CompilerCaching.typeinf!`, which (on 1.12+) recursively walks callees so their CIs +# carry stored source for `CompilerCaching.get_codeinfos` to read back into the +# `jl_emit_native` payload. Returns the root `CodeInstance` (or `nothing` if inference +# failed). The 1.10 implementation lives in `deprecated.jl`. +@static if HAS_INTEGRATED_CACHE +function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) + cache = cache_view(interp) + CompilerCaching.typeinf!(cache, interp, mi) + return get(cache, mi, nothing) +end end @@ -616,7 +430,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # gather (CI, CodeInfo) pairs for jl_emit_native (1.12+) codeinfo_pairs = if VERSION >= v"1.12.0-DEV.1823" && root_ci !== nothing - collect_codeinfos(root_ci) + CompilerCaching.get_codeinfos(root_ci) else nothing end From 626c876368268eaf91d0c53a9a8a2f98e1a6189b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 14:28:51 +0200 Subject: [PATCH 11/33] Reword GPUInterpreter docstring after extension removal. --- src/jlgen.jl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/jlgen.jl b/src/jlgen.jl index 528dd5a6..4709350a 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -127,19 +127,22 @@ get_method_table_view(world::UInt, mt::CC.MethodTable) = CC.OverlayMethodTable(w const INFERENCE_CACHE_TYPE = isdefined(CC, :InferenceCache) ? CC.InferenceCache : Vector{CC.InferenceResult} """ - GPUInterpreter{MTV, V} + GPUInterpreter{V, MTV} Foreign abstract interpreter that drives Julia inference for GPU compilation. The `V` type parameter is the consumer's results-struct type (default `Nothing`). -On 1.11+ the `GPUCompiler` package extension `GPUCompilerCompilerCachingExt` -wires `CC.finish!` so that each newly-inferred `CodeInstance` carries a fresh -`V()` on its `analysis_results` chain; when `V === Nothing` (no consumer override -of `results_type`) or the extension isn't loaded, nothing is attached and -inference behaves like the default `NativeInterpreter`. +On 1.11+ the `CC.finish!` override below — parametric on `V` — attaches a fresh +`CachedResult{V}` to every newly-inferred `CodeInstance`'s `analysis_results` +chain. When `V === Nothing` (no consumer override of [`results_type`](@ref)) +nothing is attached and inference behaves like the default `NativeInterpreter`. The interpreter is partitioned by an `owner` token (1.11+) or an in-process -`CodeCache` (1.10, see [`deprecated.jl`](deprecated.jl)). +`CodeCache` (1.10, see [`deprecated.jl`](deprecated.jl)). The owner is what +makes per-job CIs distinct in the integrated cache; carrying it on the +interpreter (rather than a full `CacheView`) keeps inference state lean — the +cache view is constructed by callers (e.g. `cache_view(job)`) only when they +actually need to look up CIs or read consumer results. """ struct GPUInterpreter{V, MTV<:CC.MethodTableView} <: CC.AbstractInterpreter world::UInt From 6e3edde3438e12e5d3a9c6482afeb9f1aa712db3 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 15:12:01 +0200 Subject: [PATCH 12/33] Migrate to CompilerCaching's @setup_results + interp-form typeinf!. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the hand-rolled `CC.finish!` override with `@setup_results GPUInterpreter` plus a one-line `CompilerCaching.results_type` trait that reads V from the interpreter's type parameter. Same generated code, less boilerplate. `drive_inference!` collapses to a one-liner calling `CompilerCaching.typeinf!(interp, mi)` — which now constructs the CacheView internally and returns the root CI directly, saving the lookup the old cache-taking form required. The `cache_view(interp)` helper inside `jlgen.jl` goes away with it. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/jlgen.jl | 57 ++++++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/src/jlgen.jl b/src/jlgen.jl index 4709350a..9431e27f 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -226,31 +226,15 @@ function CC.add_remark!(interp::GPUInterpreter, sv::CC.InferenceState, msg) @safe_debug "Inference remark during GPU compilation of $(sv.linfo): $msg" end -# Hook `CompilerCaching.CachedResult{V}` onto every CI we finish inferring, parametrically -# over `GPUInterpreter{V}`'s consumer results-type `V`. When `V === Nothing` (default — no -# back-end override of `results_type`) this collapses to a pass-through to the default -# `CC.finish!`, so back-ends that don't opt in pay no inference-time cost. -# -# Lives here (rather than CompilerCaching's `@setup_caching`) because the interpreter -# carries its `V` as a type parameter, not as a `cache::CacheView` field. Gated on the -# integrated cache (1.11+); on 1.10 there's no `analysis_results` to attach to. +# Per-CI results attachment. `results_type` reads V from the interpreter's type +# parameter; `@setup_results` generates the `CC.finish!` hook that attaches a +# fresh `CachedResult{V}` to every newly-inferred CodeInstance. When V === Nothing +# (default — no consumer override) the hook short-circuits to the default +# `CC.finish!`. Gated on the integrated cache (1.11+); on 1.10 there's no +# `analysis_results` to attach to. @static if HAS_INTEGRATED_CACHE - @static if hasmethod(CC.finish!, Tuple{CC.AbstractInterpreter, CC.InferenceState, UInt, UInt64}) - function CC.finish!(interp::GPUInterpreter{V, MTV}, caller::CC.InferenceState, - validation_world::UInt, time_before::UInt64) where {V, MTV} - V === Nothing || CC.stack_analysis_result!(caller.result, - CompilerCaching.CachedResult{V}(V())) - @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState, - validation_world::UInt, time_before::UInt64) - end - else - function CC.finish!(interp::GPUInterpreter{V, MTV}, - caller::CC.InferenceState) where {V, MTV} - V === Nothing || CC.stack_analysis_result!(caller.result, - CompilerCaching.CachedResult{V}(V())) - @invoke CC.finish!(interp::CC.AbstractInterpreter, caller::CC.InferenceState) - end - end + CompilerCaching.results_type(::GPUInterpreter{V}) where V = V + CompilerCaching.@setup_results GPUInterpreter end CC.may_optimize(interp::GPUInterpreter) = true @@ -282,25 +266,14 @@ end ## driving inference and walking callees -# Construct a `CompilerCaching.CacheView` matching the interpreter's owner+world. `V` -# doesn't matter for the inference machinery (typeinf! / get_codeinfos) — they only -# use the view for `MethodInstance → CodeInstance` lookups — so we pick `Nothing`. +# Drive type inference on `mi` using `interp`. On 1.11+ this is `CompilerCaching.typeinf!`, +# which (on 1.12+) recursively walks callees so their CIs carry stored source for +# `CompilerCaching.get_codeinfos` to read back into the `jl_emit_native` payload. Returns +# the root `CodeInstance` (or `nothing` if inference failed). The 1.10 implementation +# lives in `deprecated.jl`. @static if HAS_INTEGRATED_CACHE - @inline cache_view(interp::GPUInterpreter) = - CompilerCaching.CacheView{typeof(interp.owner), Nothing}(interp.owner, interp.world) -end - -# Drive type inference on `mi` using `interp`. On 1.11+ this delegates to -# `CompilerCaching.typeinf!`, which (on 1.12+) recursively walks callees so their CIs -# carry stored source for `CompilerCaching.get_codeinfos` to read back into the -# `jl_emit_native` payload. Returns the root `CodeInstance` (or `nothing` if inference -# failed). The 1.10 implementation lives in `deprecated.jl`. -@static if HAS_INTEGRATED_CACHE -function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) - cache = cache_view(interp) - CompilerCaching.typeinf!(cache, interp, mi) - return get(cache, mi, nothing) -end + drive_inference!(interp::GPUInterpreter, mi::MethodInstance) = + CompilerCaching.typeinf!(interp, mi) end From de2d198a43cadc02631bb25cf13d185e11df58c6 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 14 May 2026 16:27:53 +0200 Subject: [PATCH 13/33] Replace bitcode/bitcode! traits with a generic cache_get/cache_put! protocol. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old `bitcode(results)` / `bitcode!(results, bytes)` pair was a single-purpose hook bolted onto the consumer's results struct. Extending it to memoize more phases (LLVM IR, intermediate AIR, whatever) meant adding a parallel pair per phase. Worse, it required the consumer's results struct to be in the loop on every cache touch — which forced `rtlib.jl` to know about `CompilerCaching` to do the per-CI lookup that fetched the right results instance. Replace with a back-end-managed key→bytes protocol on `CompilerJob`: cache_get(job::CompilerJob, key::Symbol) -> Union{Nothing, Vector{UInt8}} cache_put!(job::CompilerJob, key::Symbol, ::Vector{UInt8}) GPUCompiler hands the back-end a job + a key (`:llvm_ir` currently — the post-irgen LLVM bitcode for runtime library functions). The back-end stores it wherever it likes — typically on a CI's `analysis_results` via CompilerCaching, but it could equally be an in-memory `Dict`, on-disk storage, or nothing. The default no-op pair means no caching. `rtlib.jl` no longer imports `CompilerCaching` — it just calls the hooks. New phase keys can be added without growing the API surface; back-ends opt in selectively by matching on keys they care about. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/interface.jl | 52 +++++++++++++++++++++++++++++------------------- src/rtlib.jl | 45 +++++++++++++---------------------------- 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 0f5aa8c6..7931c62c 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -347,27 +347,37 @@ cache_owner(@nospecialize(job::CompilerJob)) = results_type(@nospecialize(job::CompilerJob)) = Nothing """ - bitcode(results) -> Union{Nothing, Vector{UInt8}} - bitcode!(results, bytes::Vector{UInt8}) -> Nothing - -Optional consumer hooks for stashing post-irgen LLVM bitcode on the results struct. -GPUCompiler currently uses them from [`emit_function!`](@ref) to memoize each runtime -function's bitcode on its `CodeInstance` (1.11+, via `analysis_results`); back-ends can -populate the slot from any compile they want to memoize at the LLVM stage. - -Override both on your [`results_type`](@ref) to opt in. The default pair (`bitcode` → -`nothing`, `bitcode!` → no-op) means no LLVM-stage caching happens. - -The LLVM context's pointer mode (opaque vs. typed) is assumed fixed for the lifetime of -a session — and across precompile/load pairs of the same Julia version, since pkgimages -are invalidated when the toolchain changes. Back-ends don't need to gate caching on -that mode. - -On Julia 1.10 these hooks are never invoked (no integrated cache to stash bitcode on); -runtime libraries fall back to the session-local `runtime_libs` cache. + cache_get(job::CompilerJob, key::Symbol) -> Union{Nothing, Vector{UInt8}} + cache_put!(job::CompilerJob, key::Symbol, value::Vector{UInt8}) -> Nothing + +Optional caching protocol. GPUCompiler calls these around expensive compilation +phases to memoize byte artifacts; the back-end implements them however it likes +(per-CI results via CompilerCaching, an in-memory `Dict`, on-disk storage, …). + +`job` identifies the artifact's source — typically the back-end keys storage on +`job.source` and `cache_owner(job)`. `key` is a `Symbol` naming the artifact +kind. Back-ends are free to honour only the keys they care about and ignore the +rest. + +Keys currently used by GPUCompiler: + +- `:llvm_ir` — post-irgen LLVM IR (serialized as bitcode bytes) for runtime + library functions (`gpu_malloc`, `gpu_report_exception`, …) emitted by + [`emit_function!`](@ref). Each runtime function is its own `job` with its + source MI; the bytes are session-portable (they survive precompilation when + the back-end stores them somewhere pkgimage-managed, e.g. on a `CodeInstance`). + +The LLVM context's pointer mode (opaque vs. typed) is assumed fixed for the +lifetime of a session — and across precompile/load pairs of the same Julia +version, since pkgimages are invalidated when the toolchain changes. Back-ends +don't need to gate caching on that mode. + +Default no-op: no caching happens. Override on your `CompilerJob` type alias +(e.g. `MetalCompilerJob = CompilerJob{MetalCompilerTarget, MetalCompilerParams}`) +to opt in. """ -bitcode(@nospecialize(results)) = nothing -bitcode!(@nospecialize(results), bytes::Vector{UInt8}) = nothing +cache_get(@nospecialize(job::CompilerJob), key::Symbol) = nothing +cache_put!(@nospecialize(job::CompilerJob), key::Symbol, value::Vector{UInt8}) = nothing @static if HAS_INTEGRATED_CACHE """ @@ -384,7 +394,7 @@ bitcode!(@nospecialize(results), bytes::Vector{UInt8}) = nothing end end -@public GPUCompilerCacheToken, cache_owner, results_type, bitcode, bitcode! +@public GPUCompilerCacheToken, cache_owner, results_type, cache_get, cache_put! @static if HAS_INTEGRATED_CACHE @public cache_view end diff --git a/src/rtlib.jl b/src/rtlib.jl index 7e5293e2..1f572ff5 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,33 +60,24 @@ end ## functionality to build the runtime library -# Compile a single runtime function and link it into `mod`. On Julia 1.11+ each -# runtime function's renamed bitcode is cached on its `CodeInstance`'s -# `analysis_results` when the back-end opts in via the `bitcode` / `bitcode!` -# trait pair — cross-session persistence rides on package precompilation. On -# 1.10 (no integrated cache) we fall through to plain compile-and-link; the -# session-local `runtime_libs` cache below avoids repeating that work -# within a session. +# Compile a single runtime function and link it into `mod`. Memoizes the renamed +# bitcode via the back-end's `cache_get`/`cache_put!` protocol — cross-session +# persistence is the back-end's responsibility (typically by storing on a CI's +# `analysis_results` via CompilerCaching). The session-local `runtime_libs` +# cache below avoids repeating the parse-and-link work within a session. function emit_function!(mod, config::CompilerConfig, f, method) tt = Base.to_tuple_type(method.types) source = generic_methodinstance(f, tt) name = method.llvm_name + rt_job = CompilerJob(source, config) - @static if HAS_INTEGRATED_CACHE - rt_job = CompilerJob(source, config) - cache = cache_view(rt_job) - hit = CompilerCaching.lookup(cache, source) - if hit !== nothing - cached = bitcode(hit[2]) - if cached !== nothing - func_mod = parse(LLVM.Module, MemoryBuffer(cached)) - link!(mod, func_mod) - return - end - end + cached = cache_get(rt_job, :llvm_ir) + if cached !== nothing + link!(mod, parse(LLVM.Module, MemoryBuffer(cached))) + return end - new_mod, meta = compile_unhooked(:llvm, CompilerJob(source, config)) + new_mod, meta = compile_unhooked(:llvm, rt_job) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) if return_type(ft) != return_type(expected_ft) @@ -106,17 +97,9 @@ function emit_function!(mod, config::CompilerConfig, f, method) end LLVM.name!(meta.entry, name) - @static if HAS_INTEGRATED_CACHE - # Re-lookup: inference for this runtime function's MI may have happened - # inside `compile_unhooked` (callee walk or `precompile`), so the CI - # exists now even if the pre-compile lookup returned nothing. - hit = CompilerCaching.lookup(cache, source) - if hit !== nothing - io = IOBuffer() - write(io, new_mod) - bitcode!(hit[2], take!(io)) - end - end + io = IOBuffer() + write(io, new_mod) + cache_put!(rt_job, :llvm_ir, take!(io)) link!(mod, new_mod) end From afcb27fa8917d6536c9599d466d2e54cc604eca5 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 10 Jun 2026 22:43:44 +0200 Subject: [PATCH 14/33] Rework the caching interface around per-job results structs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the previous V-threaded design (GPUInterpreter{V}, results_type(job), @setup_results, cache_get/cache_put!, cache_view) with a single back-end-facing entry point: cached_results(::Type{V}, job::CompilerJob)::V which returns the (lazily created) results struct for a job. Back-ends define one mutable struct holding their per-stage artifacts, check completeness, and compile into it — a single code path on all supported Julia versions: - On 1.11+, the struct lives on the CodeInstance in Julia's integrated cache (running inference to create one when needed), wrapped in a config-keyed JobResults container. CompilerCaching attaches results lazily, so the GPUInterpreter no longer carries a results type, and independent consumers (e.g. our own runtime-library cache) can attach to the same CI. - On 1.10, the struct lives in a session-local Dict keyed by the same job identity, kept alongside the other legacy code in deprecated.jl. Keying results by the full CompilerConfig (not just cache_owner) fixes a latent bug in the previous design where two jobs differing only in codegen settings — e.g. the kernel name — would share artifacts. The owner token still covers only what affects inference, so inference results remain shared across such jobs. The runtime library now uses the same mechanism: emit_function! memoizes each runtime function's renamed bitcode in a RuntimeFunctionResults attached to its CI, replacing the cache_get/cache_put! protocol. Back-ends no longer opt in: runtime bitcode persists through precompilation automatically on 1.11+. Also moves the 1.10 drive_inference! definition from deprecated.jl to jlgen.jl: its signature references GPUInterpreter, which isn't defined yet when deprecated.jl is included (1.10 loading was broken on the previous branch), and fixes the ptx precompile test to construct its cache token from the standalone package's helper module (the sandbox copy defines a distinct CompilerParams type, so its token can never match the precompiled CIs). Co-Authored-By: Claude Fable 5 --- Project.toml | 5 +- TODO | 1 + TODO.md | 14 ++++ src/GPUCompiler.jl | 6 ++ src/deprecated.jl | 35 ++++---- src/interface.jl | 162 ++++++++++++++++++++++---------------- src/jlgen.jl | 87 ++++++++++---------- src/rtlib.jl | 27 ++++--- test/Project.toml | 6 ++ test/native.jl | 48 +++++++++++ test/native/precompile.jl | 9 ++- test/ptx/precompile.jl | 14 +++- 12 files changed, 275 insertions(+), 139 deletions(-) create mode 100644 TODO create mode 100644 TODO.md diff --git a/Project.toml b/Project.toml index d4ad4bda..1140aa80 100644 --- a/Project.toml +++ b/Project.toml @@ -25,10 +25,13 @@ AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444" NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" +[sources] +CompilerCaching = {path = "../CompilerCaching"} + [compat] AMDGPU_LLVM_Backend_jll = "22" -CompilerCaching = "0.3" +CompilerCaching = "0.4" ExprTools = "0.1" InteractiveUtils = "1" LLVM = "9.9" diff --git a/TODO b/TODO new file mode 100644 index 00000000..7db75e11 --- /dev/null +++ b/TODO @@ -0,0 +1 @@ +value materializer: just put f/new_f in value map! diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..1ba2897a --- /dev/null +++ b/TODO.md @@ -0,0 +1,14 @@ +TODO: + +The _methods_by_ftype call also (optionally) returns age bounds, which you are required to propagate (usually pretty easy, since it is just min/max calls + +_methods_by_ftype also takes mt argument; for overlays of kernels? +also `which` in driver + + +# put the function and argument types in typevars +# so that we can access it from generated functions + +remove that. we just use .f and .tt in the world_age generator +and rename f to ft field +maybe pass sig everywhere? diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index ebaacae6..57c8d7e3 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -84,6 +84,12 @@ include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) + @static if !HAS_INTEGRATED_CACHE + # session-local results keyed by objectid+world; entries serialized during + # GPUCompiler's own precompilation can never be valid in a later session + empty!(job_results) + end + @static if ENABLE_TRACY Tracy.@register_tracepoints() end diff --git a/src/deprecated.jl b/src/deprecated.jl index bed01897..032f0608 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -229,27 +229,28 @@ function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) end -## 1.10 inference driver +## 1.10 `cached_results` # -# On 1.11+ this is delegated to `CompilerCaching.typeinf!`; here we keep an inline copy -# that talks to the per-interpreter `CodeCache` instead of the integrated one. Returns -# `nothing` (no integrated CI to hand back; the caller fetches via `code_cache(interp)`). -function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) - src = CC.typeinf_ext_toplevel(interp, mi) - @assert src !== nothing "Inference of $mi failed" - - # For const-return CIs the inference result wasn't recorded — set it from the - # returned source so callers re-using the CI don't need to re-infer. - wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) - if CC.haskey(wvc, mi) - ci = CC.getindex(wvc, mi) - if ci.inferred === nothing - @atomic ci.inferred = src - end +# Session-local storage for the per-job results structs; on 1.11+ these live on the +# `CodeInstance`s of Julia's integrated cache instead (see `interface.jl`). The key +# mirrors the 1.11+ semantics — the full job identity, i.e. method instance, world, +# and entire config — but nothing persists across sessions, and redefinition +# protection comes from the world age in the key rather than from CI invalidation. +const job_results = Dict{Any,Any}() +const job_results_lock = ReentrantLock() + +function cached_results(::Type{V}, @nospecialize(job::CompilerJob)) where {V} + # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive + # boxing allocation; the MI is kept alive by its method specializations. + key = (V, objectid(job.source), job.world, job.config) + Base.@lock job_results_lock begin + get!(job_results, key) do + V() + end::V end - return nothing end + end # !HAS_INTEGRATED_CACHE diff --git a/src/interface.jl b/src/interface.jl index 7931c62c..e3778bd8 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -263,22 +263,19 @@ isintrinsic(@nospecialize(job::CompilerJob), fn::String) = false # provide a specific interpreter to use. @static if HAS_INTEGRATED_CACHE function get_interpreter(@nospecialize(job::CompilerJob)) - V = results_type(job) - GPUInterpreter{V}(job.world; - method_table_view=maybe_cached(method_table_view(job)), - owner=cache_owner(job), - inf_params=inference_params(job), - opt_params=optimization_params(job)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + owner=cache_owner(job), + inf_params=inference_params(job), + opt_params=optimization_params(job)) end else function get_interpreter(@nospecialize(job::CompilerJob)) - V = results_type(job) - cache = get_code_cache(job) - GPUInterpreter{V}(job.world; - method_table_view=maybe_cached(method_table_view(job)), - code_cache=cache, - inf_params=inference_params(job), - opt_params=optimization_params(job)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + code_cache=get_code_cache(job), + inf_params=inference_params(job), + opt_params=optimization_params(job)) end end @@ -336,67 +333,100 @@ cache_owner(@nospecialize(job::CompilerJob)) = GPUCompilerCacheToken(job.config.target, job.config.params, job.config.always_inline, method_table(job)) -# The consumer's results struct type, stored on each `CodeInstance` (1.11+) via the -# `CompilerCaching` extension. Override to attach session-portable artifacts (e.g. IR or -# object bytes) and session-local handles (e.g. `CuModule`, `MTLComputePipelineState`). -# The struct must be a `mutable struct` with a zero-arg constructor. -# -# When the consumer hasn't overridden, the default `Nothing` opts out: no results struct -# is attached during inference, and the `analysis_results` chain is untouched. This is the -# right default for reflection paths, precompile workloads, and the legacy 1.10 flow. -results_type(@nospecialize(job::CompilerJob)) = Nothing - """ - cache_get(job::CompilerJob, key::Symbol) -> Union{Nothing, Vector{UInt8}} - cache_put!(job::CompilerJob, key::Symbol, value::Vector{UInt8}) -> Nothing - -Optional caching protocol. GPUCompiler calls these around expensive compilation -phases to memoize byte artifacts; the back-end implements them however it likes -(per-CI results via CompilerCaching, an in-memory `Dict`, on-disk storage, …). - -`job` identifies the artifact's source — typically the back-end keys storage on -`job.source` and `cache_owner(job)`. `key` is a `Symbol` naming the artifact -kind. Back-ends are free to honour only the keys they care about and ignore the -rest. - -Keys currently used by GPUCompiler: - -- `:llvm_ir` — post-irgen LLVM IR (serialized as bitcode bytes) for runtime - library functions (`gpu_malloc`, `gpu_report_exception`, …) emitted by - [`emit_function!`](@ref). Each runtime function is its own `job` with its - source MI; the bytes are session-portable (they survive precompilation when - the back-end stores them somewhere pkgimage-managed, e.g. on a `CodeInstance`). - -The LLVM context's pointer mode (opaque vs. typed) is assumed fixed for the -lifetime of a session — and across precompile/load pairs of the same Julia -version, since pkgimages are invalidated when the toolchain changes. Back-ends -don't need to gate caching on that mode. - -Default no-op: no caching happens. Override on your `CompilerJob` type alias -(e.g. `MetalCompilerJob = CompilerJob{MetalCompilerTarget, MetalCompilerParams}`) -to opt in. + cached_results(::Type{V}, job::CompilerJob) -> V + +Retrieve the compilation-results struct of type `V` for `job`, creating an empty one on +first access. `V` must be a `mutable struct` with a zero-arg constructor; back-ends +define one such struct holding the artifacts of each compilation stage, check whether +the relevant fields are populated, and fill them in (running the compiler) when not: + +```julia +mutable struct MetalResults + metallib::Union{Nothing,Vector{UInt8}} + entry::Union{Nothing,String} + pipelines::Vector{Tuple{MTLDevice,MTLComputePipelineState}} # session-local + MetalResults() = new(nothing, nothing, []) +end + +res = GPUCompiler.cached_results(MetalResults, job) +if res.metallib === nothing || GPUCompiler.compile_hook[] !== nothing + ...compile, populate res... +end +``` + +Results are keyed by the *full* compiler job: its method instance, world age, and entire +`CompilerConfig` (two jobs differing only in, say, the kernel `name` get distinct +structs). Storage differs per Julia version, transparently to the back-end: + +- On Julia 1.11+, the struct lives on the `CodeInstance`'s `analysis_results` chain in + Julia's integrated code cache, partitioned by [`cache_owner`](@ref) and keyed by + config within the per-CI [`JobResults`](@ref) container. Method redefinition + invalidates the CI — and with it the attached results. When no CI exists yet, + inference is run to create one. Artifacts stored during precompilation are serialized + into the package image along with the CI: populate only session-portable values + (bytes, strings) during precompile workloads, and keep session-local handles (device + modules, pipeline objects) in fields that remain empty until first use at run time. + +- On Julia 1.10, the struct lives in a session-local Dict; redefinition protection + comes from the world age in the key, and nothing persists across sessions. + +Thread safety: concurrent calls for the same job return the same struct, but +GPUCompiler does not serialize back-end *compilation*; take a back-end lock around +the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock`). """ -cache_get(@nospecialize(job::CompilerJob), key::Symbol) = nothing -cache_put!(@nospecialize(job::CompilerJob), key::Symbol, value::Vector{UInt8}) = nothing +function cached_results end @static if HAS_INTEGRATED_CACHE - """ - cache_view(job::CompilerJob) -> CompilerCaching.CacheView - - Construct a `CacheView{typeof(cache_owner(job)), results_type(job)}` over Julia's - integrated `CodeInstance` cache at `job.world`. The handle back-ends pass to - `CompilerCaching.lookup` / `CompilerCaching.results` when populating their own - per-CI artifacts. - """ - function cache_view(@nospecialize(job::CompilerJob)) - owner = cache_owner(job) - CompilerCaching.CacheView{typeof(owner), results_type(job)}(owner, job.world) + +""" + JobResults + +Per-CodeInstance container mapping a `CompilerConfig` to the back-end's results struct. + +A `CodeInstance` is shared by every job whose [`cache_owner`](@ref) matches — the owner +token only covers what affects *inference* (target, params, `always_inline`, method +table). The remaining `CompilerConfig` fields (`kernel`, `name`, `entry_abi`, +`opt_level`, …) do affect *codegen*, so artifacts must not be shared across them. +Entries are matched by `===` (`jl_egal`): `CompilerConfig` is an immutable struct, so +structurally identical configs compare equal, including against configs deserialized +from package images. +""" +mutable struct JobResults + entries::Vector{Tuple{CompilerConfig,Any}} + JobResults() = new(Tuple{CompilerConfig,Any}[]) +end + +const cached_results_lock = ReentrantLock() + +function cached_results(::Type{V}, @nospecialize(job::CompilerJob)) where {V} + owner = cache_owner(job) + cache = CompilerCaching.CacheView{JobResults}(owner, job.world) + + # get a CI for the job, running inference to create one if needed + ci = get(cache, job.source, nothing) + if ci === nothing + interp = get_interpreter(job) + ci = CompilerCaching.typeinf!(interp, job.source) + ci === nothing && error("Inference of $(job.source) failed") + end + + jr = CompilerCaching.results(JobResults, ci) + Base.@lock cached_results_lock begin + for (config, v) in jr.entries + config === job.config && v isa V && return v::V + end + v = V() + push!(jr.entries, (job.config, v)) + return v end end -@public GPUCompilerCacheToken, cache_owner, results_type, cache_get, cache_put! +end # HAS_INTEGRATED_CACHE + +@public GPUCompilerCacheToken, cache_owner, cached_results @static if HAS_INTEGRATED_CACHE - @public cache_view + @public JobResults end # the method table to use diff --git a/src/jlgen.jl b/src/jlgen.jl index 9431e27f..b46ff997 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -127,24 +127,18 @@ get_method_table_view(world::UInt, mt::CC.MethodTable) = CC.OverlayMethodTable(w const INFERENCE_CACHE_TYPE = isdefined(CC, :InferenceCache) ? CC.InferenceCache : Vector{CC.InferenceResult} """ - GPUInterpreter{V, MTV} + GPUInterpreter Foreign abstract interpreter that drives Julia inference for GPU compilation. -The `V` type parameter is the consumer's results-struct type (default `Nothing`). -On 1.11+ the `CC.finish!` override below — parametric on `V` — attaches a fresh -`CachedResult{V}` to every newly-inferred `CodeInstance`'s `analysis_results` -chain. When `V === Nothing` (no consumer override of [`results_type`](@ref)) -nothing is attached and inference behaves like the default `NativeInterpreter`. - The interpreter is partitioned by an `owner` token (1.11+) or an in-process `CodeCache` (1.10, see [`deprecated.jl`](deprecated.jl)). The owner is what -makes per-job CIs distinct in the integrated cache; carrying it on the -interpreter (rather than a full `CacheView`) keeps inference state lean — the -cache view is constructed by callers (e.g. `cache_view(job)`) only when they -actually need to look up CIs or read consumer results. +makes per-job CIs distinct in the integrated cache; together with the world +age it is all CompilerCaching needs to drive inference into the integrated +cache. Compilation results are not attached during inference at all — they +are attached lazily when looked up (see [`cached_results`](@ref)). """ -struct GPUInterpreter{V, MTV<:CC.MethodTableView} <: CC.AbstractInterpreter +struct GPUInterpreter{MTV<:CC.MethodTableView} <: CC.AbstractInterpreter world::UInt method_table_view::MTV @@ -159,25 +153,25 @@ end end @static if HAS_INTEGRATED_CACHE -function GPUInterpreter{V}(world::UInt=Base.get_world_counter(); - method_table_view::CC.MethodTableView, - owner::Any, - inf_params::CC.InferenceParams, - opt_params::CC.OptimizationParams) where V +function GPUInterpreter(world::UInt=Base.get_world_counter(); + method_table_view::CC.MethodTableView, + owner::Any, + inf_params::CC.InferenceParams, + opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - return GPUInterpreter{V, typeof(method_table_view)}( + return GPUInterpreter{typeof(method_table_view)}( world, method_table_view, owner, INFERENCE_CACHE_TYPE(), inf_params, opt_params) end -function GPUInterpreter(interp::GPUInterpreter{V, MTV}; +function GPUInterpreter(interp::GPUInterpreter; world::UInt=interp.world, method_table_view::CC.MethodTableView=interp.method_table_view, owner::Any=interp.owner, inf_cache::INFERENCE_CACHE_TYPE=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, - opt_params::CC.OptimizationParams=interp.opt_params) where {MTV, V} - return GPUInterpreter{V, typeof(method_table_view)}( + opt_params::CC.OptimizationParams=interp.opt_params) + return GPUInterpreter{typeof(method_table_view)}( world, method_table_view, owner, inf_cache, inf_params, opt_params) end @@ -186,25 +180,25 @@ CC.cache_owner(interp::GPUInterpreter) = interp.owner else # 1.10: in-process CodeCache -function GPUInterpreter{V}(world::UInt=Base.get_world_counter(); - method_table_view::CC.MethodTableView, - code_cache::CodeCache, - inf_params::CC.InferenceParams, - opt_params::CC.OptimizationParams) where V +function GPUInterpreter(world::UInt=Base.get_world_counter(); + method_table_view::CC.MethodTableView, + code_cache::CodeCache, + inf_params::CC.InferenceParams, + opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - return GPUInterpreter{V, typeof(method_table_view)}( + return GPUInterpreter{typeof(method_table_view)}( world, method_table_view, code_cache, Vector{CC.InferenceResult}(), inf_params, opt_params) end -function GPUInterpreter(interp::GPUInterpreter{V, MTV}; +function GPUInterpreter(interp::GPUInterpreter; world::UInt=interp.world, method_table_view::CC.MethodTableView=interp.method_table_view, code_cache::CodeCache=interp.code_cache, inf_cache::Vector{CC.InferenceResult}=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, - opt_params::CC.OptimizationParams=interp.opt_params) where {MTV, V} - return GPUInterpreter{V, typeof(method_table_view)}( + opt_params::CC.OptimizationParams=interp.opt_params) + return GPUInterpreter{typeof(method_table_view)}( world, method_table_view, code_cache, inf_cache, inf_params, opt_params) end @@ -226,17 +220,6 @@ function CC.add_remark!(interp::GPUInterpreter, sv::CC.InferenceState, msg) @safe_debug "Inference remark during GPU compilation of $(sv.linfo): $msg" end -# Per-CI results attachment. `results_type` reads V from the interpreter's type -# parameter; `@setup_results` generates the `CC.finish!` hook that attaches a -# fresh `CachedResult{V}` to every newly-inferred CodeInstance. When V === Nothing -# (default — no consumer override) the hook short-circuits to the default -# `CC.finish!`. Gated on the integrated cache (1.11+); on 1.10 there's no -# `analysis_results` to attach to. -@static if HAS_INTEGRATED_CACHE - CompilerCaching.results_type(::GPUInterpreter{V}) where V = V - CompilerCaching.@setup_results GPUInterpreter -end - CC.may_optimize(interp::GPUInterpreter) = true CC.may_compress(interp::GPUInterpreter) = true CC.may_discard_trees(interp::GPUInterpreter) = true @@ -269,11 +252,29 @@ end # Drive type inference on `mi` using `interp`. On 1.11+ this is `CompilerCaching.typeinf!`, # which (on 1.12+) recursively walks callees so their CIs carry stored source for # `CompilerCaching.get_codeinfos` to read back into the `jl_emit_native` payload. Returns -# the root `CodeInstance` (or `nothing` if inference failed). The 1.10 implementation -# lives in `deprecated.jl`. +# the root `CodeInstance` (or `nothing` if inference failed). @static if HAS_INTEGRATED_CACHE drive_inference!(interp::GPUInterpreter, mi::MethodInstance) = CompilerCaching.typeinf!(interp, mi) +else + # 1.10: an inline copy that talks to the per-interpreter `CodeCache` instead of the + # integrated one. Returns `nothing` (no integrated CI to hand back; the caller + # fetches via `code_cache(interp)`). + function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) + src = CC.typeinf_ext_toplevel(interp, mi) + @assert src !== nothing "Inference of $mi failed" + + # For const-return CIs the inference result wasn't recorded — set it from the + # returned source so callers re-using the CI don't need to re-infer. + wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) + if CC.haskey(wvc, mi) + ci = CC.getindex(wvc, mi) + if ci.inferred === nothing + @atomic ci.inferred = src + end + end + return nothing + end end diff --git a/src/rtlib.jl b/src/rtlib.jl index 1f572ff5..016053fa 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,20 +60,29 @@ end ## functionality to build the runtime library -# Compile a single runtime function and link it into `mod`. Memoizes the renamed -# bitcode via the back-end's `cache_get`/`cache_put!` protocol — cross-session -# persistence is the back-end's responsibility (typically by storing on a CI's -# `analysis_results` via CompilerCaching). The session-local `runtime_libs` -# cache below avoids repeating the parse-and-link work within a session. +# Per-function compilation results for the GPU runtime library, cached through the +# same `cached_results` mechanism back-ends use for kernels. On 1.11+ the bitcode +# thus lives on the runtime function's `CodeInstance` — possibly alongside a +# back-end's own results struct — and persists through precompilation, so sessions +# loading a back-end that compiled its runtime during precompile skip codegen +# entirely. On 1.10 it is cached for the duration of the session. +mutable struct RuntimeFunctionResults + bitcode::Union{Nothing,Vector{UInt8}} + RuntimeFunctionResults() = new(nothing) +end + +# Compile a single runtime function and link it into `mod`. The renamed bitcode is +# memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache +# below additionally avoids repeating the parse-and-link work within a session. function emit_function!(mod, config::CompilerConfig, f, method) tt = Base.to_tuple_type(method.types) source = generic_methodinstance(f, tt) name = method.llvm_name rt_job = CompilerJob(source, config) - cached = cache_get(rt_job, :llvm_ir) - if cached !== nothing - link!(mod, parse(LLVM.Module, MemoryBuffer(cached))) + res = cached_results(RuntimeFunctionResults, rt_job) + if res.bitcode !== nothing + link!(mod, parse(LLVM.Module, MemoryBuffer(res.bitcode))) return end @@ -99,7 +108,7 @@ function emit_function!(mod, config::CompilerConfig, f, method) io = IOBuffer() write(io, new_mod) - cache_put!(rt_job, :llvm_ir, take!(io)) + res.bitcode = take!(io) link!(mod, new_mod) end diff --git a/test/Project.toml b/test/Project.toml index d82dab81..103e21ce 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" FileCheck = "4e644321-382b-4b05-b0b6-5d23c3d944fb" GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -18,6 +19,11 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" demumble_jll = "1e29f10c-031c-5a83-9565-69cddfc27673" +[sources] +CompilerCaching = {path = "../../CompilerCaching"} +GPUCompiler = {path = ".."} +LLVM = {path = "/Users/tim/Julia/pkg/LLVM"} + [compat] Aqua = "0.8" ParallelTestRunner = "2" diff --git a/test/native.jl b/test/native.jl index 9574ff7e..c6f8a26c 100644 --- a/test/native.jl +++ b/test/native.jl @@ -120,6 +120,54 @@ end end end + @testset "cached results" begin + mod = @eval module $(gensym()) + mutable struct Results + asm::Union{Nothing,String} + Results() = new(nothing) + end + mutable struct OtherResults + data::Any + OtherResults() = new(nothing) + end + + @noinline child(i) = i + kernel(i) = child(i)+1 + end + + # get-or-create: first access yields an empty struct, later accesses the same one + job, _ = Native.create_job(mod.kernel, (Int64,)) + res = GPUCompiler.cached_results(mod.Results, job) + @test res isa mod.Results + @test res.asm === nothing + res.asm = "compiled" + @test GPUCompiler.cached_results(mod.Results, job) === res + + # independent consumers get independent structs for the same job + other = GPUCompiler.cached_results(mod.OtherResults, job) + @test other isa mod.OtherResults + @test GPUCompiler.cached_results(mod.Results, job) === res + + # results are keyed by the full config: a job differing only in codegen-level + # settings (here: the kernel name) must not share artifacts + named_job, _ = Native.create_job(mod.kernel, (Int64,); name="custom") + @test named_job.source === job.source + named_res = GPUCompiler.cached_results(mod.Results, named_job) + @test named_res !== res + @test named_res.asm === nothing + + # ... but an equal config constructed from scratch resolves to the same struct + job2, _ = Native.create_job(mod.kernel, (Int64,)) + @test GPUCompiler.cached_results(mod.Results, job2) === res + + # redefinition invalidates: a job in the new world gets a fresh struct + @eval mod kernel(i) = child(i)+2 + new_job, _ = Native.create_job(mod.kernel, (Int64,)) + new_res = GPUCompiler.cached_results(mod.Results, new_job) + @test new_res !== res + @test new_res.asm === nothing + end + @testset "allowed mutable types" begin # when types have no fields, we should always allow them mod = @eval module $(gensym()) diff --git a/test/native/precompile.jl b/test/native/precompile.jl index 9917eeb7..c1fb91ae 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -87,7 +87,14 @@ precompile_test_harness("Inference caching") do load_path @test check_presence(square_mi, token) # check that identity survived - @test check_presence(identity_mi, token) broken=(v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.13.0-" <= VERSION < v"1.13.0-beta3"|| v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843") + # NOTE: on 1.13, external CIs from the workload survive only flakily (the + # 1.13.0-beta3 backport did not fully fix this), so skip the check there + ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || + v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" + ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" + if !ext_cis_flaky + @test check_presence(identity_mi, token) broken=ext_cis_lost + end # Recompiling a foreign method after loading precompiled owner-token CIs # may also surface a native owner-less CI for the same MethodInstance. diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl index 853dce8d..0e1a5b9b 100644 --- a/test/ptx/precompile.jl +++ b/test/ptx/precompile.jl @@ -32,8 +32,11 @@ precompile_test_harness("Inference caching") do load_path # Check that no cached entry is present identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + # NOTE: use the standalone package's module to construct the token — the + # sandbox's own PTX helper module defines a distinct CompilerParams + # type, whose token can never match the precompiled CIs. token = let - job, _ = PTX.create_job(identity, (Int,)) + job, _ = PTXCompiler.PTX.create_job(identity, (Int,)) GPUCompiler.cache_owner(job) end ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing @@ -49,6 +52,13 @@ precompile_test_harness("Inference caching") do load_path @test check_presence(kernel_mi, token) # check that identity survived - @test check_presence(identity_mi, token) broken=(v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.13.0-" <= VERSION < v"1.13.0-beta3"|| v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843") + # NOTE: on 1.13, external CIs from the workload survive only flakily (the + # 1.13.0-beta3 backport did not fully fix this), so skip the check there + ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || + v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" + ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" + if !ext_cis_flaky + @test check_presence(identity_mi, token) broken=ext_cis_lost + end end end From 3f0a3a01b61f8a07d5c760d1ae204fcb75e56a8e Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 06:45:13 +0200 Subject: [PATCH 15/33] Fix version. --- Project.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 1140aa80..d4ad4bda 100644 --- a/Project.toml +++ b/Project.toml @@ -25,13 +25,10 @@ AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444" NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" -[sources] -CompilerCaching = {path = "../CompilerCaching"} - [compat] AMDGPU_LLVM_Backend_jll = "22" -CompilerCaching = "0.4" +CompilerCaching = "0.3" ExprTools = "0.1" InteractiveUtils = "1" LLVM = "9.9" From 77468a64ef60145286712738054cf5316d3dd449 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 08:42:28 +0200 Subject: [PATCH 16/33] Specialize cache_owner and cached_results for the launch hot path. Dynamic construction of the cache token made a cached lookup ~3x slower than the legacy cached_compilation path (684 vs 234 ns); specialized, it is now faster (166 vs 182 ns). Instantiations are bounded: one per back-end (and results type). Co-Authored-By: Claude Fable 5 --- src/deprecated.jl | 3 ++- src/interface.jl | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index 032f0608..246000cb 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -239,7 +239,8 @@ end const job_results = Dict{Any,Any}() const job_results_lock = ReentrantLock() -function cached_results(::Type{V}, @nospecialize(job::CompilerJob)) where {V} +# specialized for the launch hot path, mirroring the 1.11+ implementation +function cached_results(::Type{V}, job::CompilerJob) where {V} # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive # boxing allocation; the MI is kept alive by its method specializations. key = (V, objectid(job.source), job.world, job.config) diff --git a/src/interface.jl b/src/interface.jl index e3778bd8..1102684c 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -329,7 +329,10 @@ struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParam method_table::Core.MethodTable end -cache_owner(@nospecialize(job::CompilerJob)) = +# NOTE: deliberately specialized (one instantiation per back-end): this runs on the +# kernel launch hot path, and constructing the token dynamically is an order of +# magnitude slower. +cache_owner(job::CompilerJob) = GPUCompilerCacheToken(job.config.target, job.config.params, job.config.always_inline, method_table(job)) @@ -399,7 +402,9 @@ end const cached_results_lock = ReentrantLock() -function cached_results(::Type{V}, @nospecialize(job::CompilerJob)) where {V} +# NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of +# instantiations: one per back-end and results type). +function cached_results(::Type{V}, job::CompilerJob) where {V} owner = cache_owner(job) cache = CompilerCaching.CacheView{JobResults}(owner, job.world) From a5741af3c6e919616adec04caaaded6914e59a66 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 08:44:10 +0200 Subject: [PATCH 17/33] Store SPIR-V extensions as a feature string. A Vector field made the target mutable under jl_egal, so owner tokens and configs deserialized from package images never matched and cached kernels were silently recompiled. Use a single --spirv-ext specifier string instead, mirroring LLVM feature strings (and GCN's features). Co-Authored-By: Claude Fable 5 --- src/spirv.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/spirv.jl b/src/spirv.jl index 8388fc1a..f0af2f20 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -24,7 +24,13 @@ export SPIRVCompilerTarget Base.@kwdef struct SPIRVCompilerTarget <: AbstractCompilerTarget version::Union{Nothing,VersionNumber} = nothing - extensions::Vector{String} = [] + # SPIR-V extensions, as the comma-separated specifier string passed verbatim to the + # translator/back-end via `--spirv-ext`, e.g. "+SPV_EXT_shader_atomic_float_add,+SPV_KHR_expect_assume" + # (LLVM feature-string style, cf. `GCNCompilerTarget.features`). Kept as a plain + # `String` -- not a `Vector` -- because target fields must be immutable for the + # `jl_egal`-based cache lookups (`cache_owner`, `JobResults`) to match across + # sessions; see the §2.2 rule in CACHING-REDESIGN.md. + extensions::String = "" supports_fp16::Bool = true supports_fp64::Bool = true supports_bfloat16::Bool = false @@ -149,8 +155,7 @@ end cmd = `$(SPIRV_LLVM_Backend_jll.llc()) $input -filetype=obj -o $translated` if !isempty(job.config.target.extensions) - str = join(map(ext->"+$ext", job.config.target.extensions), ",") - cmd = `$(cmd) -spirv-ext=$str` + cmd = `$(cmd) -spirv-ext=$(job.config.target.extensions)` end elseif job.config.target.backend === :khronos translator = if isavailable(SPIRV_LLVM_Translator_jll) @@ -163,8 +168,7 @@ end cmd = `$translator -o $translated $input --spirv-debug-info-version=ocl-100` if !isempty(job.config.target.extensions) - str = join(map(ext->"+$ext", job.config.target.extensions), ",") - cmd = `$(cmd) --spirv-ext=$str` + cmd = `$(cmd) --spirv-ext=$(job.config.target.extensions)` end if job.config.target.version !== nothing From e84e9b33d6bb8934eca4ee2752f73a2041ee25da Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 08:49:53 +0200 Subject: [PATCH 18/33] Wipe session-dependent results before image serialization. Artifacts derived from IR with relocated GVs embed absolute pointers from the precompilation process. Mark such jobs during output generation and drop their JobResults entries from an atexit hook, which runs before jl_write_compiler_output: within-session lookups still hit, but later sessions recompile instead of loading dangling pointers. Co-Authored-By: Claude Fable 5 --- src/deprecated.jl | 8 ++++++++ src/driver.jl | 4 ++++ src/interface.jl | 43 +++++++++++++++++++++++++++++++++++++++++++ test/native.jl | 14 ++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/src/deprecated.jl b/src/deprecated.jl index 246000cb..5b5e0627 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -252,6 +252,14 @@ function cached_results(::Type{V}, job::CompilerJob) where {V} end +## 1.10 session-dependent results +# +# Nothing to wipe: `job_results` never persists meaningfully across sessions (its +# `objectid`-based keys are session-specific, and our own image's entries are cleared +# in `__init__`; entries written by a downstream package's workload don't make it into +# that package's image at all, cf. cross-image mutation loss). +mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing + end # !HAS_INTEGRATED_CACHE diff --git a/src/driver.jl b/src/driver.jl index 94dd4778..8c52e884 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -336,6 +336,10 @@ const __llvm_initialized = Ref(false) relocate_gvs!(ir, gv_to_value) + # the IR (and any artifact derived from it) now embeds session-absolute + # addresses; keep results for this job out of a package image + isempty(gv_to_value) || mark_session_dependent!(job) + if job.config.optimize @tracepoint "optimization" begin optimize!(job, ir; job.config.opt_level) diff --git a/src/interface.jl b/src/interface.jl index 1102684c..c563a74d 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -427,6 +427,49 @@ function cached_results(::Type{V}, job::CompilerJob) where {V} end end +## session-dependent results +# +# Some compilation results embed session-specific data: `relocate_gvs!` bakes absolute +# pointers into the IR of toplevel jobs that reference `julia.constgv` globals, and any +# artifact a back-end derives from that IR (metallib, SPIR-V, ...) inherits them. Such +# results must not survive into a package image, while remaining available for +# within-session lookups during the precompilation process itself. Julia wipes its own +# session-dependent CodeInstance state during serialization (staticdata.c); we +# approximate that with an `atexit` hook, which the runtime invokes *before* +# `jl_write_compiler_output`: right before the image is written, the entries of jobs +# marked session-dependent are deleted from their `JobResults` container, so a later +# session simply recompiles them. + +const session_dependent_jobs = Vector{CompilerJob}() +const session_dependent_lock = ReentrantLock() + +function mark_session_dependent!(@nospecialize(job::CompilerJob)) + ccall(:jl_generating_output, Cint, ()) == 1 || return + Base.@lock session_dependent_lock begin + if isempty(session_dependent_jobs) + atexit(wipe_session_dependent_results) + end + push!(session_dependent_jobs, job) + end + return +end + +function wipe_session_dependent_results() + Base.@lock session_dependent_lock begin + for job in session_dependent_jobs + cache = CompilerCaching.CacheView{JobResults}(cache_owner(job), job.world) + ci = get(cache, job.source, nothing) + ci === nothing && continue + jr = CompilerCaching.results(JobResults, ci) + Base.@lock cached_results_lock begin + filter!(entry -> entry[1] !== job.config, jr.entries) + end + end + empty!(session_dependent_jobs) + end + return +end + end # HAS_INTEGRATED_CACHE @public GPUCompilerCacheToken, cache_owner, cached_results diff --git a/test/native.jl b/test/native.jl index c6f8a26c..4da30e16 100644 --- a/test/native.jl +++ b/test/native.jl @@ -166,6 +166,20 @@ end new_res = GPUCompiler.cached_results(mod.Results, new_job) @test new_res !== res @test new_res.asm === nothing + + # session-dependent results (e.g. artifacts with relocated GVs) are wiped + # before image serialization; emulate the atexit-driven wipe directly + new_res.asm = "session-dependent" + other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") + other_res = GPUCompiler.cached_results(mod.Results, other_job) + push!(GPUCompiler.session_dependent_jobs, new_job) + GPUCompiler.wipe_session_dependent_results() + @test isempty(GPUCompiler.session_dependent_jobs) + wiped_res = GPUCompiler.cached_results(mod.Results, new_job) + @test wiped_res !== new_res + @test wiped_res.asm === nothing + # ... without affecting other configs on the same CI + @test GPUCompiler.cached_results(mod.Results, other_job) === other_res end @testset "allowed mutable types" begin From 803592ee47fd8753f79b85f1c9cdd719c8ca8c7c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 08:51:32 +0200 Subject: [PATCH 19/33] Clear the 1.10 CodeCache registry on load. CIs deposited by our own precompile workload carry world ages from the precompilation process and are dead weight in later sessions. Co-Authored-By: Claude Fable 5 --- src/GPUCompiler.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index 57c8d7e3..49017261 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -88,6 +88,9 @@ function __init__() # session-local results keyed by objectid+world; entries serialized during # GPUCompiler's own precompilation can never be valid in a later session empty!(job_results) + # ditto for the in-process CodeCaches: CIs deposited by our own precompile + # workload carry world ages from the precompilation process + empty!(GLOBAL_CI_CACHES) end @static if ENABLE_TRACY From 4d02d5ad545512f2d61c68944d25e34f446413c5 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 08:51:32 +0200 Subject: [PATCH 20/33] Don't key runtime library jobs on the kernel's entry-point name. The derived runtime config inherited cosmetic fields like name=, so runtime function artifacts were cached (and persisted) once per kernel config variation instead of once per cache owner. Co-Authored-By: Claude Fable 5 --- src/rtlib.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rtlib.jl b/src/rtlib.jl index 016053fa..839a0efe 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -118,7 +118,11 @@ function build_runtime(@nospecialize(job::CompilerJob)) # the compiler job passed into here identifies the job that requires the runtime. # derive a job that represents the runtime itself (notably with kernel=false). - config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, strip=false) + # fields that identify the *kernel* job, like its entry-point name, are reset so + # that runtime artifacts are keyed (and persisted) identically for all kernels + # sharing a cache owner, instead of once per cosmetic config variation. + config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, + strip=false, name=nothing) for method in values(Runtime.methods) def = if isa(method.def, Symbol) From 2356a23ed603d35a9502e23e88ce83edf459617a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 11 Jun 2026 08:51:32 +0200 Subject: [PATCH 21/33] Fix stale comment about the removed 1.10 methodinstance generator. Co-Authored-By: Claude Fable 5 --- src/jlgen.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jlgen.jl b/src/jlgen.jl index b46ff997..85900e87 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -87,8 +87,8 @@ end return mi end -# On 1.10 we use a generated function that performs the lookup at world+specialization -# resolution time. See deprecated.jl for `methodinstance_generator`. +# On 1.10 we always use the generic lookup. (A @generated fast path used to exist, +# but it was unreachable: the (ft::Type, tt::Type) method above it always won dispatch.) else const methodinstance = generic_methodinstance end From 209e3b10b492cf465ff3a418b6b306ec285f8381 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Tue, 16 Jun 2026 20:26:15 +0200 Subject: [PATCH 22/33] Remove irrelevant TODOs. --- TODO | 1 - TODO.md | 14 -------------- 2 files changed, 15 deletions(-) delete mode 100644 TODO delete mode 100644 TODO.md diff --git a/TODO b/TODO deleted file mode 100644 index 7db75e11..00000000 --- a/TODO +++ /dev/null @@ -1 +0,0 @@ -value materializer: just put f/new_f in value map! diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 1ba2897a..00000000 --- a/TODO.md +++ /dev/null @@ -1,14 +0,0 @@ -TODO: - -The _methods_by_ftype call also (optionally) returns age bounds, which you are required to propagate (usually pretty easy, since it is just min/max calls - -_methods_by_ftype also takes mt argument; for overlays of kernels? -also `which` in driver - - -# put the function and argument types in typevars -# so that we can access it from generated functions - -remove that. we just use .f and .tt in the world_age generator -and rename f to ft field -maybe pass sig everywhere? From 19fb90d46c9d2e55242a431004a7c6c1b4823295 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 17 Jun 2026 14:20:50 +0200 Subject: [PATCH 23/33] Fixes. --- .buildkite/pipeline.yml | 2 +- .github/workflows/Test.yml | 15 ++++++++++ src/jlgen.jl | 29 ++++++++++++++----- test/Project.toml | 5 ---- test/native.jl | 57 +++++++++++++++++++++++--------------- test/native/precompile.jl | 15 ++++++---- test/ptx/precompile.jl | 23 +++++++++------ test/spirv.jl | 8 ++++-- 8 files changed, 102 insertions(+), 52 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 39e8c997..8a990ec2 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -95,7 +95,7 @@ steps: - label: "OpenCL.jl" plugins: - JuliaCI/julia#v1: - version: "1.11" + version: "1.10" - JuliaCI/julia-coverage#v1: codecov: true command: | diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index fe550da4..5bd4869e 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -25,6 +25,21 @@ jobs: include: # starting with Julia 1.10, we can enable opaque pointers # from Julia 1.12 on, this is the default. + - version: '1.10' + os: 'ubuntu-24.04' + llvm_args: '--opaque-pointers' + - version: '1.10' + os: 'ubuntu-24.04-arm' + llvm_args: '--opaque-pointers' + - version: '1.10' + os: 'macOS-15' + llvm_args: '--opaque-pointers' + - version: '1.10' + os: 'macOS-15-intel' + llvm_args: '--opaque-pointers' + - version: '1.10' + os: 'windows-2025' + llvm_args: '--opaque-pointers' - version: '1.11' os: 'ubuntu-24.04' llvm_args: '--opaque-pointers' diff --git a/src/jlgen.jl b/src/jlgen.jl index 85900e87..1eb26039 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -630,9 +630,9 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) ci′.min_world <= job.world <= ci′.max_world end - # When `jl_get_llvm_cis` returns CIs the cache may contain both an - # interpreter-token-owned CI (ours) and a `nothing`-owner native CI for the same MI; - # prefer the foreign one. + # `jl_get_llvm_cis` may return both our owner-token CI and a native + # owner-less CI for the same MI. Prefer ours when present, but keep + # owner-less CIs that are the only compiled entry for their method. owned_mis = Set{MethodInstance}() for ci′ in code_instances if ci′.owner !== nothing @@ -667,14 +667,16 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # process all compiled method instances compiled = Dict() - for (ci′, mi) in zip(code_instances, method_instances) - # get the function index + function compiled_entry(ci′::CodeInstance; required::Bool=true) llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall(:jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), native_code, ci′, llvm_func_idx, llvm_specfunc_idx) - @assert llvm_func_idx[] != -1 || llvm_specfunc_idx[] != -1 "Static compilation failed" + if llvm_func_idx[] == -1 && llvm_specfunc_idx[] == -1 + required && error("Static compilation failed") + return nothing + end # get the function llvm_func = if llvm_func_idx[] >= 1 @@ -695,11 +697,24 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) nothing end + (; ci=ci′, func=llvm_func, specfunc=llvm_specfunc) + end + + for (ci′, mi) in zip(code_instances, method_instances) @assert !haskey(compiled, mi) # NOTE: it's not safe to store raw LLVM functions here, since those may get # removed or renamed during optimization, so we store their name instead. - compiled[mi] = (; ci=ci′, func=llvm_func, specfunc=llvm_specfunc) + compiled[mi] = compiled_entry(ci′) + end + + if !haskey(compiled, job.source) && root_ci !== nothing + # On Julia 1.14-dev, `jl_get_llvm_cis` may omit the root CI. That root + # can also be less-specific than `job.source` when Julia deliberately + # reuses unspecialized code, so recover the entry that codegen actually + # emitted and expose it under the requested source. + entry = compiled_entry(root_ci; required=false) + entry !== nothing && (compiled[job.source] = entry) end # ensure that the requested method instance was compiled diff --git a/test/Project.toml b/test/Project.toml index 103e21ce..c6fbed19 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -19,11 +19,6 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" demumble_jll = "1e29f10c-031c-5a83-9565-69cddfc27673" -[sources] -CompilerCaching = {path = "../../CompilerCaching"} -GPUCompiler = {path = ".."} -LLVM = {path = "/Users/tim/Julia/pkg/LLVM"} - [compat] Aqua = "0.8" ParallelTestRunner = "2" diff --git a/test/native.jl b/test/native.jl index 4da30e16..871c83fb 100644 --- a/test/native.jl +++ b/test/native.jl @@ -1,5 +1,8 @@ @testset "reflection" begin - job, _ = Native.create_job(identity, (Int,)) + mod = @eval module $(gensym()) + f(x::Int) = x + end + job, _ = Native.create_job(mod.f, (Int,)) @test only(GPUCompiler.code_lowered(job)) isa Core.CodeInfo @@ -7,17 +10,17 @@ @test rt === Int @test @filecheck begin - @check "MethodInstance for identity" + @check "MethodInstance for {{.*}}f" GPUCompiler.code_warntype(job) end @test @filecheck begin - @check "@{{(julia|j)_identity_[0-9]+}}" + @check "@{{(julia|j)_f_[0-9]+}}" GPUCompiler.code_llvm(job) end @test @filecheck begin - @check "@{{(julia|j)_identity_[0-9]+}}" + @check "@{{(julia|j)_f_[0-9]+}}" GPUCompiler.code_native(job) end end @@ -167,31 +170,35 @@ end @test new_res !== res @test new_res.asm === nothing - # session-dependent results (e.g. artifacts with relocated GVs) are wiped - # before image serialization; emulate the atexit-driven wipe directly - new_res.asm = "session-dependent" - other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") - other_res = GPUCompiler.cached_results(mod.Results, other_job) - push!(GPUCompiler.session_dependent_jobs, new_job) - GPUCompiler.wipe_session_dependent_results() - @test isempty(GPUCompiler.session_dependent_jobs) - wiped_res = GPUCompiler.cached_results(mod.Results, new_job) - @test wiped_res !== new_res - @test wiped_res.asm === nothing - # ... without affecting other configs on the same CI - @test GPUCompiler.cached_results(mod.Results, other_job) === other_res + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # session-dependent results (e.g. artifacts with relocated GVs) are wiped + # before image serialization; emulate the atexit-driven wipe directly + new_res.asm = "session-dependent" + other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") + other_res = GPUCompiler.cached_results(mod.Results, other_job) + push!(GPUCompiler.session_dependent_jobs, new_job) + GPUCompiler.wipe_session_dependent_results() + @test isempty(GPUCompiler.session_dependent_jobs) + wiped_res = GPUCompiler.cached_results(mod.Results, new_job) + @test wiped_res !== new_res + @test wiped_res.asm === nothing + # ... without affecting other configs on the same CI + @test GPUCompiler.cached_results(mod.Results, other_job) === other_res + end end @testset "allowed mutable types" begin # when types have no fields, we should always allow them mod = @eval module $(gensym()) struct Empty end + accept_empty(::Empty) = nothing + accept_symbol(::Symbol) = nothing end - Native.code_execution(Returns(nothing), (mod.Empty,)) + Native.code_execution(mod.accept_empty, (mod.Empty,)) # this also applies to Symbols - Native.code_execution(Returns(nothing), (Symbol,)) + Native.code_execution(mod.accept_symbol, (Symbol,)) end @testset "code coverage" begin @@ -389,19 +396,23 @@ end end @testset "function entry safepoint emission" begin + mod = @eval module $(gensym()) + f(::Nothing) = nothing + end + @test @filecheck begin - @check_label "define void @{{(julia|j)_identity_[0-9]+}}" + @check_label "define void @{{(julia|j)_f_[0-9]+}}" @check_not "%safepoint" - Native.code_llvm(identity, Tuple{Nothing}; entry_safepoint=false, optimize=false, dump_module=true) + Native.code_llvm(mod.f, Tuple{Nothing}; entry_safepoint=false, optimize=false, dump_module=true) end # XXX: broken by JuliaLang/julia#57010, # see https://github.com/JuliaLang/julia/pull/57010/files#r2079576894 if VERSION < v"1.13.0-DEV.533" @test @filecheck begin - @check_label "define void @{{(julia|j)_identity_[0-9]+}}" + @check_label "define void @{{(julia|j)_f_[0-9]+}}" @check "%safepoint" - Native.code_llvm(identity, Tuple{Nothing}; entry_safepoint=true, optimize=false, dump_module=true) + Native.code_llvm(mod.f, Tuple{Nothing}; entry_safepoint=true, optimize=false, dump_module=true) end end end diff --git a/test/native/precompile.jl b/test/native/precompile.jl index c1fb91ae..90c46be2 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -66,13 +66,17 @@ precompile_test_harness("Inference caching") do load_path import NativeCompiler # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + identity_mis = Any[ + GPUCompiler.methodinstance(typeof(identity), Tuple{Int}), + GPUCompiler.CompilerCaching.method_instance(identity, (Int,)), + ] + unique!(identity_mis) token = let job, _ = NativeCompiler.Native.create_job(identity, (Int,)) GPUCompiler.cache_owner(job) end - @test !check_presence(identity_mi, token) + @test all(!check_presence(mi, token) for mi in identity_mis) using NativeBackend @@ -87,13 +91,14 @@ precompile_test_harness("Inference caching") do load_path @test check_presence(square_mi, token) # check that identity survived - # NOTE: on 1.13, external CIs from the workload survive only flakily (the - # 1.13.0-beta3 backport did not fully fix this), so skip the check there + # NOTE: external CIs from the workload survive only flakily on 1.13 + # (the 1.13.0-beta3 backport did not fully fix this), so skip the + # check there. ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" if !ext_cis_flaky - @test check_presence(identity_mi, token) broken=ext_cis_lost + @test any(mi -> check_presence(mi, token), identity_mis) broken=ext_cis_lost end # Recompiling a foreign method after loading precompiled owner-token CIs diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl index 0e1a5b9b..84c10abd 100644 --- a/test/ptx/precompile.jl +++ b/test/ptx/precompile.jl @@ -30,7 +30,11 @@ precompile_test_harness("Inference caching") do load_path import PTXCompiler # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + identity_mis = Any[ + GPUCompiler.methodinstance(typeof(identity), Tuple{Int}), + GPUCompiler.CompilerCaching.method_instance(identity, (Int,)), + ] + unique!(identity_mis) # NOTE: use the standalone package's module to construct the token — the # sandbox's own PTX helper module defines a distinct CompilerParams @@ -39,10 +43,12 @@ precompile_test_harness("Inference caching") do load_path job, _ = PTXCompiler.PTX.create_job(identity, (Int,)) GPUCompiler.cache_owner(job) end - ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing - while ci !== nothing - @test ci.owner !== token - ci = isdefined(ci, :next) ? ci.next : nothing + for identity_mi in identity_mis + ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing + while ci !== nothing + @test ci.owner !== token + ci = isdefined(ci, :next) ? ci.next : nothing + end end using PTXBackend @@ -52,13 +58,14 @@ precompile_test_harness("Inference caching") do load_path @test check_presence(kernel_mi, token) # check that identity survived - # NOTE: on 1.13, external CIs from the workload survive only flakily (the - # 1.13.0-beta3 backport did not fully fix this), so skip the check there + # NOTE: external CIs from the workload survive only flakily on 1.13 + # (the 1.13.0-beta3 backport did not fully fix this), so skip the + # check there. ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" if !ext_cis_flaky - @test check_presence(identity_mi, token) broken=ext_cis_lost + @test any(mi -> check_presence(mi, token), identity_mis) broken=ext_cis_lost end end end diff --git a/test/spirv.jl b/test/spirv.jl index b5006194..a100f4f1 100644 --- a/test/spirv.jl +++ b/test/spirv.jl @@ -182,7 +182,7 @@ end """, NTuple{N, Core.VecElement{Float32}}, NTuple{2, NTuple{N, Core.VecElement{Float32}}}, x.data, y.data)) end end - kernel(x...) = @noinline fadd(x...) + kernel(x, y) = @noinline fadd(x, y) end @test @filecheck begin @@ -191,7 +191,8 @@ end @check_label "define void @{{(julia|j)_kernel_[0-9]+}}" @check cond=(v"1.12" <= VERSION < v"1.12.5") "alloca <2 x i64>, align 16" @check cond=(VERSION >= v"1.12.5") "alloca [2 x i64], align 16" - SPIRV.code_llvm(mod.kernel, NTuple{2, mod.Vec{4, Float32}}; backend, dump_module=true) + SPIRV.code_llvm(mod.kernel, Tuple{mod.Vec{4, Float32}, mod.Vec{4, Float32}}; + backend, dump_module=true) end @test @filecheck begin @@ -199,7 +200,8 @@ end @check_label "define void @{{(julia|j)_kernel_[0-9]+}}" @check cond=(v"1.12" <= VERSION < v"1.12.5") "alloca [2 x <2 x i64>], align 16" @check cond=(VERSION >= v"1.12.5") "alloca [4 x i64], align 16" - SPIRV.code_llvm(mod.kernel, NTuple{2, mod.Vec{8, Float32}}; backend, dump_module=true) + SPIRV.code_llvm(mod.kernel, Tuple{mod.Vec{8, Float32}, mod.Vec{8, Float32}}; + backend, dump_module=true) end end From 501d2a7d166993ede457bdb94388e98bae12e1d1 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 3 Jul 2026 07:32:43 +0200 Subject: [PATCH 24/33] Remove stale runtime_slug test references --- src/ptx.jl | 5 ++--- test/gcn.jl | 10 ++++++---- test/helpers/metal.jl | 2 -- test/ptx.jl | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/ptx.jl b/src/ptx.jl index c425b6bc..3c22f5c9 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -144,9 +144,8 @@ function finish_module!(@nospecialize(job::CompilerJob{PTXCompilerTarget}), # for `-fcuda-flush-denormals-to-zero` and is the only `__nvvm_reflect` key # LLVM's NVVMReflectPass honors besides `__CUDA_ARCH`. only emit it on the # toplevel module that runs through `optimize!`, as sub-modules (the cached - # runtime, deferred jobs) don't need it, and the cached runtime in - # particular would otherwise conflict on link if it was built with a - # different `fastmath` setting (which isn't part of `runtime_slug`). + # runtime, deferred jobs) don't need it and should not get module-level + # flags that can collide when linked into the toplevel module. if job.config.toplevel flags(mod)["nvvm-reflect-ftz", LLVM.API.LLVMModuleFlagBehaviorOverride] = Metadata(ConstantInt(Int32(job.config.target.fastmath ? 1 : 0))) diff --git a/test/gcn.jl b/test/gcn.jl index e91c7579..4d035351 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -20,12 +20,14 @@ sink_gcn(i) = sink(i, Val(5)) kernel() = return end - # the backend participates in the runtime slug, so different back-ends don't share a cache + # the backend participates in the cache owner, so different back-ends don't share a cache job_ext, _ = GCN.create_job(mod.kernel, Tuple{}; backend=:external) job_inp, _ = GCN.create_job(mod.kernel, Tuple{}; backend=:inprocess) - @test endswith(GPUCompiler.runtime_slug(job_ext), "-external") - @test endswith(GPUCompiler.runtime_slug(job_inp), "-inprocess") - @test GPUCompiler.runtime_slug(job_ext) != GPUCompiler.runtime_slug(job_inp) + owner_ext = GPUCompiler.cache_owner(job_ext) + owner_inp = GPUCompiler.cache_owner(job_inp) + @test owner_ext.target.backend === :external + @test owner_inp.target.backend === :inprocess + @test owner_ext != owner_inp # the explicit :external backend generates machine code through the external llc @test (GCN.code_native(devnull, mod.kernel, Tuple{}; backend=:external); true) diff --git a/test/helpers/metal.jl b/test/helpers/metal.jl index 0000a022..8bea2a25 100644 --- a/test/helpers/metal.jl +++ b/test/helpers/metal.jl @@ -25,8 +25,6 @@ end ThreadedRuntimeCompilerJob = CompilerJob{MetalCompilerTarget,ThreadedRuntimeCompilerParams} GPUCompiler.runtime_module(::ThreadedRuntimeCompilerJob) = ThreadedRuntime -GPUCompiler.runtime_slug(job::ThreadedRuntimeCompilerJob) = - "metal-threadedruntime-macos$(job.config.target.macos)-debuginfo=$(Int(GPUCompiler.llvm_debug_info(job)))" function create_job(@nospecialize(func), @nospecialize(types); kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) diff --git a/test/ptx.jl b/test/ptx.jl index f76717c4..7425f759 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -554,8 +554,8 @@ end @test GPUCompiler.cpu_name(PTXCompilerTarget(cap=v"10.0", feature_set=:family)) == "sm_100f" @test_throws ErrorException GPUCompiler.cpu_name(PTXCompilerTarget(cap=v"9.0", feature_set=:bogus)) - # hash must discriminate, otherwise two configs differing only on feature_set - # would share the same on-disk runtime slug and collide in the compiler cache. + # hash must discriminate, otherwise two targets differing only on feature_set + # could compare equal inside cache-owner keys. @test hash(PTXCompilerTarget(cap=v"9.0", feature_set=:baseline)) != hash(PTXCompilerTarget(cap=v"9.0", feature_set=:arch)) From a77d49d1d1cc39281fe27f9ff26167b73e25c18a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 3 Jul 2026 07:36:24 +0200 Subject: [PATCH 25/33] Remove legacy link_libraries dispatch --- src/optim.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/optim.jl b/src/optim.jl index 8c6d1500..b0b339a3 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -503,13 +503,7 @@ function (self::LinkLibraries)(mod::LLVM.Module) # references after the normal library-linking phase has run. Give back-ends a # second chance to resolve those references before they lower their own runtime # intrinsics. This is a no-op for back-ends without a link_libraries! override. - if has_legacy_link_libraries(self.job) - undefined_fns = [LLVM.name(f) for f in functions(mod) - if isdeclaration(f) && !LLVM.isintrinsic(f)] - link_libraries!(self.job, mod, undefined_fns) - else - link_libraries!(self.job, mod) - end + link_libraries!(self.job, mod) return true end GPULinkLibrariesPass(job) = NewPMModulePass("GPULinkLibraries", LinkLibraries(job)) From 76d6459cdd944ae120c36d73aa98664d05eaf7ef Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 6 Jul 2026 15:07:56 +0200 Subject: [PATCH 26/33] Key the runtime library cache on the derived runtime config. The debug level (among other codegen-relevant settings) is baked into the runtime IR as a constant, so keying the session memo on cache_owner alone shared one assembled library across debug levels. Back-ends no longer need to widen their owner token to compensate. Co-Authored-By: Claude Fable 5 --- src/rtlib.jl | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/rtlib.jl b/src/rtlib.jl index 839a0efe..bd05bf74 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -113,16 +113,19 @@ function emit_function!(mod, config::CompilerConfig, f, method) link!(mod, new_mod) end -function build_runtime(@nospecialize(job::CompilerJob)) - mod = LLVM.Module("GPUCompiler run-time library") +# the compiler job passed into here identifies the job that requires the runtime. +# derive a config that represents the runtime itself (notably with kernel=false). +# fields that identify the *kernel* job, like its entry-point name, are reset so +# that runtime artifacts are keyed (and persisted) identically for all kernels +# sharing the remaining codegen-relevant settings, instead of once per cosmetic +# config variation. +function runtime_config(@nospecialize(job::CompilerJob)) + CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, + strip=false, name=nothing) +end - # the compiler job passed into here identifies the job that requires the runtime. - # derive a job that represents the runtime itself (notably with kernel=false). - # fields that identify the *kernel* job, like its entry-point name, are reset so - # that runtime artifacts are keyed (and persisted) identically for all kernels - # sharing a cache owner, instead of once per cosmetic config variation. - config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, - strip=false, name=nothing) +function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) + mod = LLVM.Module("GPUCompiler run-time library") for method in values(Runtime.methods) def = if isa(method.def, Symbol) @@ -143,16 +146,25 @@ function build_runtime(@nospecialize(job::CompilerJob)) end # Session-local cache of assembled runtime libraries, keyed by -# `(cache_owner(job), opaque_pointers)`. Cross-session persistence is the back-end's -# concern: rebuild on first use of each session, then reuse within the session. -const runtime_libs = Dict{Tuple{Any, Bool}, Vector{UInt8}}() +# `(runtime_config(job), opaque_pointers)`: the derived runtime config covers every +# codegen-relevant setting (e.g. the debug level, which is baked into the runtime IR +# as a constant), while cosmetic kernel-job fields are normalized away. Cross-session +# persistence happens at the per-function level (see `RuntimeFunctionResults`): +# reassemble on first use of each session, then reuse within the session. +# +# NOTE: unlike the per-function bitcode, whose CodeInstances Julia invalidates on +# method redefinition, the assembled library is world-blind: redefining a runtime +# function mid-session leaves a stale assembly behind. That only affects interactive +# development of a back-end's runtime module (restart, or `empty!` this cache). +const runtime_libs = Dict{Tuple{CompilerConfig, Bool}, Vector{UInt8}}() const runtime_libs_lock = ReentrantLock() @locked function load_runtime(@nospecialize(job::CompilerJob)) - key = (cache_owner(job), !supports_typed_pointers(context())) + config = runtime_config(job) + key = (config, !supports_typed_pointers(context())) bytes = Base.@lock runtime_libs_lock get!(runtime_libs, key) do - lib = build_runtime(job) + lib = build_runtime(job, config) io = IOBuffer() write(io, lib) take!(io) From cded0966f32be6b94012375187dd889d15743f6f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 6 Jul 2026 15:08:10 +0200 Subject: [PATCH 27/33] Remove reset_runtime. A vestige of the on-disk runtime cache: per-function bitcode now lives on CodeInstances that Julia invalidates automatically, and the session memo cannot contain stale entries at back-end load time. Also fixes the PTX runtime-rebuild test, which still poked the removed disk-cache global. Co-Authored-By: Claude Fable 5 --- src/rtlib.jl | 3 --- test/ptx.jl | 18 +++++++----------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/rtlib.jl b/src/rtlib.jl index bd05bf74..d0757c43 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -172,6 +172,3 @@ const runtime_libs_lock = ReentrantLock() return parse(LLVM.Module, MemoryBuffer(bytes); lazy=true) end - -# clear the session-local runtime library cache -reset_runtime() = Base.@lock runtime_libs_lock empty!(runtime_libs) diff --git a/test/ptx.jl b/test/ptx.jl index 7425f759..4f6d2209 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -10,23 +10,19 @@ end end -@testset "kernel state survives a runtime rebuild during optimization" begin - # Clearing the cache forces the relink pass inside `optimize!` to rebuild the runtime - # (nested compilation); the kernel must still get its state argument afterwards. +@testset "kernel state survives a runtime rebuild" begin + # Clearing the runtime cache forces the library link inside `emit_llvm` to rebuild + # the runtime (nested compilation); the kernel must still get its state argument + # afterwards. mod = @eval module $(gensym()) function kernel(x) x < 1 && throw(DivideError()) return end end - old_cache = GPUCompiler.compile_cache - ir = try - GPUCompiler.compile_cache = nothing - sprint() do io - PTX.code_llvm(io, mod.kernel, Tuple{Int}; kernel=true, dump_module=true) - end - finally - GPUCompiler.compile_cache = old_cache + Base.@lock GPUCompiler.runtime_libs_lock empty!(GPUCompiler.runtime_libs) + ir = sprint() do io + PTX.code_llvm(io, mod.kernel, Tuple{Int}; kernel=true, dump_module=true) end @test occursin("gpu_report_exception", ir) @test occursin("[1 x i64] %state", ir) From 688de6a243d354a351ecf10522844b402c60b7ad Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Mon, 6 Jul 2026 15:08:24 +0200 Subject: [PATCH 28/33] Test persistence of edge-less CodeInstances. Kernels that make no calls produce CodeInstances without inference edges; on Julia 1.12 those are serialized with the revalidation sentinel but excluded from the edge-verification list, so they never revalidate after loading. Upstream bug, fixed by the 1.13 serialization rework. Co-Authored-By: Claude Fable 5 --- test/native/precompile.jl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test/native/precompile.jl b/test/native/precompile.jl index 90c46be2..b48ed86e 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -14,6 +14,9 @@ precompile_test_harness("Inference caching") do load_path return end + # a kernel that makes no calls, so its CodeInstance has no inference edges + leaf_kernel(A) = nothing + function kernel_w_global(A, x, sym) if sym == :A A[1] = x @@ -34,6 +37,11 @@ precompile_test_harness("Inference caching") do load_path precompile(job) end + let + job, _ = NativeCompiler.Native.create_job(leaf_kernel, (Vector{Int},)) + precompile(job) + end + let job, _ = NativeCompiler.Native.create_job(kernel_w_global, (Vector{Int}, Int, Symbol)) precompile(job) @@ -87,6 +95,15 @@ precompile_test_harness("Inference caching") do load_path kernel_w_global_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel_w_global), Tuple{Vector{Int}, Int, Symbol}) @test check_presence(kernel_w_global_mi, token) + # a CodeInstance without inference edges (a kernel making no calls) is + # serialized with the revalidation sentinel but excluded from the edge + # verification list on Julia 1.12 (jl_record_edges skips empty-edge CIs), + # so it deserializes permanently invalid. Fixed by the serialization + # rework in 1.13; 1.11 uses the old scheme and is unaffected. + leaf_edges_lost = v"1.12-" <= VERSION < v"1.13-" + leaf_kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.leaf_kernel), Tuple{Vector{Int}}) + @test check_presence(leaf_kernel_mi, token) broken=leaf_edges_lost + square_mi = GPUCompiler.methodinstance(typeof(NativeBackend.square), Tuple{Float64}) @test check_presence(square_mi, token) From 9dba1028386655f52fcfe38e01c81f5895c9c6eb Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jul 2026 10:24:56 +0200 Subject: [PATCH 29/33] Review by Codex. --- LocalPreferences.toml | 4 - src/deprecated.jl | 15 ++-- src/interface.jl | 162 ++++++++++++++++++++++++++++---------- src/jlgen.jl | 14 ++-- src/rtlib.jl | 138 ++++++++++++++++++++++++-------- src/spirv.jl | 5 +- test/helpers/native.jl | 3 + test/native.jl | 47 +++++++++++ test/native/precompile.jl | 29 +++++++ 9 files changed, 324 insertions(+), 93 deletions(-) delete mode 100644 LocalPreferences.toml diff --git a/LocalPreferences.toml b/LocalPreferences.toml deleted file mode 100644 index f5638dd8..00000000 --- a/LocalPreferences.toml +++ /dev/null @@ -1,4 +0,0 @@ -[GPUCompiler] -# wether caching of object files should be enabled. If the disk cache is enabled -# cache files are storied in scratch memory. -#disk_cache = "false" diff --git a/src/deprecated.jl b/src/deprecated.jl index 5b5e0627..b0987db6 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -251,6 +251,10 @@ function cached_results(::Type{V}, job::CompilerJob) where {V} end end +# The 1.10 results store is independent of CodeInstances, so obtaining its empty result does +# not trigger inference. Match the integrated-cache API used by back-end compile-or-lookup paths. +cached_results_if_present(::Type{V}, job::CompilerJob) where {V} = cached_results(V, job) + ## 1.10 session-dependent results # @@ -265,10 +269,9 @@ end # !HAS_INTEGRATED_CACHE ## Legacy `cached_compilation` (1.10+) -# A session-local, MI-keyed kernel cache modeled after the pre-CompilerCaching API. Used -# by back-ends that haven't migrated to the `CacheView`-based flow (and by all back-ends -# on Julia 1.10, where the new flow doesn't apply because there's no integrated cache / -# `analysis_results`). +# A session-local, MI-keyed kernel cache modeled after the pre-CompilerCaching API. Kept +# only for back-ends that have not migrated to `cached_results`; migrated back-ends use +# that API on every Julia version, including its lightweight 1.10 implementation above. """ cached_compilation(cache::AbstractDict, src::MethodInstance, cfg::CompilerConfig, @@ -282,8 +285,8 @@ whatever the `linker` function returns. The `compiler` function should take a `CompilerJob` and return data that the `linker` function then turns into a session-local artifact (e.g. a `CuModule`). -This is the legacy caching API used before GPUCompiler 2.0. New code on Julia 1.11+ -should prefer `CompilerCaching.CacheView`-based caching (see the package extension). +This is the legacy caching API used before GPUCompiler 2.0. New code should use +[`cached_results`](@ref) and keep portable artifacts separate from session-local handles. """ function cached_compilation(cache::AbstractDict{<:Any,V}, src::MethodInstance, cfg::CompilerConfig, diff --git a/src/interface.jl b/src/interface.jl index c563a74d..433bc6f6 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -66,12 +66,48 @@ abstract type AbstractCompilerParams end nest_params(params::AbstractCompilerParams, parent::AbstractCompilerParams) = params +## cache owner + +# Inference results are shared by jobs with the same target, params, inlining policy, and +# method-table identity. Store this token on the CompilerConfig (as `Any`) so the immutable +# value is boxed once when the config is created, instead of on every integrated-cache lookup. +struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams, M} + target::T + params::P + always_inline::Bool + method_table::M +end + +""" + cache_owner(target, params, always_inline) + +Construct the immutable token that partitions inference for a compiler configuration. +Back-ends that override `method_table` or `method_table_view` must override this form as +well and include every method-table identity used by the view. On Julia 1.11+, the +job-level `cache_owner(job)` returns the pre-boxed token stored on its `CompilerConfig`. + +The returned value must match under `===`/`jl_egal` after package-image deserialization; +use immutable containers and canonical method-table objects. +""" +cache_owner(target::AbstractCompilerTarget, params::AbstractCompilerParams, + always_inline::Bool) = + GPUCompilerCacheToken(target, params, always_inline, GLOBAL_METHOD_TABLE) + + ## config export CompilerConfig # the configuration of the compiler +# Keep the 1.10 layout concrete: an `Any` field makes otherwise-inline configs escape when +# used in the legacy results key. Integrated caching needs the pre-boxed owner on 1.11+. +const CompilerConfigCacheOwner = @static if HAS_INTEGRATED_CACHE + Any +else + Nothing +end + const CONFIG_KWARGS = [:kernel, :name, :entry_abi, :always_inline, :opt_level, :libraries, :optimize, :cleanup, :validate, :strip] @@ -128,6 +164,9 @@ struct CompilerConfig{T,P} toplevel::Bool only_entry::Bool + # Boxed once so calls into Julia's Any-typed integrated cache don't allocate on every hit. + cache_owner::CompilerConfigCacheOwner + function CompilerConfig(target::AbstractCompilerTarget, params::AbstractCompilerParams; kernel=true, name=nothing, entry_abi=:specfunc, toplevel=true, always_inline=false, opt_level=2, @@ -137,10 +176,15 @@ struct CompilerConfig{T,P} if entry_abi ∉ (:specfunc, :func) error("Unknown entry_abi=$entry_abi") end + owner = @static if HAS_INTEGRATED_CACHE + cache_owner(target, params, always_inline) + else + nothing + end new{typeof(target), typeof(params)}(target, params, kernel, name, entry_abi, always_inline, opt_level, debug_level, libraries, optimize, cleanup, validate, strip, toplevel, - only_entry) + only_entry, owner) end end @@ -187,7 +231,9 @@ function Base.hash(cfg::CompilerConfig, h::UInt) h = hash(cfg.strip, h) h = hash(cfg.toplevel, h) h = hash(cfg.only_entry, h) - + # `cache_owner` deliberately is not hashed: its target/params/inlining components were + # already included above, and a method-table-only difference may safely collide. Hashing + # the full token here would duplicate the most expensive parts of every config hash. return h end @@ -307,13 +353,12 @@ pass_by_ref(@nospecialize(job::CompilerJob)) = false valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false # Cache partitioning. On Julia 1.11+, the owner is stored on every `CodeInstance` and -# compared via `jl_egal`, so it (and every field) must be immutable for cross-session -# matches (e.g. via package precompilation); custom `target` / `params` types must be -# `struct`s, not `mutable struct`s. +# compared via `jl_egal`; custom `target` / `params` containers must be immutable so +# equivalent values can match across package-image deserialization. # # On Julia 1.10, where there's no per-CI owner field, this token only identifies the -# session-local cache partition (see `cached_compilation` / `GLOBAL_CI_CACHES` in -# `deprecated.jl`); the immutability requirement is still useful for stable hashing. +# session-local `GLOBAL_CI_CACHES` partition in `deprecated.jl`; the immutability +# requirement is still useful for stable hashing. # # Care is required for anything that impacts: # - method_table @@ -322,20 +367,18 @@ valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false # The default covers the full target+params instances (so backends with version- or # arch-specific knobs partition cleanly), `always_inline` (which feeds optimization_params), # and the method table. -struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams} - target::T - params::P - always_inline::Bool - method_table::Core.MethodTable +# The token is created and boxed by the CompilerConfig constructor. Returning the boxed field +# is important: passing a freshly-constructed immutable token to the Any-typed C cache API +# allocates on every kernel-cache hit. +@static if HAS_INTEGRATED_CACHE + cache_owner(job::CompilerJob) = job.config.cache_owner +else + # Preserve the allocation-free 1.10 path: constructing this specialized immutable token + # is cheaper than storing an Any-typed box on every CompilerConfig. + cache_owner(job::CompilerJob) = GPUCompilerCacheToken( + job.config.target, job.config.params, job.config.always_inline, method_table(job)) end -# NOTE: deliberately specialized (one instantiation per back-end): this runs on the -# kernel launch hot path, and constructing the token dynamically is an order of -# magnitude slower. -cache_owner(job::CompilerJob) = - GPUCompilerCacheToken(job.config.target, job.config.params, - job.config.always_inline, method_table(job)) - """ cached_results(::Type{V}, job::CompilerJob) -> V @@ -352,9 +395,11 @@ mutable struct MetalResults MetalResults() = new(nothing, nothing, []) end -res = GPUCompiler.cached_results(MetalResults, job) -if res.metallib === nothing || GPUCompiler.compile_hook[] !== nothing - ...compile, populate res... +res = GPUCompiler.cached_results_if_present(MetalResults, job) +if res === nothing || res.metallib === nothing || GPUCompiler.compile_hook[] !== nothing + artifacts = ...compile... + res === nothing && (res = GPUCompiler.cached_results(MetalResults, job)) + ...populate res from artifacts... end ``` @@ -395,38 +440,69 @@ Entries are matched by `===` (`jl_egal`): `CompilerConfig` is an immutable struc structurally identical configs compare equal, including against configs deserialized from package images. """ +# `CompilerConfig` is an abstract UnionAll here. Keeping it in a tuple stored inline in a +# Vector boxes the config again on every iteration. A non-isbits entry object is stored by +# reference, so both fields are boxed once and hot-path scans allocate nothing. +struct JobResultEntry + config::CompilerConfig + value::Any +end + mutable struct JobResults - entries::Vector{Tuple{CompilerConfig,Any}} - JobResults() = new(Tuple{CompilerConfig,Any}[]) + entries::Vector{JobResultEntry} + JobResults() = new(JobResultEntry[]) end const cached_results_lock = ReentrantLock() # NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of # instantiations: one per back-end and results type). -function cached_results(::Type{V}, job::CompilerJob) where {V} - owner = cache_owner(job) - cache = CompilerCaching.CacheView{JobResults}(owner, job.world) - - # get a CI for the job, running inference to create one if needed - ci = get(cache, job.source, nothing) - if ci === nothing - interp = get_interpreter(job) - ci = CompilerCaching.typeinf!(interp, job.source) - ci === nothing && error("Inference of $(job.source) failed") - end - +function job_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} jr = CompilerCaching.results(JobResults, ci) Base.@lock cached_results_lock begin - for (config, v) in jr.entries - config === job.config && v isa V && return v::V + for entry in jr.entries + entry.config === config && entry.value isa V && return entry.value::V end v = V() - push!(jr.entries, (job.config, v)) + push!(jr.entries, JobResultEntry(config, v)) return v end end +function cache_view(@nospecialize(job::CompilerJob)) + # `cache_owner` is deliberately stored as Any on CompilerConfig: preserve that box in the + # CacheView instead of re-specializing and re-boxing the immutable token for the ccall. + CompilerCaching.CacheView{Any,JobResults}(cache_owner(job), job.world) +end + +""" + cached_results_if_present(::Type{V}, job::CompilerJob) -> Union{Nothing,V} + +Return the results struct for `job` when its CodeInstance already exists, creating only the +per-config `V` entry. Return `nothing` without running inference when no CI exists. + +Back-end compile-or-lookup paths should use this first: on a miss, run codegen (which drives +inference itself), then call [`cached_results`](@ref) to attach the artifacts. This avoids a +standalone inference walk immediately followed by the compiler's own walk. +""" +function cached_results_if_present(::Type{V}, job::CompilerJob) where {V} + ci = get(cache_view(job), job.source, nothing) + ci === nothing && return nothing + return job_results(V, ci, job.config) +end + +function cached_results(::Type{V}, job::CompilerJob) where {V} + v = cached_results_if_present(V, job) + v === nothing || return v + + # No CI exists yet. This path is useful to consumers that need a results object before + # codegen; back-end compile-or-lookup paths avoid it via `cached_results_if_present`. + interp = get_interpreter(job) + ci = CompilerCaching.typeinf!(interp, job.source) + ci === nothing && error("Inference of $(job.source) failed") + return job_results(V, ci, job.config) +end + ## session-dependent results # # Some compilation results embed session-specific data: `relocate_gvs!` bakes absolute @@ -457,12 +533,12 @@ end function wipe_session_dependent_results() Base.@lock session_dependent_lock begin for job in session_dependent_jobs - cache = CompilerCaching.CacheView{JobResults}(cache_owner(job), job.world) + cache = cache_view(job) ci = get(cache, job.source, nothing) ci === nothing && continue jr = CompilerCaching.results(JobResults, ci) Base.@lock cached_results_lock begin - filter!(entry -> entry[1] !== job.config, jr.entries) + filter!(entry -> entry.config !== job.config, jr.entries) end end empty!(session_dependent_jobs) @@ -472,9 +548,9 @@ end end # HAS_INTEGRATED_CACHE -@public GPUCompilerCacheToken, cache_owner, cached_results +@public GPUCompilerCacheToken, cache_owner, cached_results, cached_results_if_present @static if HAS_INTEGRATED_CACHE - @public JobResults + @public JobResults, JobResultEntry end # the method table to use diff --git a/src/jlgen.jl b/src/jlgen.jl index 1eb26039..8a6eab48 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -630,17 +630,21 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) ci′.min_world <= job.world <= ci′.max_world end - # `jl_get_llvm_cis` may return both our owner-token CI and a native - # owner-less CI for the same MI. Prefer ours when present, but keep - # owner-less CIs that are the only compiled entry for their method. + # `jl_get_llvm_cis` may return CIs belonging to several foreign interpreters as + # well as a native owner-less CI for the same MI. Keep only this job's owner, and + # retain an owner-less CI only when this job has no owned entry for that method. + # Treating every non-nothing owner as ours breaks when multiple GPU back-ends are + # loaded in the same process and eventually creates duplicate compiled entries. + owner = cache_owner(job) owned_mis = Set{MethodInstance}() for ci′ in code_instances - if ci′.owner !== nothing + if ci′.owner === owner push!(owned_mis, ci′.def::MethodInstance) end end filter!(code_instances) do ci′ - return ci′.owner !== nothing || ci′.def ∉ owned_mis + return ci′.owner === owner || + (ci′.owner === nothing && ci′.def ∉ owned_mis) end end diff --git a/src/rtlib.jl b/src/rtlib.jl index d0757c43..1dcfff9d 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -74,16 +74,19 @@ end # Compile a single runtime function and link it into `mod`. The renamed bitcode is # memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache # below additionally avoids repeating the parse-and-link work within a session. -function emit_function!(mod, config::CompilerConfig, f, method) - tt = Base.to_tuple_type(method.types) - source = generic_methodinstance(f, tt) +function emit_function!(mod, config::CompilerConfig, source::MethodInstance, method, + world::UInt) name = method.llvm_name - rt_job = CompilerJob(source, config) + rt_job = CompilerJob(source, config, world) - res = cached_results(RuntimeFunctionResults, rt_job) - if res.bitcode !== nothing + # On 1.11+, don't run a standalone inference walk on a miss: `compile_unhooked` below + # drives inference itself. The 1.10 implementation returns its session-local result + # directly, without touching inference. + ci, res = runtime_function_results(rt_job) + if res !== nothing && res.bitcode !== nothing link!(mod, parse(LLVM.Module, MemoryBuffer(res.bitcode))) - return + ci === nothing && (ci = runtime_code_instance(rt_job)) + return ci::CodeInstance end new_mod, meta = compile_unhooked(:llvm, rt_job) @@ -108,33 +111,72 @@ function emit_function!(mod, config::CompilerConfig, f, method) io = IOBuffer() write(io, new_mod) + ci === nothing && (ci = runtime_code_instance(rt_job)) + res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) res.bitcode = take!(io) link!(mod, new_mod) + return ci::CodeInstance +end + +function runtime_function_results(@nospecialize(job::CompilerJob)) + @static if HAS_INTEGRATED_CACHE + ci = get(cache_view(job), job.source, nothing) + ci === nothing && return nothing, nothing + return ci, job_results(RuntimeFunctionResults, ci, job.config) + else + # The 1.10 results store is independent of the CodeCache. Avoid querying the latter + # until the caller actually needs the contributing CI. + return nothing, cached_results(RuntimeFunctionResults, job) + end +end + +function runtime_method_instance(@nospecialize(job::CompilerJob), method) + def = if isa(method.def, Symbol) + isdefined(runtime_module(job), method.def) || return nothing + getfield(runtime_module(job), method.def) + else + method.def + end + # Resolve at the requesting job's explicit world, rather than this task's TLS world. + # Runtime methods may be redefined while a long-lived compilation task is running. + return generic_methodinstance( + typeof(def), Base.to_tuple_type(method.types), job.world) +end + +function runtime_code_instance(@nospecialize(job::CompilerJob)) + ci = @static if HAS_INTEGRATED_CACHE + get(cache_view(job), job.source, nothing) + else + cache = WorldView(get_code_cache(job), job.world, job.world) + CC.get(cache, job.source, nothing) + end + ci === nothing && error("Missing CodeInstance after compiling $(job.source)") + return ci::CodeInstance end # the compiler job passed into here identifies the job that requires the runtime. # derive a config that represents the runtime itself (notably with kernel=false). -# fields that identify the *kernel* job, like its entry-point name, are reset so -# that runtime artifacts are keyed (and persisted) identically for all kernels -# sharing the remaining codegen-relevant settings, instead of once per cosmetic -# config variation. +# Fields that identify or optimize only the *kernel* job are reset so runtime artifacts are +# keyed identically for all kernels sharing the remaining codegen-relevant settings. Runtime +# functions always use the specfunc ABI and are deliberately left unoptimized until linked +# into the toplevel module, making the kernel's entry ABI and LLVM opt level irrelevant. function runtime_config(@nospecialize(job::CompilerJob)) - CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, - strip=false, name=nothing) + CompilerConfig(job.config; kernel=false, entry_abi=:specfunc, opt_level=0, + toplevel=false, only_entry=false, strip=false, name=nothing) end function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) mod = LLVM.Module("GPUCompiler run-time library") + sources = MethodInstance[] + code_instances = CodeInstance[] for method in values(Runtime.methods) - def = if isa(method.def, Symbol) - isdefined(runtime_module(job), method.def) || continue - getfield(runtime_module(job), method.def) - else - method.def - end - emit_function!(mod, config, typeof(def), method) + resolved = runtime_method_instance(job, method) + resolved === nothing && continue + source = resolved + push!(sources, source) + push!(code_instances, emit_function!(mod, config, source, method, job.world)) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -142,7 +184,7 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) # removes Julia address spaces, which would then lead to type mismatches when using # functions from the runtime library from IR that has not been stripped of AS info. - mod + return mod, sources, code_instances end # Session-local cache of assembled runtime libraries, keyed by @@ -152,23 +194,55 @@ end # persistence happens at the per-function level (see `RuntimeFunctionResults`): # reassemble on first use of each session, then reuse within the session. # -# NOTE: unlike the per-function bitcode, whose CodeInstances Julia invalidates on -# method redefinition, the assembled library is world-blind: redefining a runtime -# function mid-session leaves a stale assembly behind. That only affects interactive -# development of a back-end's runtime module (restart, or `empty!` this cache). -const runtime_libs = Dict{Tuple{CompilerConfig, Bool}, Vector{UInt8}}() +# Keep both the selected MethodInstances and their contributing CodeInstances with the +# assembled bytes. Re-resolving methods after the world changes detects direct method-table +# changes; clipped CI ranges detect invalidated transitive callees. This makes the assembled +# cache follow Julia's validity model without a manual `reset_runtime` hook. +mutable struct RuntimeLibrary + bytes::Vector{UInt8} + sources::Vector{MethodInstance} + code_instances::Vector{CodeInstance} + validated_world::UInt +end + +function runtime_library_valid(lib::RuntimeLibrary, @nospecialize(job::CompilerJob)) + # Method-table changes and CI invalidations always advance the world counter. A runtime + # already validated for this exact world therefore needs no per-function scan on the + # common cache-hit path. + job.world == lib.validated_world && return true + + i = 0 + for method in values(Runtime.methods) + resolved = runtime_method_instance(job, method) + resolved === nothing && continue + i += 1 + i <= length(lib.sources) || return false + resolved === lib.sources[i] || return false + end + i == length(lib.sources) || return false + all(ci -> ci.min_world <= job.world <= ci.max_world, lib.code_instances) || return false + lib.validated_world = job.world + return true +end + +const runtime_libs = Dict{Tuple{CompilerConfig, Bool}, RuntimeLibrary}() const runtime_libs_lock = ReentrantLock() @locked function load_runtime(@nospecialize(job::CompilerJob)) config = runtime_config(job) key = (config, !supports_typed_pointers(context())) - bytes = Base.@lock runtime_libs_lock get!(runtime_libs, key) do - lib = build_runtime(job, config) - io = IOBuffer() - write(io, lib) - take!(io) + cached = Base.@lock runtime_libs_lock begin + cached = get(runtime_libs, key, nothing) + if cached === nothing || !runtime_library_valid(cached, job) + lib, sources, code_instances = build_runtime(job, config) + io = IOBuffer() + write(io, lib) + cached = RuntimeLibrary(take!(io), sources, code_instances, job.world) + runtime_libs[key] = cached + end + cached end - return parse(LLVM.Module, MemoryBuffer(bytes); lazy=true) + return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true) end diff --git a/src/spirv.jl b/src/spirv.jl index f0af2f20..97b3a15b 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -27,9 +27,8 @@ Base.@kwdef struct SPIRVCompilerTarget <: AbstractCompilerTarget # SPIR-V extensions, as the comma-separated specifier string passed verbatim to the # translator/back-end via `--spirv-ext`, e.g. "+SPV_EXT_shader_atomic_float_add,+SPV_KHR_expect_assume" # (LLVM feature-string style, cf. `GCNCompilerTarget.features`). Kept as a plain - # `String` -- not a `Vector` -- because target fields must be immutable for the - # `jl_egal`-based cache lookups (`cache_owner`, `JobResults`) to match across - # sessions; see the §2.2 rule in CACHING-REDESIGN.md. + # `String` -- not a `Vector` -- so `jl_egal`-based owner/config lookups can match + # structurally equivalent targets after package-image deserialization. extensions::String = "" supports_fp16::Bool = true supports_fp64::Bool = true diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 63d8a0f3..738811a6 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -20,6 +20,9 @@ NativeCompilerJob = CompilerJob{NativeCompilerTarget,CompilerParams} GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table +GPUCompiler.cache_owner(target::NativeCompilerTarget, params::CompilerParams, + always_inline::Bool) = + GPUCompiler.GPUCompilerCacheToken(target, params, always_inline, params.method_table) GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint function create_job(@nospecialize(func), @nospecialize(types); diff --git a/test/native.jl b/test/native.jl index 871c83fb..b32c6996 100644 --- a/test/native.jl +++ b/test/native.jl @@ -125,6 +125,8 @@ end @testset "cached results" begin mod = @eval module $(gensym()) + Base.Experimental.@MethodTable(other_method_table) + mutable struct Results asm::Union{Nothing,String} Results() = new(nothing) @@ -163,6 +165,20 @@ end job2, _ = Native.create_job(mod.kernel, (Int64,)) @test GPUCompiler.cached_results(mod.Results, job2) === res + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # The compiler may report CIs for several foreign owners when the same MI has + # been inferred through multiple interpreters. Codegen must select this job's + # owner rather than treating every non-native CI as interchangeable. + other_owner_job, _ = Native.create_job( + mod.kernel, (Int64,); method_table=mod.other_method_table) + other_owner_res = GPUCompiler.cached_results(mod.Results, other_owner_job) + @test other_owner_res !== res + JuliaContext() do ctx + _, meta = GPUCompiler.compile(:llvm, job) + @test meta.compiled[job.source].ci.owner === GPUCompiler.cache_owner(job) + end + end + # redefinition invalidates: a job in the new world gets a fresh struct @eval mod kernel(i) = child(i)+2 new_job, _ = Native.create_job(mod.kernel, (Int64,)) @@ -187,6 +203,37 @@ end end end + @testset "runtime cache invalidation" begin + # The assembled runtime cache must follow Julia's CodeInstance invalidation. Runtime + # functions are ordinary Julia methods and can be redefined during a session. + @eval Native.Runtime signal_exception() = nothing + job, _ = Native.create_job(identity, (Nothing,)) + + func_job, _ = Native.create_job(identity, (Nothing,); entry_abi=:func, opt_level=3) + rt_config = GPUCompiler.runtime_config(func_job) + @test rt_config.entry_abi === :specfunc + @test rt_config.opt_level == 0 + + JuliaContext() do ctx + empty!(GPUCompiler.runtime_libs) + GPUCompiler.load_runtime(job) + + key = (GPUCompiler.runtime_config(job), !GPUCompiler.supports_typed_pointers(ctx)) + old = GPUCompiler.runtime_libs[key] + @test GPUCompiler.runtime_library_valid(old, job) + + @eval Native.Runtime signal_exception() = return + new_job, _ = Native.create_job(identity, (Nothing,)) + new_job = CompilerJob(new_job.source, new_job.config, Base.get_world_counter()) + @test !GPUCompiler.runtime_library_valid(old, new_job) + + GPUCompiler.load_runtime(new_job) + new = GPUCompiler.runtime_libs[key] + @test new !== old + @test GPUCompiler.runtime_library_valid(new, new_job) + end + end + @testset "allowed mutable types" begin # when types have no fields, we should always allow them mod = @eval module $(gensym()) diff --git a/test/native/precompile.jl b/test/native/precompile.jl index b48ed86e..0481b636 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -32,6 +32,27 @@ precompile_test_harness("Inference caching") do load_path return Int(x) end + mutable struct Results + artifact::Union{Nothing,String} + Results() = new(nothing) + end + + portable_kernel(x) = x + 1 + session_kernel(x) = x + 2 + + # Attach representative back-end artifacts while the package image is built. The + # portable entry should survive serialization; the session-dependent one should be + # removed by GPUCompiler's pre-output atexit hook. + let + job, _ = NativeCompiler.Native.create_job(portable_kernel, (Int,)) + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "portable" + end + let + job, _ = NativeCompiler.Native.create_job(session_kernel, (Int,)) + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "session" + NativeCompiler.GPUCompiler.mark_session_dependent!(job) + end + let job, _ = NativeCompiler.Native.create_job(kernel, (Vector{Int}, Int)) precompile(job) @@ -88,6 +109,14 @@ precompile_test_harness("Inference caching") do load_path using NativeBackend + portable_job, _ = NativeCompiler.Native.create_job(NativeBackend.portable_kernel, (Int,)) + portable_res = GPUCompiler.cached_results(NativeBackend.Results, portable_job) + @test portable_res.artifact == "portable" + + session_job, _ = NativeCompiler.Native.create_job(NativeBackend.session_kernel, (Int,)) + session_res = GPUCompiler.cached_results(NativeBackend.Results, session_job) + @test session_res.artifact === nothing + # Check that kernel survived kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel), Tuple{Vector{Int}, Int}) @test check_presence(kernel_mi, token) From 07aef089fafc4cb37963cb2b9fad4578148cc87c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jul 2026 11:33:45 +0200 Subject: [PATCH 30/33] Relocate constgv globals in the runtime library. Runtime functions like box_bool reference Julia singletons through julia.constgv globals. Kernel modules get those relocated when the fully-linked toplevel module is finalized, but runtime bitcode is cached before that step ever sees it: bake the session-absolute addresses into the per-function bitcode instead, and mark such jobs session-dependent so the baked pointers stay out of package images. Co-Authored-By: Claude Fable 5 --- src/rtlib.jl | 11 +++++++++++ test/native.jl | 28 ++++++++++++++++++++++++++++ test/ptx.jl | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/src/rtlib.jl b/src/rtlib.jl index 1dcfff9d..8752c7a9 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -99,6 +99,17 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) + # runtime functions may reference Julia objects through `julia.constgv` globals (e.g. + # `box_bool` returning the `jl_true`/`jl_false` singletons). Kernels get theirs + # relocated when the fully-linked toplevel module is finalized, but the runtime's + # mappings would be dropped along with the rest of its per-function metadata: bake + # the session-absolute addresses into the cached bitcode instead, and keep such + # functions out of package images (their pointers don't survive into other sessions). + if !isempty(meta.gv_to_value) + relocate_gvs!(new_mod, meta.gv_to_value) + mark_session_dependent!(rt_job) + end + # rename to the final `gpu_*` name on the per-function module, so the cached bitcode # is immediately link-ready (no per-session rename pass on a cache hit). if haskey(functions(new_mod), name) && functions(new_mod)[name] !== meta.entry diff --git a/test/native.jl b/test/native.jl index b32c6996..9082e61a 100644 --- a/test/native.jl +++ b/test/native.jl @@ -234,6 +234,34 @@ end end end + @testset "runtime constgv relocation" begin + # runtime functions like `box_bool` may reference Julia singletons through + # `julia.constgv` globals. Their session-absolute addresses must be baked into + # the cached runtime bitcode when it is built: only kernel modules go through + # `relocate_gvs!`, so a slot left null here would stay null on the device. + job, _ = Native.create_job(identity, (Nothing,)) + JuliaContext() do ctx + GPUCompiler.load_runtime(job) + key = (GPUCompiler.runtime_config(job), + !GPUCompiler.supports_typed_pointers(ctx)) + lib = Base.@lock GPUCompiler.runtime_libs_lock GPUCompiler.runtime_libs[key] + # NOTE: parse eagerly; a lazily-parsed module doesn't expose uses + rt = parse(LLVM.Module, MemoryBuffer(lib.bytes)) + used = 0 + for gv in globals(rt) + haskey(metadata(gv), "julia.constgv") || continue + isempty(uses(gv)) && continue + used += 1 + init = LLVM.initializer(gv) + @test init !== nothing && !LLVM.isnull(init) + end + @static if VERSION >= v"1.12-" + # on older versions, Julia bakes addresses without tagging globals + @test used > 0 + end + end + end + @testset "allowed mutable types" begin # when types have no fields, we should always allow them mod = @eval module $(gensym()) diff --git a/test/ptx.jl b/test/ptx.jl index 4f6d2209..6e9ac9fa 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -412,6 +412,41 @@ end PTX.code_native(devnull, mod.kernel, Tuple{Float32,Ptr{Float32}}) end +@testset "bool boxes" begin + # boxing a non-constant Bool links the runtime's `box_bool` into the kernel module. + # every `julia.constgv` slot that remains in use in the final module — whether from + # the kernel or from linked runtime bitcode — must have an address relocated into it. + mod = @eval module $(gensym()) + function kernel(a, b) + val = unsafe_load(a) + # box `val < -1f0` for the @nospecialize InexactError ctor + val < 0f0 && throw(InexactError(:kernel, Int, val < -1f0)) + unsafe_store!(b, val) + return + end + end + + JuliaContext() do ctx + job, _ = PTX.create_job(mod.kernel, Tuple{Ptr{Float32},Ptr{Float32}}; + kernel=true, optimize=false, validate=false) + ir, meta = GPUCompiler.compile(:llvm, job) + @test haskey(functions(ir), "ijl_box_bool") + # all used constant-global slots must have been relocated to actual addresses + used_constgvs = 0 + for gv in globals(ir) + haskey(metadata(gv), "julia.constgv") || continue + isempty(uses(gv)) && continue + used_constgvs += 1 + init = LLVM.initializer(gv) + @test init !== nothing && !LLVM.isnull(init) + end + if VERSION >= v"1.12-" + # on older versions, Julia bakes addresses directly instead of using constgvs + @test used_constgvs > 0 + end + end +end + @testset "fastmath" begin # `fastmath=true` on the target calls `apply_fastmath!` from # `finish_linked_module!`, stamping `unsafe-fp-math` + fast-math flags on From 536691eaff893abc42a9e3b51852a7d05e425e3f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jul 2026 11:34:07 +0200 Subject: [PATCH 31/33] Simplify the caching interface. Fold cached_results_if_present into cached_results, which is now lookup-only: it returns nothing when no code exists for the job, and back-ends compile and re-fetch (codegen drives inference itself, so a standalone inference walk is never needed). Consumers that want to attach results without generating code call precompile(job) first. Drop the method table from GPUCompilerCacheToken: everything inference derives from a job (method_table, method_table_view, inference_params, optimization_params) must be a pure function of the config's target and params values plus always_inline, so the default token already partitions correctly and back-ends no longer override cache_owner. Also chase the compilable MethodInstance in cache lookups on 1.14+, where inference caches vararg kernels under the vararg-widened signature rather than the fully-specialized job.source. Co-Authored-By: Claude Fable 5 --- src/deprecated.jl | 7 +- src/interface.jl | 132 +++++++++++++++++++------------------- src/rtlib.jl | 4 +- test/helpers/native.jl | 3 - test/native.jl | 27 +++++++- test/native/precompile.jl | 4 ++ 6 files changed, 101 insertions(+), 76 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index b0987db6..4b9ee01d 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -239,7 +239,8 @@ end const job_results = Dict{Any,Any}() const job_results_lock = ReentrantLock() -# specialized for the launch hot path, mirroring the 1.11+ implementation +# specialized for the launch hot path, mirroring the 1.11+ implementation. As the store is +# independent of CodeInstances, the lookup always succeeds (`nothing` is never returned). function cached_results(::Type{V}, job::CompilerJob) where {V} # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive # boxing allocation; the MI is kept alive by its method specializations. @@ -251,10 +252,6 @@ function cached_results(::Type{V}, job::CompilerJob) where {V} end end -# The 1.10 results store is independent of CodeInstances, so obtaining its empty result does -# not trigger inference. Match the integrated-cache API used by back-end compile-or-lookup paths. -cached_results_if_present(::Type{V}, job::CompilerJob) where {V} = cached_results(V, job) - ## 1.10 session-dependent results # diff --git a/src/interface.jl b/src/interface.jl index 433bc6f6..0882229a 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -68,30 +68,34 @@ nest_params(params::AbstractCompilerParams, parent::AbstractCompilerParams) = pa ## cache owner -# Inference results are shared by jobs with the same target, params, inlining policy, and -# method-table identity. Store this token on the CompilerConfig (as `Any`) so the immutable -# value is boxed once when the config is created, instead of on every integrated-cache lookup. -struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams, M} +# Inference results are shared by jobs with the same target, params, and inlining policy. +# On 1.11+, store this token on the CompilerConfig (as `Any`) so the immutable value is +# boxed once when the config is created, instead of on every integrated-cache lookup. +struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams} target::T params::P always_inline::Bool - method_table::M end """ cache_owner(target, params, always_inline) -Construct the immutable token that partitions inference for a compiler configuration. -Back-ends that override `method_table` or `method_table_view` must override this form as -well and include every method-table identity used by the view. On Julia 1.11+, the -job-level `cache_owner(job)` returns the pre-boxed token stored on its `CompilerConfig`. +Construct the immutable token that partitions inference for a compiler configuration. On +Julia 1.11+, the job-level `cache_owner(job)` returns the pre-boxed token stored on its +`CompilerConfig`. -The returned value must match under `===`/`jl_egal` after package-image deserialization; -use immutable containers and canonical method-table objects. +The default token covers the full `target` and `params` instances plus `always_inline`. +That is sufficient because the inference inputs derived from a job — `method_table`, +`method_table_view`, `inference_params` and `optimization_params` — must be pure +functions of those values, so back-ends normally leave this untouched. + +When overriding, the returned value must match under `===`/`jl_egal` after package-image +deserialization: use immutable containers, and only reference mutable objects (like method +tables) that are module-level singletons. """ cache_owner(target::AbstractCompilerTarget, params::AbstractCompilerParams, always_inline::Bool) = - GPUCompilerCacheToken(target, params, always_inline, GLOBAL_METHOD_TABLE) + GPUCompilerCacheToken(target, params, always_inline) ## config @@ -360,13 +364,6 @@ valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false # session-local `GLOBAL_CI_CACHES` partition in `deprecated.jl`; the immutability # requirement is still useful for stable hashing. # -# Care is required for anything that impacts: -# - method_table -# - inference_params -# - optimization_params -# The default covers the full target+params instances (so backends with version- or -# arch-specific knobs partition cleanly), `always_inline` (which feeds optimization_params), -# and the method table. # The token is created and boxed by the CompilerConfig constructor. Returning the boxed field # is important: passing a freshly-constructed immutable token to the Any-typed C cache API # allocates on every kernel-cache hit. @@ -375,17 +372,19 @@ valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false else # Preserve the allocation-free 1.10 path: constructing this specialized immutable token # is cheaper than storing an Any-typed box on every CompilerConfig. - cache_owner(job::CompilerJob) = GPUCompilerCacheToken( - job.config.target, job.config.params, job.config.always_inline, method_table(job)) + cache_owner(job::CompilerJob) = + cache_owner(job.config.target, job.config.params, job.config.always_inline) end """ - cached_results(::Type{V}, job::CompilerJob) -> V + cached_results(::Type{V}, job::CompilerJob) -> Union{Nothing,V} -Retrieve the compilation-results struct of type `V` for `job`, creating an empty one on -first access. `V` must be a `mutable struct` with a zero-arg constructor; back-ends -define one such struct holding the artifacts of each compilation stage, check whether -the relevant fields are populated, and fill them in (running the compiler) when not: +Retrieve the compilation-results struct of type `V` for `job`. Returns `nothing` when no +code has been compiled (or inferred) for the job yet; once code exists, an empty `V` is +created on first access and the same struct is returned ever after. `V` must be a +`mutable struct` with a zero-arg constructor; back-ends define one such struct holding +the artifacts of each compilation stage, check whether the relevant fields are +populated, and fill them in (running the compiler) when not: ```julia mutable struct MetalResults @@ -395,14 +394,19 @@ mutable struct MetalResults MetalResults() = new(nothing, nothing, []) end -res = GPUCompiler.cached_results_if_present(MetalResults, job) +res = GPUCompiler.cached_results(MetalResults, job) if res === nothing || res.metallib === nothing || GPUCompiler.compile_hook[] !== nothing artifacts = ...compile... - res === nothing && (res = GPUCompiler.cached_results(MetalResults, job)) + res = @something res GPUCompiler.cached_results(MetalResults, job) ...populate res from artifacts... end ``` +Compiling the job (through `GPUCompiler.compile`) populates Julia's code cache, so the +post-compile lookup in the example is guaranteed to succeed. To attach results without +generating code — e.g. from an inference-only precompilation workload — run +`precompile(job)` first. + Results are keyed by the *full* compiler job: its method instance, world age, and entire `CompilerConfig` (two jobs differing only in, say, the kernel `name` get distinct structs). Storage differs per Julia version, transparently to the back-end: @@ -410,14 +414,15 @@ structs). Storage differs per Julia version, transparently to the back-end: - On Julia 1.11+, the struct lives on the `CodeInstance`'s `analysis_results` chain in Julia's integrated code cache, partitioned by [`cache_owner`](@ref) and keyed by config within the per-CI [`JobResults`](@ref) container. Method redefinition - invalidates the CI — and with it the attached results. When no CI exists yet, - inference is run to create one. Artifacts stored during precompilation are serialized - into the package image along with the CI: populate only session-portable values - (bytes, strings) during precompile workloads, and keep session-local handles (device - modules, pipeline objects) in fields that remain empty until first use at run time. + invalidates the CI — and with it the attached results. Artifacts stored during + precompilation are serialized into the package image along with the CI: populate only + session-portable values (bytes, strings) during precompile workloads, and keep + session-local handles (device modules, pipeline objects) in fields that remain empty + until first use at run time. -- On Julia 1.10, the struct lives in a session-local Dict; redefinition protection - comes from the world age in the key, and nothing persists across sessions. +- On Julia 1.10, the struct lives in a session-local Dict keyed by the job, so the + lookup always succeeds (`nothing` is never returned); redefinition protection comes + from the world age in the key, and nothing persists across sessions. Thread safety: concurrent calls for the same job return the same struct, but GPUCompiler does not serialize back-end *compilation*; take a back-end lock around @@ -433,8 +438,8 @@ function cached_results end Per-CodeInstance container mapping a `CompilerConfig` to the back-end's results struct. A `CodeInstance` is shared by every job whose [`cache_owner`](@ref) matches — the owner -token only covers what affects *inference* (target, params, `always_inline`, method -table). The remaining `CompilerConfig` fields (`kernel`, `name`, `entry_abi`, +token only covers what affects *inference* (target, params, `always_inline`). The +remaining `CompilerConfig` fields (`kernel`, `name`, `entry_abi`, `opt_level`, …) do affect *codegen*, so artifacts must not be shared across them. Entries are matched by `===` (`jl_egal`): `CompilerConfig` is an immutable struct, so structurally identical configs compare equal, including against configs deserialized @@ -475,31 +480,28 @@ function cache_view(@nospecialize(job::CompilerJob)) CompilerCaching.CacheView{Any,JobResults}(cache_owner(job), job.world) end -""" - cached_results_if_present(::Type{V}, job::CompilerJob) -> Union{Nothing,V} - -Return the results struct for `job` when its CodeInstance already exists, creating only the -per-config `V` entry. Return `nothing` without running inference when no CI exists. - -Back-end compile-or-lookup paths should use this first: on a miss, run codegen (which drives -inference itself), then call [`cached_results`](@ref) to attach the artifacts. This avoids a -standalone inference walk immediately followed by the compiler's own walk. -""" -function cached_results_if_present(::Type{V}, job::CompilerJob) where {V} - ci = get(cache_view(job), job.source, nothing) - ci === nothing && return nothing - return job_results(V, ci, job.config) +# Fetch the CodeInstance backing `job` from the integrated cache. On 1.14+, inference +# caches vararg methods under their compilable (vararg-widened) MethodInstance, which +# for such methods differs from the fully-specialized `job.source`: retry there. +function job_code_instance(@nospecialize(job::CompilerJob)) + cache = cache_view(job) + ci = get(cache, job.source, nothing) + @static if VERSION >= v"1.14-" + if ci === nothing + mi = ccall(:jl_normalize_to_compilable_mi, Any, (Any,), + job.source)::MethodInstance + mi === job.source || (ci = get(cache, mi, nothing)) + end + end + return ci end +# Deliberately lookup-only: when no CI exists, back-ends run codegen (which drives +# inference itself) and re-fetch, instead of us running a standalone inference walk here +# that the compiler's own walk would immediately repeat. function cached_results(::Type{V}, job::CompilerJob) where {V} - v = cached_results_if_present(V, job) - v === nothing || return v - - # No CI exists yet. This path is useful to consumers that need a results object before - # codegen; back-end compile-or-lookup paths avoid it via `cached_results_if_present`. - interp = get_interpreter(job) - ci = CompilerCaching.typeinf!(interp, job.source) - ci === nothing && error("Inference of $(job.source) failed") + ci = job_code_instance(job) + ci === nothing && return nothing return job_results(V, ci, job.config) end @@ -533,8 +535,7 @@ end function wipe_session_dependent_results() Base.@lock session_dependent_lock begin for job in session_dependent_jobs - cache = cache_view(job) - ci = get(cache, job.source, nothing) + ci = job_code_instance(job) ci === nothing && continue jr = CompilerCaching.results(JobResults, ci) Base.@lock cached_results_lock begin @@ -548,12 +549,13 @@ end end # HAS_INTEGRATED_CACHE -@public GPUCompilerCacheToken, cache_owner, cached_results, cached_results_if_present -@static if HAS_INTEGRATED_CACHE - @public JobResults, JobResultEntry -end +@public GPUCompilerCacheToken, cache_owner, cached_results # the method table to use +# +# NOTE: these (like `inference_params` and `optimization_params` below) may only depend on +# the job's world and its config's `target`/`params` values (+ `always_inline`); +# [`cache_owner`](@ref) relies on that to partition inference results correctly. # deprecate method_table on next-breaking release method_table(@nospecialize(job::CompilerJob)) = GLOBAL_METHOD_TABLE method_table_view(@nospecialize(job::CompilerJob)) = get_method_table_view(job.world, method_table(job)) diff --git a/src/rtlib.jl b/src/rtlib.jl index 8752c7a9..3089d026 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -132,7 +132,7 @@ end function runtime_function_results(@nospecialize(job::CompilerJob)) @static if HAS_INTEGRATED_CACHE - ci = get(cache_view(job), job.source, nothing) + ci = job_code_instance(job) ci === nothing && return nothing, nothing return ci, job_results(RuntimeFunctionResults, ci, job.config) else @@ -157,7 +157,7 @@ end function runtime_code_instance(@nospecialize(job::CompilerJob)) ci = @static if HAS_INTEGRATED_CACHE - get(cache_view(job), job.source, nothing) + job_code_instance(job) else cache = WorldView(get_code_cache(job), job.world, job.world) CC.get(cache, job.source, nothing) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 738811a6..63d8a0f3 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -20,9 +20,6 @@ NativeCompilerJob = CompilerJob{NativeCompilerTarget,CompilerParams} GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table -GPUCompiler.cache_owner(target::NativeCompilerTarget, params::CompilerParams, - always_inline::Bool) = - GPUCompiler.GPUCompilerCacheToken(target, params, always_inline, params.method_table) GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint function create_job(@nospecialize(func), @nospecialize(types); diff --git a/test/native.jl b/test/native.jl index 9082e61a..bb9abeee 100644 --- a/test/native.jl +++ b/test/native.jl @@ -140,8 +140,16 @@ end kernel(i) = child(i)+1 end - # get-or-create: first access yields an empty struct, later accesses the same one job, _ = Native.create_job(mod.kernel, (Int64,)) + + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # before any code exists for the job, the lookup comes up empty + @test GPUCompiler.cached_results(mod.Results, job) === nothing + end + + # get-or-create: first access after inference yields an empty struct, later + # accesses return the same one + precompile(job) res = GPUCompiler.cached_results(mod.Results, job) @test res isa mod.Results @test res.asm === nothing @@ -165,12 +173,23 @@ end job2, _ = Native.create_job(mod.kernel, (Int64,)) @test GPUCompiler.cached_results(mod.Results, job2) === res + # vararg kernels: on 1.14+, inference caches these under the compilable + # (vararg-widened) MethodInstance rather than the fully-specialized job.source, + # which the lookup needs to chase + vmod = @eval module $(gensym()) + kernel(args...) = nothing + end + vjob, _ = Native.create_job(vmod.kernel, (Int64, Int64)) + precompile(vjob) + @test GPUCompiler.cached_results(mod.Results, vjob) isa mod.Results + @static if GPUCompiler.HAS_INTEGRATED_CACHE # The compiler may report CIs for several foreign owners when the same MI has # been inferred through multiple interpreters. Codegen must select this job's # owner rather than treating every non-native CI as interchangeable. other_owner_job, _ = Native.create_job( mod.kernel, (Int64,); method_table=mod.other_method_table) + precompile(other_owner_job) other_owner_res = GPUCompiler.cached_results(mod.Results, other_owner_job) @test other_owner_res !== res JuliaContext() do ctx @@ -182,6 +201,12 @@ end # redefinition invalidates: a job in the new world gets a fresh struct @eval mod kernel(i) = child(i)+2 new_job, _ = Native.create_job(mod.kernel, (Int64,)) + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # ... after first showing up empty, as the old CodeInstance no longer covers + # the new world + @test GPUCompiler.cached_results(mod.Results, new_job) === nothing + end + precompile(new_job) new_res = GPUCompiler.cached_results(mod.Results, new_job) @test new_res !== res @test new_res.asm === nothing diff --git a/test/native/precompile.jl b/test/native/precompile.jl index 0481b636..eb9abc4f 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -45,10 +45,12 @@ precompile_test_harness("Inference caching") do load_path # removed by GPUCompiler's pre-output atexit hook. let job, _ = NativeCompiler.Native.create_job(portable_kernel, (Int,)) + precompile(job) NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "portable" end let job, _ = NativeCompiler.Native.create_job(session_kernel, (Int,)) + precompile(job) NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "session" NativeCompiler.GPUCompiler.mark_session_dependent!(job) end @@ -111,10 +113,12 @@ precompile_test_harness("Inference caching") do load_path portable_job, _ = NativeCompiler.Native.create_job(NativeBackend.portable_kernel, (Int,)) portable_res = GPUCompiler.cached_results(NativeBackend.Results, portable_job) + @test portable_res !== nothing @test portable_res.artifact == "portable" session_job, _ = NativeCompiler.Native.create_job(NativeBackend.session_kernel, (Int,)) session_res = GPUCompiler.cached_results(NativeBackend.Results, session_job) + @test session_res !== nothing @test session_res.artifact === nothing # Check that kernel survived From 5954d9b671b501d452508900beaa2eeb0e60287c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jul 2026 11:50:27 +0200 Subject: [PATCH 32/33] Remove the runtime disk cache test. The on-disk runtime library cache it exercises does not exist anymore; runtime bitcode is cached per-CodeInstance and assembled libraries are memoized in-memory. Co-Authored-By: Claude Fable 5 --- test/ptx.jl | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/test/ptx.jl b/test/ptx.jl index 6e9ac9fa..d3617334 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -28,34 +28,6 @@ end @test occursin("[1 x i64] %state", ir) end -@testset "runtime cache disappearing" begin - # the cache may be deleted or unusable at any point; compilation should keep working - mod = @eval module $(gensym()) - kernel() = return - end - old_cache = GPUCompiler.compile_cache - try - GPUCompiler.compile_cache = mktempdir() - PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true) - @test !isempty(readdir(GPUCompiler.compile_cache)) - - # remove the cache directory altogether - GPUCompiler.reset_runtime() - PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true) - @test !isempty(readdir(GPUCompiler.compile_cache)) - - # make the cache unwritable; compilation should proceed with only a warning - GPUCompiler.reset_runtime() - write(GPUCompiler.compile_cache, "not a directory") - @test_logs (:warn, r"Failed to cache GPU runtime library") match_mode=:any begin - PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true) - end - rm(GPUCompiler.compile_cache) - finally - GPUCompiler.compile_cache = old_cache - end -end - @testset "kernel functions" begin @testset "kernel argument attributes" begin mod = @eval module $(gensym()) From 02ebc1351851e0235d6a3809988ccecdb4fe5780 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 10 Jul 2026 12:44:43 +0200 Subject: [PATCH 33/33] Remove the invalid PTX bool-boxes test. Julia codegen never emits an ijl_box_bool call when boxing a Bool: it inlines a select over the jl_true/jl_false singletons instead, so the runtime's box_bool is never linked into this kernel and the module contains no constgv slots to inspect. The test only appeared to pass locally because the NVPTX back-end is unavailable there, skipping the entire assembly section; CI was the first to actually run it. Relocation of runtime-library constgv globals remains covered by the native-target "runtime constgv relocation" testset, which runs (and passes) on all CI platforms. Co-Authored-By: Claude Fable 5 --- test/ptx.jl | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/test/ptx.jl b/test/ptx.jl index d3617334..6b82db5e 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -384,41 +384,6 @@ end PTX.code_native(devnull, mod.kernel, Tuple{Float32,Ptr{Float32}}) end -@testset "bool boxes" begin - # boxing a non-constant Bool links the runtime's `box_bool` into the kernel module. - # every `julia.constgv` slot that remains in use in the final module — whether from - # the kernel or from linked runtime bitcode — must have an address relocated into it. - mod = @eval module $(gensym()) - function kernel(a, b) - val = unsafe_load(a) - # box `val < -1f0` for the @nospecialize InexactError ctor - val < 0f0 && throw(InexactError(:kernel, Int, val < -1f0)) - unsafe_store!(b, val) - return - end - end - - JuliaContext() do ctx - job, _ = PTX.create_job(mod.kernel, Tuple{Ptr{Float32},Ptr{Float32}}; - kernel=true, optimize=false, validate=false) - ir, meta = GPUCompiler.compile(:llvm, job) - @test haskey(functions(ir), "ijl_box_bool") - # all used constant-global slots must have been relocated to actual addresses - used_constgvs = 0 - for gv in globals(ir) - haskey(metadata(gv), "julia.constgv") || continue - isempty(uses(gv)) && continue - used_constgvs += 1 - init = LLVM.initializer(gv) - @test init !== nothing && !LLVM.isnull(init) - end - if VERSION >= v"1.12-" - # on older versions, Julia bakes addresses directly instead of using constgvs - @test used_constgvs > 0 - end - end -end - @testset "fastmath" begin # `fastmath=true` on the target calls `apply_fastmath!` from # `finish_linked_module!`, stamping `unsafe-fp-math` + fast-math flags on