diff --git a/LocalPreferences.toml b/LocalPreferences.toml deleted file mode 100644 index f5638dd8..00000000 --- a/LocalPreferences.toml +++ /dev/null @@ -1,4 +0,0 @@ -[GPUCompiler] -# wether caching of object files should be enabled. If the disk cache is enabled -# cache files are storied in scratch memory. -#disk_cache = "false" diff --git a/Project.toml b/Project.toml index 64b5851c..d4ad4bda 100644 --- a/Project.toml +++ b/Project.toml @@ -1,12 +1,13 @@ name = "GPUCompiler" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "1.23.0" +version = "2.0.0" authors = ["Tim Besard "] [workspace] projects = ["test"] [deps] +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" ExprTools = "e2ba6199-217a-4e67-a87a-7c52f15ade04" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" @@ -15,8 +16,6 @@ Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Scratch = "6c6a2e73-6563-6170-7368-637461726353" -Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tracy = "e689c965-62c8-4b79-b2c5-8359227902fd" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -26,8 +25,10 @@ AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" LLVMDowngrader_jll = "f52de702-fb25-5922-94ba-81dd59b07444" NVPTX_LLVM_Backend_jll = "ef6e0fe3-e6ef-59c0-bde6-4989574699e0" + [compat] AMDGPU_LLVM_Backend_jll = "22" +CompilerCaching = "0.3" ExprTools = "0.1" InteractiveUtils = "1" LLVM = "9.9" @@ -38,8 +39,6 @@ NVPTX_LLVM_Backend_jll = "22" PrecompileTools = "1" Preferences = "1" REPL = "1" -Scratch = "1" -Serialization = "1" TOML = "1" Tracy = "0.1.4" UUIDs = "1" diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index 2e646267..49017261 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -8,8 +8,6 @@ using ExprTools: splitdef, combinedef using Libdl -using Serialization -using Scratch: @get_scratch! using Preferences const ENABLE_TRACY = parse(Bool, @load_preference("tracy", "false")) @@ -35,8 +33,21 @@ end const CC = Core.Compiler using Core: MethodInstance, CodeInstance, CodeInfo -compile_cache = nothing # set during __init__() -const pkgver = Base.pkgversion(GPUCompiler) +# `HAS_INTEGRATED_CACHE` distinguishes 1.11+ (owner-keyed `Core.Compiler.InternalCodeCache`) +# from 1.10 (per-interpreter `CodeCache` IdDict + invalidation callbacks). The two have +# disjoint shapes; everything cache-related fans on this flag. +const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" + +# Loads as an empty shell on 1.10; on 1.11+ provides `CacheView`, `typeinf!`, +# `get_codeinfos`, `lookup`, `results`, etc. We `import` the module name (not its +# exports) to avoid clashing with `LLVM.lookup`; internal call sites qualify with +# `CompilerCaching.`. +import CompilerCaching + +# Optional callback invoked from `compile(...)` / `cached_compilation(...)` before +# compilation runs. Set by `@device_code_*` reflection macros. Defined here (early) +# so the legacy `cached_compilation` in deprecated.jl can reference it. +const compile_hook = Ref{Union{Nothing,Function}}(nothing) include("utils.jl") include("mangling.jl") @@ -54,6 +65,7 @@ include("metal.jl") include("runtime.jl") # compiler implementation +include("deprecated.jl") include("jlgen.jl") include("irgen.jl") include("optim.jl") @@ -67,24 +79,19 @@ include("driver.jl") include("execution.jl") include("reflection.jl") -include("deprecated.jl") - include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) - dir = @get_scratch!("compiled") - ## add the Julia version - dir = joinpath(dir, "v$(VERSION.major).$(VERSION.minor)") - ## also add the package version - if pkgver !== nothing - # XXX: `Base.pkgversion` is buggy and sometimes returns `nothing`, see e.g. - # JuliaLang/PackageCompiler.jl#896 and JuliaGPU/GPUCompiler.jl#593 - dir = joinpath(dir, "v$(pkgver.major).$(pkgver.minor)") + @static if !HAS_INTEGRATED_CACHE + # session-local results keyed by objectid+world; entries serialized during + # GPUCompiler's own precompilation can never be valid in a later session + empty!(job_results) + # ditto for the in-process CodeCaches: CIs deposited by our own precompile + # workload carry world ages from the precompilation process + empty!(GLOBAL_CI_CACHES) end - mkpath(dir) - global compile_cache = dir @static if ENABLE_TRACY Tracy.@register_tracepoints() diff --git a/src/bpf.jl b/src/bpf.jl index 80070d6d..ff75ea23 100644 --- a/src/bpf.jl +++ b/src/bpf.jl @@ -26,8 +26,6 @@ end ## job -runtime_slug(job::CompilerJob{BPFCompilerTarget}) = "bpf" - const bpf_intrinsics = () # TODO isintrinsic(::CompilerJob{BPFCompilerTarget}, fn::String) = in(fn, bpf_intrinsics) diff --git a/src/deprecated.jl b/src/deprecated.jl index c5848854..4b9ee01d 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -1,45 +1,327 @@ -# Deprecations scheduled for removal in the next major release. +# Legacy and deprecated functionality. +# +# This file houses code paths that are kept for Julia 1.10 (which lacks the integrated +# CodeInstance cache and `analysis_results`) plus older interfaces retained for the 2.0 +# breaking release. Everything here is expected to go away once 1.10 falls off the LTS +# line and downstream packages have migrated. + +# `WorldView` is used by the integrated-cache JIT lookup callback (1.11–1.13) and by the +# legacy `CodeCache` interface (1.10). Moved out of `Core.Compiler` on 1.14+. +@static if VERSION < v"1.14-" + using Core.Compiler: WorldView +end + + +## Julia 1.10: in-process `CodeCache` + WorldView + +@static if !HAS_INTEGRATED_CACHE + +""" + CodeCache + +Per-job CodeInstance store used on Julia 1.10, where there's no integrated cache and no +`cache_owner` partition on `CodeInstance`. Inference deposits CIs via a `WorldView` over +this store; invalidation callbacks attached to each `MethodInstance` clip `max_world` on +stale CIs when methods are redefined. +""" +struct CodeCache + dict::IdDict{MethodInstance, Vector{CodeInstance}} + + CodeCache() = new(IdDict{MethodInstance, Vector{CodeInstance}}()) +end + +function Base.show(io::IO, ::MIME"text/plain", cc::CodeCache) + print(io, "CodeCache with $(mapreduce(length, +, values(cc.dict); init=0)) entries") + if !isempty(cc.dict) + print(io, ": ") + for (mi, cis) in cc.dict + println(io) + print(io, " ") + show(io, mi) + for ci in cis + println(io) + worldstr = if ci.min_world == typemax(UInt) + "empty world range" + elseif ci.max_world == typemax(UInt) + "worlds $(Int(ci.min_world))+" + else + "worlds $(Int(ci.min_world)) to $(Int(ci.max_world))" + end + print(io, " CodeInstance for ", worldstr) + end + end + end +end + +Base.empty!(cc::CodeCache) = empty!(cc.dict) + +# Session-local registry of `CodeCache`s, keyed by the configuration that produced them. +# 1.10 has no per-CI partitioning, so we partition by `cache_owner(job)` ourselves. +const GLOBAL_CI_CACHES = Dict{Any, CodeCache}() +const GLOBAL_CI_CACHES_LOCK = ReentrantLock() + +function get_code_cache(@nospecialize(job::CompilerJob)) + key = cache_owner(job) + Base.@lock GLOBAL_CI_CACHES_LOCK get!(GLOBAL_CI_CACHES, key) do + CodeCache() + end +end + +# Invalidation: keep `max_world` honest when callees in a CI's edge graph change. +struct CodeCacheCallback + cache::CodeCache +end + +function CC.setindex!(cache::CodeCache, ci::CodeInstance, mi::MethodInstance) + add_codecache_callback!(cache, mi) + cis = get!(cache.dict, mi, CodeInstance[]) + push!(cis, ci) +end + +@static if VERSION ≥ v"1.11.0-DEV.798" + +function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) + callback = CodeCacheCallback(cache) + CC.add_invalidation_callback!(callback, mi) +end +function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32) + cis = get(callback.cache.dict, replaced, nothing) + cis === nothing && return + for ci in cis + if ci.max_world == ~0 % Csize_t + @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" +@static if VERSION >= v"1.11.0-DEV.1390" + @atomic ci.max_world = max_world +else + ci.max_world = max_world +end + end + @assert ci.max_world <= max_world + end +end + +else + +function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) + callback = CodeCacheCallback(cache) + if !isdefined(mi, :callbacks) + mi.callbacks = Any[callback] + elseif !in(callback, mi.callbacks) + push!(mi.callbacks, callback) + end +end +function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32, + seen::Set{MethodInstance}=Set{MethodInstance}()) + push!(seen, replaced) + + cis = get(callback.cache.dict, replaced, nothing) + if cis !== nothing + for ci in cis + if ci.max_world == ~0 % Csize_t + @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" + ci.max_world = max_world + end + @assert ci.max_world <= max_world + end + end + + # recurse into backedges to update their valid range too + if isdefined(replaced, :backedges) + backedges = filter(replaced.backedges) do @nospecialize(mi) + if mi isa MethodInstance + mi ∉ seen + elseif mi isa Type + # an `invoke` call, which is a `(sig, MethodInstance)` pair. + # let's ignore the `sig` and process the `MethodInstance` next. + false + else + error("invalid backedge") + end + end + + # Don't touch/empty backedges — `invalidate_method_instance` in C does that later. + + for mi in backedges + callback(mi::MethodInstance, max_world, seen) + end + end +end + +end -function defs(mod::LLVM.Module) - safe_depwarn("`GPUCompiler.defs(mod)` is deprecated; inline `filter(f -> !isdeclaration(f), collect(functions(mod)))`.", - :defs) - filter(f -> !isdeclaration(f), collect(functions(mod))) +## 1.10 cache view interface + +function CC.haskey(wvc::WorldView{CodeCache}, mi::MethodInstance) + CC.get(wvc, mi, nothing) !== nothing +end + +function CC.get(wvc::WorldView{CodeCache}, mi::MethodInstance, default) + for ci in get!(wvc.cache.dict, mi, CodeInstance[]) + if ci.min_world <= wvc.worlds.min_world && wvc.worlds.max_world <= ci.max_world + return ci + end + end + return default +end + +function CC.getindex(wvc::WorldView{CodeCache}, mi::MethodInstance) + r = CC.get(wvc, mi, nothing) + r === nothing && throw(KeyError(mi)) + return r::CodeInstance end -function decls(mod::LLVM.Module) - safe_depwarn("`GPUCompiler.decls(mod)` is deprecated; inline `filter(f -> isdeclaration(f) && !LLVM.isintrinsic(f), collect(functions(mod)))`.", - :decls) - filter(f -> isdeclaration(f) && !LLVM.isintrinsic(f), collect(functions(mod))) +function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodInstance) + CC.setindex!(wvc.cache, ci, mi) end -link_library!(mod::LLVM.Module, lib::LLVM.Module) = link_library!(mod, [lib]) -function link_library!(mod::LLVM.Module, libs::Vector{LLVM.Module}) - safe_depwarn("`GPUCompiler.link_library!` is deprecated; call `LLVM.link!(mod, copy(lib))` directly, or `LLVM.link!(mod, lib; only_needed=true)` with a freshly-parsed library.", - :link_library!) - libs = [copy(lib) for lib in libs] - for lib in libs - link!(mod, lib) + +## 1.10 StackedMethodTable +# +# Priority-lookup method table. The 1.11+ version lives in CompilerCaching; here we keep +# the 1.10 copy, which has to produce the older `MethodMatchResult` shape from +# `CC.findall` / `CC.findsup`. +struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView + world::UInt + mt::Core.MethodTable + parent::MTV +end +StackedMethodTable(world::UInt, mt::Core.MethodTable) = + StackedMethodTable(world, mt, CC.InternalMethodTable(world)) +StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) = + StackedMethodTable(world, mt, StackedMethodTable(world, parent)) + +CC.isoverlayed(::StackedMethodTable) = true + +function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) + result = CC._findall(sig, table.mt, table.world, limit) + result === nothing && return nothing # too many matches + nr = CC.length(result) + if nr ≥ 1 && CC.getindex(result, nr).fully_covers + return CC.MethodMatchResult(result, true) end + + parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} + parent_result === nothing && return nothing # too many matches + + overlayed = parent_result.overlayed | !CC.isempty(result) + parent_result = parent_result.matches::CC.MethodLookupResult + + return CC.MethodMatchResult( + CC.MethodLookupResult( + CC.vcat(result.matches, parent_result.matches), + CC.WorldRange( + CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), + CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), + result.ambig | parent_result.ambig), + overlayed) +end + +function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) + match, valid_worlds = CC._findsup(sig, table.mt, table.world) + match !== nothing && return match, valid_worlds, true + parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) + return ( + parent_match, + CC.WorldRange( + max(valid_worlds.min_world, parent_valid_worlds.min_world), + min(valid_worlds.max_world, parent_valid_worlds.max_world)), + overlayed) end -# no-op 3-arg fallback so downstream overrides that chain via -# `invoke(GPUCompiler.link_libraries!, Tuple{CompilerJob, Module, -# Vector{String}}, ...)` still resolve. -link_libraries!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - undefined_fns::Vector{String}) = return -# `true` when a downstream package has defined a 3-arg `link_libraries!` -# override for `job`, i.e. the dispatched method isn't our fallback above. +## 1.10 `cached_results` # -# Uses the same `jl_gf_invoke_lookup` path as `Core._hasmethod` rather than -# `which`, so it's safe to call from generated-function-adjacent contexts -# where `Base.get_world_counter()` returns `typemax(UInt)` and reflection -# queries like `which` / `methods` fail (see JuliaLang/julia#48611). -# All this because Enzyme.jl calls GPUCompiler.jl from a generated function. -function has_legacy_link_libraries(@nospecialize(job::CompilerJob)) - tt = Tuple{typeof(link_libraries!), typeof(job), - LLVM.Module, Vector{String}} - world = ccall(:jl_get_tls_world_age, UInt, ()) - m = ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) - return m !== nothing && (m::Method).module !== @__MODULE__ +# Session-local storage for the per-job results structs; on 1.11+ these live on the +# `CodeInstance`s of Julia's integrated cache instead (see `interface.jl`). The key +# mirrors the 1.11+ semantics — the full job identity, i.e. method instance, world, +# and entire config — but nothing persists across sessions, and redefinition +# protection comes from the world age in the key rather than from CI invalidation. +const job_results = Dict{Any,Any}() +const job_results_lock = ReentrantLock() + +# specialized for the launch hot path, mirroring the 1.11+ implementation. As the store is +# independent of CodeInstances, the lookup always succeeds (`nothing` is never returned). +function cached_results(::Type{V}, job::CompilerJob) where {V} + # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive + # boxing allocation; the MI is kept alive by its method specializations. + key = (V, objectid(job.source), job.world, job.config) + Base.@lock job_results_lock begin + get!(job_results, key) do + V() + end::V + end end + + +## 1.10 session-dependent results +# +# Nothing to wipe: `job_results` never persists meaningfully across sessions (its +# `objectid`-based keys are session-specific, and our own image's entries are cleared +# in `__init__`; entries written by a downstream package's workload don't make it into +# that package's image at all, cf. cross-image mutation loss). +mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing + +end # !HAS_INTEGRATED_CACHE + + +## Legacy `cached_compilation` (1.10+) + +# A session-local, MI-keyed kernel cache modeled after the pre-CompilerCaching API. Kept +# only for back-ends that have not migrated to `cached_results`; migrated back-ends use +# that API on every Julia version, including its lightweight 1.10 implementation above. + +""" + cached_compilation(cache::AbstractDict, src::MethodInstance, cfg::CompilerConfig, + compiler, linker) + +Compile a method instance `src` with configuration `cfg`, by invoking `compiler` and +`linker` and storing the result in `cache`. + +The `cache` argument should be a dictionary that can be indexed using any value and store +whatever the `linker` function returns. The `compiler` function should take a +`CompilerJob` and return data that the `linker` function then turns into a session-local +artifact (e.g. a `CuModule`). + +This is the legacy caching API used before GPUCompiler 2.0. New code should use +[`cached_results`](@ref) and keep portable artifacts separate from session-local handles. +""" +function cached_compilation(cache::AbstractDict{<:Any,V}, + src::MethodInstance, cfg::CompilerConfig, + compiler::Function, linker::Function) where {V} + world = tls_world_age() + key = (objectid(src), world, cfg) + # NOTE: store the MethodInstance's objectid (not the MI) to avoid an expensive boxing + # allocation. Base does this with a multi-level lookup; we use a single-level dict. + + Base.@lock cached_compilation_lock begin + obj = get(cache, key, nothing) + end + + if obj === nothing || compile_hook[] !== nothing + obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V + Base.@lock cached_compilation_lock begin + cache[key] = obj + end + end + obj::V +end + +const cached_compilation_lock = ReentrantLock() + +@noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, + cfg::CompilerConfig, compiler::Function, + linker::Function) + job = CompilerJob(src, cfg, world) + asm = nothing + + # provide a hook to use in tools / debuggers + if compile_hook[] !== nothing + Base.invokelatest(compile_hook[], job) + end + + asm = compiler(job) + obj = linker(job, asm) + obj +end + +@public cached_compilation diff --git a/src/driver.jl b/src/driver.jl index 30529c70..8c52e884 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -42,42 +42,23 @@ end ## compiler entrypoint export compile - -# (::CompilerJob) -const compile_hook = Ref{Union{Nothing,Function}}(nothing) +# `compile_hook` is declared in `GPUCompiler.jl` so files included earlier +# (notably `deprecated.jl`'s `cached_compilation`) can reference it too. """ compile(target::Symbol, job::CompilerJob) Compile a `job` to one of the following formats as specified by the `target` argument: -`:julia` for Julia IR, `:llvm` for LLVM IR and `:asm` for machine code. +`:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. """ -function compile(target::Symbol, @nospecialize(job::CompilerJob); kwargs...) - # XXX: remove on next major version - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `compile` API does not take keyword arguments anymore. Use CompilerConfig instead.", :compile) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - +function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing Base.invokelatest(compile_hook[], job) end - return compile_unhooked(target, job) end -# XXX: remove on next major version -function codegen(output::Symbol, @nospecialize(job::CompilerJob); kwargs...) - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `codegen` function is an internal API. Use `GPUCompiler.compile` (with any kwargs passed to `CompilerConfig`) instead.", :codegen) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - compile_unhooked(output, job) -end - -function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob); kwargs...) +function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) if context(; throw_error=false) === nothing error("No active LLVM context. Use `JuliaContext()` do-block syntax to create one.") end @@ -188,14 +169,7 @@ end const __llvm_initialized = Ref(false) -@locked function emit_llvm(@nospecialize(job::CompilerJob); kwargs...) - # XXX: remove on next major version - if !isempty(kwargs) - safe_depwarn("The GPUCompiler `emit_llvm` function is an internal API. Use `GPUCompiler.compile` (with any kwargs passed to `CompilerConfig`) instead.", :emit_llvm) - config = CompilerConfig(job.config; kwargs...) - job = CompilerJob(job.source, config) - end - +@locked function emit_llvm(@nospecialize(job::CompilerJob)) if !__llvm_initialized[] InitializeAllTargets() InitializeAllTargetInfos() @@ -259,7 +233,7 @@ const __llvm_initialized = Ref(false) target = nest_target(dyn_job.config.target, job.config.target) params = nest_params(dyn_job.config.params, job.config.params) config = CompilerConfig(dyn_job.config; toplevel=false, target, params) - return codegen(:llvm, CompilerJob(dyn_job; config)) + return compile_unhooked(:llvm, CompilerJob(dyn_job; config)) end # compile and link @@ -323,23 +297,7 @@ const __llvm_initialized = Ref(false) @tracepoint "Library linking" begin # target-specific libraries - @tracepoint "target libraries" begin - if has_legacy_link_libraries(job) - safe_depwarn( - "3-arg `link_libraries!(job, mod, undefined_fns)` is deprecated; " * - "migrate your override to the 2-arg form `link_libraries!(job, mod)`. " * - "Instead of inspecting `undefined_fns` to decide what to link, " * - "parse the library lazily with `parse(LLVM.Module, bytes; lazy=true)` " * - "and link it with `LLVM.link!(mod, lib; only_needed=true)` — " * - "the linker will then materialize only the referenced symbols.", - :link_libraries!) - undefined_fns = [LLVM.name(f) for f in functions(ir) - if isdeclaration(f) && !LLVM.isintrinsic(f)] - link_libraries!(job, ir, undefined_fns) - else - link_libraries!(job, ir) - end - end + @tracepoint "target libraries" link_libraries!(job, ir) # GPU run-time library if !uses_julia_runtime(job) @@ -376,6 +334,12 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) + relocate_gvs!(ir, gv_to_value) + + # the IR (and any artifact derived from it) now embeds session-absolute + # addresses; keep results for this job out of a package image + isempty(gv_to_value) || mark_session_dependent!(job) + if job.config.optimize @tracepoint "optimization" begin optimize!(job, ir; job.config.opt_level) diff --git a/src/execution.jl b/src/execution.jl index 9b4940a7..dcdd4236 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -66,224 +66,3 @@ function assign_args!(code, _args) return vars, var_exprs end - - -## cached compilation - -### Notes on interactions with package images and disk cache. -# Julia uses package images (pkgimg) to cache both the result of inference, -# and the result of native code emissions. Up until Julia v1.11 neither the -# inferred nor the nativce code of foreign abstract interpreters was cached -# across sessions. Julia v1.11 allows for caching of inference results across -# sessions as long as those inference results are created during precompilation. -# -# Julia cache hierarchy is roughly as follows: -# Function (name of a thing) -# -> Method (particular piece of code to dispatch to with a signature) -# -> MethodInstance (A particular Method + particular signature) -# -> CodeInstance (A MethodInstance compiled for a world) -# -# In order to cache code across sessions we need to insert CodeInstance(owner=GPUCompilerCacheToken) -# into the internal cache. Once we have done so we know that a particular CodeInstance is unique in -# the system. (During pkgimg loading conflicts will be resolved). -# -# When a pkgimg is loaded we check it's validity, this means checking that all depdencies are the same, -# the pkgimg was created for the right set of compiler flags, and that all source code that was used -# to create this pkgimg is the same. When a CodeInstance is inside a pkgimg we can extend the chain of -# validity even for GPU code, we cannot verify a "runtime" CodeInstance in the same way. -# -# Therefore when we see a compilation request for a CodeInstance that is originating from a pkgimg -# we can use it as part of the hash for the on-disk cache. (see `cache_file`) - -""" - disk_cache_enabled() - -Query if caching to disk is enabled. -""" -disk_cache_enabled() = parse(Bool, @load_preference("disk_cache", "false")) - -""" - enable_disk_cache!(state::Bool=true) - -Activate the GPUCompiler disk cache in the current environment. -You will need to restart your Julia environment for it to take effect. - -!!! note - The cache functionality requires Julia 1.11 -""" -function enable_disk_cache!(state::Bool=true) - @set_preferences!("disk_cache"=>string(state)) -end - -disk_cache_path() = @get_scratch!("disk_cache") -clear_disk_cache!() = rm(disk_cache_path(); recursive=true, force=true) - -const cache_lock = ReentrantLock() - -""" - cached_compilation(cache::Dict{Any}, src::MethodInstance, cfg::CompilerConfig, - compiler, linker) - -Compile a method instance `src` with configuration `cfg`, by invoking `compiler` and -`linker` and storing the result in `cache`. - -The `cache` argument should be a dictionary that can be indexed using any value and store -whatever the `linker` function returns. The `compiler` function should take a `CompilerJob` -and return data that can be cached across sessions (e.g., LLVM IR). This data is then -forwarded, along with the `CompilerJob`, to the `linker` function which is allowed to create -session-dependent objects (e.g., a `CuModule`). -""" -function cached_compilation(cache::AbstractDict{<:Any,V}, - src::MethodInstance, cfg::CompilerConfig, - compiler::Function, linker::Function) where {V} - # NOTE: we index the cach both using (mi, world, cfg) keys, for the fast look-up, - # and using CodeInfo keys for the slow look-up. we need to cache both for - # performance, but cannot use a separate private cache for the ci->obj lookup - # (e.g. putting it next to the CodeInfo's in the CodeCache) because some clients - # expect to be able to wipe the cache (e.g. CUDA.jl's `device_reset!`) - - # fast path: index the cache directly for the *current* world + compiler config - - world = tls_world_age() - key = (objectid(src), world, cfg) - # NOTE: we store the MethodInstance's objectid to avoid an expensive allocation. - # Base does this with a multi-level lookup, first keyed on the mi, - # then a linear scan over the (typically few) entries. - - # NOTE: no use of lock(::Function)/@lock/get! to avoid try/catch and closure overhead - lock(cache_lock) - obj = get(cache, key, nothing) - unlock(cache_lock) - - if obj === nothing || compile_hook[] !== nothing - obj = actual_compilation(cache, src, world, cfg, compiler, linker)::V - lock(cache_lock) - cache[key] = obj - unlock(cache_lock) - end - return obj::V -end - -@noinline function cache_file(ci::CodeInstance, cfg::CompilerConfig) - h = hash(Base.objectid(ci)) - @static if isdefined(Base, :object_build_id) - bid = Base.object_build_id(ci) - if bid === nothing # CI is from a runtime compilation, not worth caching on disk - return nothing - else - bid = bid % UInt64 # The upper 64bit are a checksum, unavailable during precompilation - end - h = hash(bid, h) - end - h = hash(cfg, h) - - gpucompiler_buildid = Base.module_build_id(@__MODULE__) - if (gpucompiler_buildid >> 64) % UInt64 == 0xffffffffffffffff - return nothing # Don't cache during precompilation of GPUCompiler - end - - return joinpath( - disk_cache_path(), - # bifurcate the cache by build id of GPUCompiler - string(gpucompiler_buildid), - string(h, ".jls")) -end - -struct DiskCacheEntry - src::Type # Originally MethodInstance, but upon deserialize they were not uniqued... - cfg::CompilerConfig - asm -end - -@noinline function actual_compilation(cache::AbstractDict, src::MethodInstance, world::UInt, - cfg::CompilerConfig, compiler::Function, linker::Function) - job = CompilerJob(src, cfg, world) - obj = nothing - - # fast path: find an applicable CodeInstance and see if we have compiled it before - ci = ci_cache_lookup(ci_cache(job), src, world, world)::Union{Nothing,CodeInstance} - if ci !== nothing - key = (ci, cfg) - obj = get(cache, key, nothing) - end - - # slow path: compile and link - if obj === nothing || compile_hook[] !== nothing - asm = nothing - path = nothing - ondisk_hit = false - @static if VERSION >= v"1.11.0-" - # Don't try to hit the disk cache if we are for a *compile* hook - # TODO: - # - Sould we hit disk cache if Base.generating_output() - # - Should we allow backend to opt out? - if ci !== nothing && obj === nothing && disk_cache_enabled() - path = cache_file(ci, cfg) - @debug "Looking for on-disk cache" job path - if path !== nothing && isfile(path) - ondisk_hit = true - try - @debug "Loading compiled kernel" job path - # The MI we deserialize here didn't get uniqued... - entry = deserialize(path)::DiskCacheEntry - if entry.src == src.specTypes && entry.cfg == cfg - asm = entry.asm - else - @show entry.src == src.specTypes - @show entry.cfg == cfg - @warn "Cache missmatch" src.specTypes cfg entry.src entry.cfg - end - catch ex - @warn "Failed to load compiled kernel" job path exception=(ex, catch_backtrace()) - end - end - end - end - - if asm === nothing || compile_hook[] !== nothing - # Run the compiler in-case we need to hook it. - asm = compiler(job) - end - if obj !== nothing - # we got here because of a *compile* hook; don't bother linking - return obj - end - - @static if VERSION >= v"1.11.0-" - if !ondisk_hit && path !== nothing && disk_cache_enabled() - @debug "Writing out on-disk cache" job path - mkpath(dirname(path)) - entry = DiskCacheEntry(src.specTypes, cfg, asm) - - # atomic write to disk - tmppath, io = mktemp(dirname(path); cleanup=false) - serialize(io, entry) - close(io) - @static if VERSION >= v"1.12.0-DEV.1023" - mv(tmppath, path; force=true) - else - Base.rename(tmppath, path, force=true) - end - end - end - - obj = linker(job, asm) - - if ci === nothing - ci = ci_cache_lookup(ci_cache(job), src, world, world) - if ci === nothing - error("""Did not find CodeInstance for $job. - - Pleaase make sure that the `compiler` function passed to `cached_compilation` - invokes GPUCompiler with exactly the same configuration as passed to the API. - - Note that you should do this by calling `GPUCompiler.compile`, and not by - using reflection functions (which alter the compiler configuration).""") - end - key = (ci, cfg) - end - cache[key] = obj - end - - return obj -end diff --git a/src/gcn.jl b/src/gcn.jl index be047143..dc06c8b3 100644 --- a/src/gcn.jl +++ b/src/gcn.jl @@ -39,10 +39,6 @@ end ## job -# TODO: encode debug build or not in the compiler job -# https://github.com/JuliaGPU/CUDAnative.jl/issues/368 -runtime_slug(job::CompilerJob{GCNCompilerTarget}) = "gcn-$(job.config.target.dev_isa)$(job.config.target.features)-$(job.config.target.backend)" - const gcn_intrinsics = () # TODO: ("vprintf", "__assertfail", "malloc", "free") isintrinsic(::CompilerJob{GCNCompilerTarget}, fn::String) = in(fn, gcn_intrinsics) diff --git a/src/interface.jl b/src/interface.jl index 6aec9f1e..0882229a 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -66,12 +66,52 @@ abstract type AbstractCompilerParams end nest_params(params::AbstractCompilerParams, parent::AbstractCompilerParams) = params +## cache owner + +# Inference results are shared by jobs with the same target, params, and inlining policy. +# On 1.11+, store this token on the CompilerConfig (as `Any`) so the immutable value is +# boxed once when the config is created, instead of on every integrated-cache lookup. +struct GPUCompilerCacheToken{T<:AbstractCompilerTarget, P<:AbstractCompilerParams} + target::T + params::P + always_inline::Bool +end + +""" + cache_owner(target, params, always_inline) + +Construct the immutable token that partitions inference for a compiler configuration. On +Julia 1.11+, the job-level `cache_owner(job)` returns the pre-boxed token stored on its +`CompilerConfig`. + +The default token covers the full `target` and `params` instances plus `always_inline`. +That is sufficient because the inference inputs derived from a job — `method_table`, +`method_table_view`, `inference_params` and `optimization_params` — must be pure +functions of those values, so back-ends normally leave this untouched. + +When overriding, the returned value must match under `===`/`jl_egal` after package-image +deserialization: use immutable containers, and only reference mutable objects (like method +tables) that are module-level singletons. +""" +cache_owner(target::AbstractCompilerTarget, params::AbstractCompilerParams, + always_inline::Bool) = + GPUCompilerCacheToken(target, params, always_inline) + + ## config export CompilerConfig # the configuration of the compiler +# Keep the 1.10 layout concrete: an `Any` field makes otherwise-inline configs escape when +# used in the legacy results key. Integrated caching needs the pre-boxed owner on 1.11+. +const CompilerConfigCacheOwner = @static if HAS_INTEGRATED_CACHE + Any +else + Nothing +end + const CONFIG_KWARGS = [:kernel, :name, :entry_abi, :always_inline, :opt_level, :libraries, :optimize, :cleanup, :validate, :strip] @@ -128,6 +168,9 @@ struct CompilerConfig{T,P} toplevel::Bool only_entry::Bool + # Boxed once so calls into Julia's Any-typed integrated cache don't allocate on every hit. + cache_owner::CompilerConfigCacheOwner + function CompilerConfig(target::AbstractCompilerTarget, params::AbstractCompilerParams; kernel=true, name=nothing, entry_abi=:specfunc, toplevel=true, always_inline=false, opt_level=2, @@ -137,10 +180,15 @@ struct CompilerConfig{T,P} if entry_abi ∉ (:specfunc, :func) error("Unknown entry_abi=$entry_abi") end + owner = @static if HAS_INTEGRATED_CACHE + cache_owner(target, params, always_inline) + else + nothing + end new{typeof(target), typeof(params)}(target, params, kernel, name, entry_abi, always_inline, opt_level, debug_level, libraries, optimize, cleanup, validate, strip, toplevel, - only_entry) + only_entry, owner) end end @@ -187,7 +235,9 @@ function Base.hash(cfg::CompilerConfig, h::UInt) h = hash(cfg.strip, h) h = hash(cfg.toplevel, h) h = hash(cfg.only_entry, h) - + # `cache_owner` deliberately is not hashed: its target/params/inlining components were + # already included above, and a method-table-only difference may safely collide. Hashing + # the full token here would duplicate the most expensive parts of every config hash. return h end @@ -261,17 +311,23 @@ runtime_module(@nospecialize(job::CompilerJob)) = error("Not implemented") isintrinsic(@nospecialize(job::CompilerJob), fn::String) = false # provide a specific interpreter to use. -if VERSION >= v"1.11.0-DEV.1552" -get_interpreter(@nospecialize(job::CompilerJob)) = - GPUInterpreter(job.world; method_table_view=maybe_cached(method_table_view(job)), - token=ci_cache_token(job), inf_params=inference_params(job), +@static if HAS_INTEGRATED_CACHE +function get_interpreter(@nospecialize(job::CompilerJob)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + owner=cache_owner(job), + inf_params=inference_params(job), opt_params=optimization_params(job)) +end else -get_interpreter(@nospecialize(job::CompilerJob)) = - GPUInterpreter(job.world; method_table_view=maybe_cached(method_table_view(job)), - code_cache=ci_cache(job), inf_params=inference_params(job), +function get_interpreter(@nospecialize(job::CompilerJob)) + GPUInterpreter(job.world; + method_table_view=maybe_cached(method_table_view(job)), + code_cache=get_code_cache(job), + inf_params=inference_params(job), opt_params=optimization_params(job)) end +end # does this target support throwing Julia exceptions with jl_throw? # if not, calls to throw will be replaced with calls to the GPU runtime @@ -281,11 +337,6 @@ can_throw(@nospecialize(job::CompilerJob)) = uses_julia_runtime(job) # if not, safepoints at function entry will not be emitted can_safepoint(@nospecialize(job::CompilerJob)) = uses_julia_runtime(job) -# generate a string that represents the type of compilation, for selecting a compiled -# instance of the runtime library. this slug should encode everything that affects -# the generated code of this compiler job (with exception of the function source) -runtime_slug(@nospecialize(job::CompilerJob)) = error("Not implemented") - # the type of the kernel state object, or Nothing if this back-end doesn't need one. # # the generated code will be rewritten to include an object of this type as the first @@ -305,37 +356,206 @@ pass_by_ref(@nospecialize(job::CompilerJob)) = false # whether pointer is a valid call target valid_function_pointer(@nospecialize(job::CompilerJob), ptr::Ptr{Cvoid}) = false -# Care is required for anything that impacts: -# - method_table -# - inference_params -# - optimization_params -# By default that is just always_inline -# the cache token is compared with jl_egal -struct GPUCompilerCacheToken - target_type::Type - always_inline::Bool - method_table::Core.MethodTable +# Cache partitioning. On Julia 1.11+, the owner is stored on every `CodeInstance` and +# compared via `jl_egal`; custom `target` / `params` containers must be immutable so +# equivalent values can match across package-image deserialization. +# +# On Julia 1.10, where there's no per-CI owner field, this token only identifies the +# session-local `GLOBAL_CI_CACHES` partition in `deprecated.jl`; the immutability +# requirement is still useful for stable hashing. +# +# The token is created and boxed by the CompilerConfig constructor. Returning the boxed field +# is important: passing a freshly-constructed immutable token to the Any-typed C cache API +# allocates on every kernel-cache hit. +@static if HAS_INTEGRATED_CACHE + cache_owner(job::CompilerJob) = job.config.cache_owner +else + # Preserve the allocation-free 1.10 path: constructing this specialized immutable token + # is cheaper than storing an Any-typed box on every CompilerConfig. + cache_owner(job::CompilerJob) = + cache_owner(job.config.target, job.config.params, job.config.always_inline) end -ci_cache_token(@nospecialize(job::CompilerJob)) = - GPUCompilerCacheToken(typeof(job.config.target), job.config.always_inline, method_table(job)) +""" + cached_results(::Type{V}, job::CompilerJob) -> Union{Nothing,V} + +Retrieve the compilation-results struct of type `V` for `job`. Returns `nothing` when no +code has been compiled (or inferred) for the job yet; once code exists, an empty `V` is +created on first access and the same struct is returned ever after. `V` must be a +`mutable struct` with a zero-arg constructor; back-ends define one such struct holding +the artifacts of each compilation stage, check whether the relevant fields are +populated, and fill them in (running the compiler) when not: + +```julia +mutable struct MetalResults + metallib::Union{Nothing,Vector{UInt8}} + entry::Union{Nothing,String} + pipelines::Vector{Tuple{MTLDevice,MTLComputePipelineState}} # session-local + MetalResults() = new(nothing, nothing, []) +end -# the codeinstance cache to use -- should only be used for the constructor -if VERSION >= v"1.11.0-DEV.1552" - # Soft deprecated user should use `CC.code_cache(get_interpreter(job))` - ci_cache(@nospecialize(job::CompilerJob)) = CC.code_cache(get_interpreter(job)) -else -function ci_cache(@nospecialize(job::CompilerJob)) - lock(GLOBAL_CI_CACHES_LOCK) do - cache = get!(GLOBAL_CI_CACHES, job.config) do - CodeCache() +res = GPUCompiler.cached_results(MetalResults, job) +if res === nothing || res.metallib === nothing || GPUCompiler.compile_hook[] !== nothing + artifacts = ...compile... + res = @something res GPUCompiler.cached_results(MetalResults, job) + ...populate res from artifacts... +end +``` + +Compiling the job (through `GPUCompiler.compile`) populates Julia's code cache, so the +post-compile lookup in the example is guaranteed to succeed. To attach results without +generating code — e.g. from an inference-only precompilation workload — run +`precompile(job)` first. + +Results are keyed by the *full* compiler job: its method instance, world age, and entire +`CompilerConfig` (two jobs differing only in, say, the kernel `name` get distinct +structs). Storage differs per Julia version, transparently to the back-end: + +- On Julia 1.11+, the struct lives on the `CodeInstance`'s `analysis_results` chain in + Julia's integrated code cache, partitioned by [`cache_owner`](@ref) and keyed by + config within the per-CI [`JobResults`](@ref) container. Method redefinition + invalidates the CI — and with it the attached results. Artifacts stored during + precompilation are serialized into the package image along with the CI: populate only + session-portable values (bytes, strings) during precompile workloads, and keep + session-local handles (device modules, pipeline objects) in fields that remain empty + until first use at run time. + +- On Julia 1.10, the struct lives in a session-local Dict keyed by the job, so the + lookup always succeeds (`nothing` is never returned); redefinition protection comes + from the world age in the key, and nothing persists across sessions. + +Thread safety: concurrent calls for the same job return the same struct, but +GPUCompiler does not serialize back-end *compilation*; take a back-end lock around +the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock`). +""" +function cached_results end + +@static if HAS_INTEGRATED_CACHE + +""" + JobResults + +Per-CodeInstance container mapping a `CompilerConfig` to the back-end's results struct. + +A `CodeInstance` is shared by every job whose [`cache_owner`](@ref) matches — the owner +token only covers what affects *inference* (target, params, `always_inline`). The +remaining `CompilerConfig` fields (`kernel`, `name`, `entry_abi`, +`opt_level`, …) do affect *codegen*, so artifacts must not be shared across them. +Entries are matched by `===` (`jl_egal`): `CompilerConfig` is an immutable struct, so +structurally identical configs compare equal, including against configs deserialized +from package images. +""" +# `CompilerConfig` is an abstract UnionAll here. Keeping it in a tuple stored inline in a +# Vector boxes the config again on every iteration. A non-isbits entry object is stored by +# reference, so both fields are boxed once and hot-path scans allocate nothing. +struct JobResultEntry + config::CompilerConfig + value::Any +end + +mutable struct JobResults + entries::Vector{JobResultEntry} + JobResults() = new(JobResultEntry[]) +end + +const cached_results_lock = ReentrantLock() + +# NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of +# instantiations: one per back-end and results type). +function job_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} + jr = CompilerCaching.results(JobResults, ci) + Base.@lock cached_results_lock begin + for entry in jr.entries + entry.config === config && entry.value isa V && return entry.value::V end - return cache + v = V() + push!(jr.entries, JobResultEntry(config, v)) + return v end end + +function cache_view(@nospecialize(job::CompilerJob)) + # `cache_owner` is deliberately stored as Any on CompilerConfig: preserve that box in the + # CacheView instead of re-specializing and re-boxing the immutable token for the ccall. + CompilerCaching.CacheView{Any,JobResults}(cache_owner(job), job.world) +end + +# Fetch the CodeInstance backing `job` from the integrated cache. On 1.14+, inference +# caches vararg methods under their compilable (vararg-widened) MethodInstance, which +# for such methods differs from the fully-specialized `job.source`: retry there. +function job_code_instance(@nospecialize(job::CompilerJob)) + cache = cache_view(job) + ci = get(cache, job.source, nothing) + @static if VERSION >= v"1.14-" + if ci === nothing + mi = ccall(:jl_normalize_to_compilable_mi, Any, (Any,), + job.source)::MethodInstance + mi === job.source || (ci = get(cache, mi, nothing)) + end + end + return ci end +# Deliberately lookup-only: when no CI exists, back-ends run codegen (which drives +# inference itself) and re-fetch, instead of us running a standalone inference walk here +# that the compiler's own walk would immediately repeat. +function cached_results(::Type{V}, job::CompilerJob) where {V} + ci = job_code_instance(job) + ci === nothing && return nothing + return job_results(V, ci, job.config) +end + +## session-dependent results +# +# Some compilation results embed session-specific data: `relocate_gvs!` bakes absolute +# pointers into the IR of toplevel jobs that reference `julia.constgv` globals, and any +# artifact a back-end derives from that IR (metallib, SPIR-V, ...) inherits them. Such +# results must not survive into a package image, while remaining available for +# within-session lookups during the precompilation process itself. Julia wipes its own +# session-dependent CodeInstance state during serialization (staticdata.c); we +# approximate that with an `atexit` hook, which the runtime invokes *before* +# `jl_write_compiler_output`: right before the image is written, the entries of jobs +# marked session-dependent are deleted from their `JobResults` container, so a later +# session simply recompiles them. + +const session_dependent_jobs = Vector{CompilerJob}() +const session_dependent_lock = ReentrantLock() + +function mark_session_dependent!(@nospecialize(job::CompilerJob)) + ccall(:jl_generating_output, Cint, ()) == 1 || return + Base.@lock session_dependent_lock begin + if isempty(session_dependent_jobs) + atexit(wipe_session_dependent_results) + end + push!(session_dependent_jobs, job) + end + return +end + +function wipe_session_dependent_results() + Base.@lock session_dependent_lock begin + for job in session_dependent_jobs + ci = job_code_instance(job) + ci === nothing && continue + jr = CompilerCaching.results(JobResults, ci) + Base.@lock cached_results_lock begin + filter!(entry -> entry.config !== job.config, jr.entries) + end + end + empty!(session_dependent_jobs) + end + return +end + +end # HAS_INTEGRATED_CACHE + +@public GPUCompilerCacheToken, cache_owner, cached_results + # the method table to use +# +# NOTE: these (like `inference_params` and `optimization_params` below) may only depend on +# the job's world and its config's `target`/`params` values (+ `always_inline`); +# [`cache_owner`](@ref) relies on that to partition inference results correctly. # deprecate method_table on next-breaking release method_table(@nospecialize(job::CompilerJob)) = GLOBAL_METHOD_TABLE method_table_view(@nospecialize(job::CompilerJob)) = get_method_table_view(job.world, method_table(job)) @@ -401,9 +621,3 @@ finish_ir!(@nospecialize(job::CompilerJob), mod::LLVM.Module, entry::LLVM.Functi # whether an LLVM function is valid for this back-end validate_ir(@nospecialize(job::CompilerJob), mod::LLVM.Module) = IRError[] - -# deprecated -struct DeprecationMarker end -process_module!(@nospecialize(job::CompilerJob), mod::LLVM.Module) = DeprecationMarker() -process_entry!(@nospecialize(job::CompilerJob), mod::LLVM.Module, entry::LLVM.Function) = - DeprecationMarker() diff --git a/src/irgen.jl b/src/irgen.jl index 3df1d659..9129e20c 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -43,11 +43,6 @@ function irgen(@nospecialize(job::CompilerJob)) end end - deprecation_marker = process_module!(job, mod) - if deprecation_marker != DeprecationMarker() - safe_depwarn("GPUCompiler.process_module! is deprecated; implement GPUCompiler.finish_module! instead", :process_module) - end - # sanitize global values (Julia doesn't when using the external codegen policy) for val in [collect(globals(mod)); collect(functions(mod))] isdeclaration(val) && continue @@ -69,11 +64,6 @@ function irgen(@nospecialize(job::CompilerJob)) elseif job.config.kernel LLVM.name!(entry, mangle_sig(job.source.specTypes)) end - deprecation_marker = process_entry!(job, mod, entry) - if deprecation_marker != DeprecationMarker() - safe_depwarn("GPUCompiler.process_entry! is deprecated; implement GPUCompiler.finish_module! instead", :process_entry) - entry = deprecation_marker - end if job.config.entry_abi === :specfunc func = compiled[job.source].func specfunc = LLVM.name(entry) diff --git a/src/jlgen.jl b/src/jlgen.jl index 8db65c89..8a6eab48 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -3,15 +3,13 @@ ## world age lookups -# `tls_world_age` should be used to look up the current world age. in most cases, this is -# what you should use to invoke the compiler with. - -if isdefined(Base, :tls_world_age) +@static if isdefined(Base, :tls_world_age) import Base: tls_world_age else tls_world_age() = ccall(:jl_get_tls_world_age, UInt, ()) end + ## looking up method instances export methodinstance, generic_methodinstance @@ -38,11 +36,6 @@ function MethodError(ft::Type{<:Function}, tt::Type, world::Integer=typemax(UInt end MethodError(ft, tt, world=typemax(UInt)) = Base.MethodError(ft, tt, world) -# generate a LineInfoNode for the current source code location -macro LineInfoNode(method) - Core.LineInfoNode(__module__, method, __source__.file, Int32(__source__.line), Int32(0)) -end - """ methodinstance(ft::Type, tt::Type, [world::UInt]) @@ -56,7 +49,6 @@ This function is highly optimized, and results do not need to be cached addition Only use this function with concrete signatures, i.e., using the types of values you would pass at run time. For non-concrete signatures, use `generic_methodinstance` instead. - """ methodinstance @@ -72,11 +64,10 @@ function generic_methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type) return mi::MethodInstance end -# on 1.11 (JuliaLang/julia#52572, merged as part of JuliaLang/julia#52233) we can use -# Julia's cached method lookup to simply look up method instances at run time. +# On 1.11+ (JuliaLang/julia#52572 / JuliaLang/julia#52233) `jl_method_lookup_by_tt` returns +# a usable `MethodInstance` directly, so we just call it. @static if VERSION >= v"1.11.0-DEV.1552" -# XXX: version of Base.method_instance that uses a function type @inline function methodinstance(@nospecialize(ft::Type), @nospecialize(tt::Type), world::Integer=tls_world_age()) sig = signature_type_by_tt(ft, tt) @@ -96,294 +87,23 @@ end return mi end -# on older versions of Julia, we always need to use the generic lookup +# On 1.10 we always use the generic lookup. (A @generated fast path used to exist, +# but it was unreachable: the (ft::Type, tt::Type) method above it always won dispatch.) else - const methodinstance = generic_methodinstance - -function methodinstance_generator(world::UInt, source, self, ft::Type, tt::Type) - @nospecialize - @assert CC.isType(ft) && CC.isType(tt) - ft = ft.parameters[1] - tt = tt.parameters[1] - - stub = Core.GeneratedFunctionStub(identity, Core.svec(:methodinstance, :ft, :tt), Core.svec()) - - # look up the method match - method_error = :(throw(MethodError(ft, tt, $world))) - sig = Tuple{ft, tt.parameters...} - min_world = Ref{UInt}(typemin(UInt)) - max_world = Ref{UInt}(typemax(UInt)) - match = ccall(:jl_gf_invoke_lookup_worlds, Any, - (Any, Any, Csize_t, Ref{Csize_t}, Ref{Csize_t}), - sig, #=mt=# nothing, world, min_world, max_world) - match === nothing && return stub(world, source, method_error) - - # look up the method and code instance - mi = ccall(:jl_specializations_get_linfo, Ref{MethodInstance}, - (Any, Any, Any), match.method, match.spec_types, match.sparams) - ci = CC.retrieve_code_info(mi, world) - - # prepare a new code info - new_ci = copy(ci) - empty!(new_ci.code) - empty!(new_ci.codelocs) - empty!(new_ci.linetable) - empty!(new_ci.ssaflags) - new_ci.ssavaluetypes = 0 - - # propagate edge metadata - new_ci.min_world = min_world[] - new_ci.max_world = max_world[] - new_ci.edges = Any[mi] - - # prepare the slots - new_ci.slotnames = Symbol[Symbol("#self#"), :ft, :tt] - new_ci.slotflags = UInt8[0x00 for i = 1:3] - - # return the method instance - push!(new_ci.code, CC.ReturnNode(mi)) - push!(new_ci.ssaflags, 0x00) - push!(new_ci.linetable, @LineInfoNode(methodinstance)) - push!(new_ci.codelocs, 1) - new_ci.ssavaluetypes += 1 - - return new_ci -end - -@eval function methodinstance(ft, tt) - $(Expr(:meta, :generated_only)) - $(Expr(:meta, :generated, methodinstance_generator)) -end - -end - - -## code instance cache -const HAS_INTEGRATED_CACHE = VERSION >= v"1.11.0-DEV.1552" - -if !HAS_INTEGRATED_CACHE -struct CodeCache - dict::IdDict{MethodInstance,Vector{CodeInstance}} - - CodeCache() = new(IdDict{MethodInstance,Vector{CodeInstance}}()) -end - -function Base.show(io::IO, ::MIME"text/plain", cc::CodeCache) - print(io, "CodeCache with $(mapreduce(length, +, values(cc.dict); init=0)) entries") - if !isempty(cc.dict) - print(io, ": ") - for (mi, cis) in cc.dict - println(io) - print(io, " ") - show(io, mi) - - function worldstr(min_world, max_world) - if min_world == typemax(UInt) - "empty world range" - elseif max_world == typemax(UInt) - "worlds $(Int(min_world))+" - else - "worlds $(Int(min_world)) to $(Int(max_world))" - end - end - - for (i,ci) in enumerate(cis) - println(io) - print(io, " CodeInstance for ", worldstr(ci.min_world, ci.max_world)) - end - end - end -end - -Base.empty!(cc::CodeCache) = empty!(cc.dict) - -const GLOBAL_CI_CACHES = Dict{CompilerConfig, CodeCache}() -const GLOBAL_CI_CACHES_LOCK = ReentrantLock() - - -## method invalidations - -function CC.setindex!(cache::CodeCache, ci::CodeInstance, mi::MethodInstance) - # make sure the invalidation callback is attached to the method instance - add_codecache_callback!(cache, mi) - cis = get!(cache.dict, mi, CodeInstance[]) - push!(cis, ci) -end - -# invalidation (like invalidate_method_instance, but for our cache) -struct CodeCacheCallback - cache::CodeCache -end - -@static if VERSION ≥ v"1.11.0-DEV.798" - -function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) - callback = CodeCacheCallback(cache) - CC.add_invalidation_callback!(callback, mi) -end -function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32) - cis = get(callback.cache.dict, replaced, nothing) - if cis === nothing - return - end - for ci in cis - if ci.max_world == ~0 % Csize_t - @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" -@static if VERSION >= v"1.11.0-DEV.1390" - @atomic ci.max_world = max_world -else - ci.max_world = max_world -end - end - @assert ci.max_world <= max_world - end -end - -else - -function add_codecache_callback!(cache::CodeCache, mi::MethodInstance) - callback = CodeCacheCallback(cache) - if !isdefined(mi, :callbacks) - mi.callbacks = Any[callback] - elseif !in(callback, mi.callbacks) - push!(mi.callbacks, callback) - end -end -function (callback::CodeCacheCallback)(replaced::MethodInstance, max_world::UInt32, - seen::Set{MethodInstance}=Set{MethodInstance}()) - push!(seen, replaced) - - cis = get(callback.cache.dict, replaced, nothing) - if cis === nothing - return - end - for ci in cis - if ci.max_world == ~0 % Csize_t - @assert ci.min_world - 1 <= max_world "attempting to set illogical constraints" - ci.max_world = max_world - end - @assert ci.max_world <= max_world - end - - # recurse to all backedges to update their valid range also - if isdefined(replaced, :backedges) - backedges = filter(replaced.backedges) do @nospecialize(mi) - if mi isa MethodInstance - mi ∉ seen - elseif mi isa Type - # an `invoke` call, which is a `(sig, MethodInstance)` pair. - # let's ignore the `sig` and process the `MethodInstance` next. - false - else - error("invalid backedge") - end - end - - # Don't touch/empty backedges `invalidate_method_instance` in C will do that later - # replaced.backedges = Any[] - - for mi in backedges - callback(mi::MethodInstance, max_world, seen) - end - end end -end -end # !HAS_INTEGRATED_CACHE - ## method overrides Base.Experimental.@MethodTable(GLOBAL_METHOD_TABLE) -# Implements a priority lookup for method tables, where the first match in the stack get's returned. -# An alternative to this would be to use a "Union" where we would query the parent method table and -# do a most-specific match. -struct StackedMethodTable{MTV<:CC.MethodTableView} <: CC.MethodTableView - world::UInt - mt::Core.MethodTable - parent::MTV +# Priority-lookup method table. On 1.11+ we re-export CompilerCaching's; the 1.10 copy +# (with the older `MethodMatchResult`-returning `findall` signature) lives in deprecated.jl. +@static if HAS_INTEGRATED_CACHE + using CompilerCaching: StackedMethodTable end -StackedMethodTable(world::UInt, mt::Core.MethodTable) = StackedMethodTable(world, mt, CC.InternalMethodTable(world)) -StackedMethodTable(world::UInt, mt::Core.MethodTable, parent::Core.MethodTable) = StackedMethodTable(world, mt, StackedMethodTable(world, parent)) - -CC.isoverlayed(::StackedMethodTable) = true - -@static if VERSION >= v"1.11.0-DEV.363" - # https://github.com/JuliaLang/julia/pull/51078 - # same API as before but without returning isoverlayed flag - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return result - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodLookupResult} - parent_result === nothing && return nothing #too many matches - # merge the parent match results with the internal method table - return CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig) - end - - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds - parent_match, parent_valid_worlds = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)) - ) - end -else - function CC.findall(@nospecialize(sig::Type), table::StackedMethodTable; limit::Int=-1) - result = CC._findall(sig, table.mt, table.world, limit) - result === nothing && return nothing # to many matches - nr = CC.length(result) - if nr ≥ 1 && CC.getindex(result, nr).fully_covers - # no need to fall back to the parent method view - return CC.MethodMatchResult(result, true) - end - - parent_result = CC.findall(sig, table.parent; limit)::Union{Nothing, CC.MethodMatchResult} - parent_result === nothing && return nothing #too many matches - - overlayed = parent_result.overlayed | !CC.isempty(result) - parent_result = parent_result.matches::CC.MethodLookupResult - - # merge the parent match results with the internal method table - return CC.MethodMatchResult( - CC.MethodLookupResult( - CC.vcat(result.matches, parent_result.matches), - CC.WorldRange( - CC.max(result.valid_worlds.min_world, parent_result.valid_worlds.min_world), - CC.min(result.valid_worlds.max_world, parent_result.valid_worlds.max_world)), - result.ambig | parent_result.ambig), - overlayed) - end - - function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) - match, valid_worlds = CC._findsup(sig, table.mt, table.world) - match !== nothing && return match, valid_worlds, true - parent_match, parent_valid_worlds, overlayed = CC.findsup(sig, table.parent) - return ( - parent_match, - CC.WorldRange( - max(valid_worlds.min_world, parent_valid_worlds.min_world), - min(valid_worlds.max_world, parent_valid_worlds.max_world)), - overlayed) - end -end ## interpreter @@ -394,7 +114,6 @@ else import Core.Compiler: get_world_counter, get_world_counter as get_inference_world end -const MTType = Core.MethodTable if isdefined(Core.Compiler, :CachedMethodTable) using Core.Compiler: CachedMethodTable maybe_cached(mtv::CC.MethodTableView) = CachedMethodTable(mtv) @@ -407,17 +126,28 @@ get_method_table_view(world::UInt, mt::CC.MethodTable) = CC.OverlayMethodTable(w # VERSION >= v"1.14.0-DEV.1691" const INFERENCE_CACHE_TYPE = isdefined(CC, :InferenceCache) ? CC.InferenceCache : Vector{CC.InferenceResult} +""" + GPUInterpreter + +Foreign abstract interpreter that drives Julia inference for GPU compilation. + +The interpreter is partitioned by an `owner` token (1.11+) or an in-process +`CodeCache` (1.10, see [`deprecated.jl`](deprecated.jl)). The owner is what +makes per-job CIs distinct in the integrated cache; together with the world +age it is all CompilerCaching needs to drive inference into the integrated +cache. Compilation results are not attached during inference at all — they +are attached lazily when looked up (see [`cached_results`](@ref)). +""" struct GPUInterpreter{MTV<:CC.MethodTableView} <: CC.AbstractInterpreter world::UInt method_table_view::MTV @static if HAS_INTEGRATED_CACHE - token::Any + owner::Any else code_cache::CodeCache end inf_cache::INFERENCE_CACHE_TYPE - inf_params::CC.InferenceParams opt_params::CC.OptimizationParams end @@ -425,31 +155,30 @@ end @static if HAS_INTEGRATED_CACHE function GPUInterpreter(world::UInt=Base.get_world_counter(); method_table_view::CC.MethodTableView, - token::Any, + owner::Any, inf_params::CC.InferenceParams, opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - - inf_cache = INFERENCE_CACHE_TYPE() - - return GPUInterpreter(world, method_table_view, - token, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, owner, INFERENCE_CACHE_TYPE(), + inf_params, opt_params) end function GPUInterpreter(interp::GPUInterpreter; world::UInt=interp.world, method_table_view::CC.MethodTableView=interp.method_table_view, - token::Any=interp.token, + owner::Any=interp.owner, inf_cache::INFERENCE_CACHE_TYPE=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, opt_params::CC.OptimizationParams=interp.opt_params) - return GPUInterpreter(world, method_table_view, - token, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, owner, inf_cache, + inf_params, opt_params) end -else +CC.cache_owner(interp::GPUInterpreter) = interp.owner + +else # 1.10: in-process CodeCache function GPUInterpreter(world::UInt=Base.get_world_counter(); method_table_view::CC.MethodTableView, @@ -457,12 +186,9 @@ function GPUInterpreter(world::UInt=Base.get_world_counter(); inf_params::CC.InferenceParams, opt_params::CC.OptimizationParams) @assert world <= Base.get_world_counter() - - inf_cache = Vector{CC.InferenceResult}() - - return GPUInterpreter(world, method_table_view, - code_cache, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, code_cache, Vector{CC.InferenceResult}(), + inf_params, opt_params) end function GPUInterpreter(interp::GPUInterpreter; @@ -472,21 +198,19 @@ function GPUInterpreter(interp::GPUInterpreter; inf_cache::Vector{CC.InferenceResult}=interp.inf_cache, inf_params::CC.InferenceParams=interp.inf_params, opt_params::CC.OptimizationParams=interp.opt_params) - return GPUInterpreter(world, method_table_view, - code_cache, inf_cache, - inf_params, opt_params) + return GPUInterpreter{typeof(method_table_view)}( + world, method_table_view, code_cache, inf_cache, + inf_params, opt_params) end + +CC.code_cache(interp::GPUInterpreter) = WorldView(interp.code_cache, interp.world) + end # HAS_INTEGRATED_CACHE CC.InferenceParams(interp::GPUInterpreter) = interp.inf_params CC.OptimizationParams(interp::GPUInterpreter) = interp.opt_params #=CC.=#get_inference_world(interp::GPUInterpreter) = interp.world CC.get_inference_cache(interp::GPUInterpreter) = interp.inf_cache -@static if HAS_INTEGRATED_CACHE - CC.cache_owner(interp::GPUInterpreter) = interp.token -else - CC.code_cache(interp::GPUInterpreter) = WorldView(interp.code_cache, interp.world) -end # No need to do any locking since we're not putting our results into the runtime cache CC.lock_mi_inference(interp::GPUInterpreter, mi::MethodInstance) = nothing @@ -507,8 +231,6 @@ CC.method_table(interp::GPUInterpreter) = interp.method_table_view # semi-concrete interepretation is broken with overlays (JuliaLang/julia#47349) function CC.concrete_eval_eligible(interp::GPUInterpreter, @nospecialize(f), result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) - # NOTE it's fine to skip overloading with `sv::IRInterpretationState` since we disables - # semi-concrete interpretation anyway. ret = @invoke CC.concrete_eval_eligible(interp::CC.AbstractInterpreter, f::Any, result::CC.MethodCallResult, arginfo::CC.ArgInfo, sv::CC.InferenceState) if ret === :semi_concrete_eval @@ -525,163 +247,46 @@ function CC.concrete_eval_eligible(interp::GPUInterpreter, end -## world view of the cache -@static if VERSION < v"1.14-" - using Core.Compiler: WorldView -end - -if !HAS_INTEGRATED_CACHE - -function CC.haskey(wvc::WorldView{CodeCache}, mi::MethodInstance) - CC.get(wvc, mi, nothing) !== nothing -end - -function CC.get(wvc::WorldView{CodeCache}, mi::MethodInstance, default) - # check the cache - for ci in get!(wvc.cache.dict, mi, CodeInstance[]) - if ci.min_world <= wvc.worlds.min_world && wvc.worlds.max_world <= ci.max_world - # TODO: if (code && (code == jl_nothing || jl_ir_flag_inferred((jl_array_t*)code))) - src = if ci.inferred isa Vector{UInt8} - ccall(:jl_uncompress_ir, Any, (Any, Ptr{Cvoid}, Any), - mi.def, C_NULL, ci.inferred) - else - ci.inferred - end - return ci - end - end - - return default -end +## driving inference and walking callees -function CC.getindex(wvc::WorldView{CodeCache}, mi::MethodInstance) - r = CC.get(wvc, mi, nothing) - r === nothing && throw(KeyError(mi)) - return r::CodeInstance -end - -function CC.setindex!(wvc::WorldView{CodeCache}, ci::CodeInstance, mi::MethodInstance) - CC.setindex!(wvc.cache, ci, mi) -end - -end # HAS_INTEGRATED_CACHE - -## codegen/inference integration - -function ci_cache_populate(interp, cache, mi, min_world, max_world) - codeinfos = Pair{CodeInstance, CodeInfo}[] - @static if VERSION >= v"1.12.0-DEV.1434" - # see typeinfer.jl: typeinf_ext_toplevel - has_compilequeue = VERSION >= v"1.13.0-DEV.499" || v"1.12-beta3" <= VERSION < v"1.13-" - ci = CC.typeinf_ext(interp, mi, CC.SOURCE_MODE_NOT_REQUIRED) - if has_compilequeue - workqueue = CC.CompilationQueue(; interp) - push!(workqueue, ci) - else - workqueue = CodeInstance[ci] - inspected = IdSet{CodeInstance}() - end - while !isempty(workqueue) - callee = pop!(workqueue) - if has_compilequeue - CC.isinspected(workqueue, callee) && continue - CC.markinspected!(workqueue, callee) - else - callee in inspected && continue - push!(inspected, callee) - end - - # now make sure everything has source code, if desired - mi = CC.get_ci_mi(callee) - if CC.use_const_api(callee) - if VERSION >= v"1.13.0-DEV.1121" - src = CC.codeinfo_for_const(interp, mi, CC.WorldRange(callee.min_world, callee.max_world), callee.edges, callee.rettype_const) - else - src = CC.codeinfo_for_const(interp, mi, callee.rettype_const) - end - else - # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here - src = CC.typeinf_code(interp, mi, true) - end - if src isa CodeInfo - if has_compilequeue - sptypes = CC.sptypes_from_meth_instance(mi) - CC.collectinvokes!(workqueue, src, sptypes) - else - CC.collectinvokes!(workqueue, src) - end - push!(codeinfos, callee => src) - end - end - elseif VERSION >= v"1.12.0-DEV.15" - inferred_ci = CC.typeinf_ext_toplevel(interp, mi, CC.SOURCE_MODE_FORCE_SOURCE) - @assert inferred_ci !== nothing "Inference of $mi failed" - - # inference should have populated our cache - wvc = WorldView(cache, min_world, max_world) - @assert CC.haskey(wvc, mi) "GPUCompiler: Failed to compile method for $mi, between worlds $min_world and $max_world" - ci = CC.getindex(wvc, mi) - - # if ci is rettype_const, the inference result won't have been cached - # (because it is normally not supposed to be used ever again). - # to avoid the need to re-infer, set that field here. - if ci.inferred === nothing - CC.setindex!(wvc, inferred_ci, mi) - ci = CC.getindex(wvc, mi) - end - else +# Drive type inference on `mi` using `interp`. On 1.11+ this is `CompilerCaching.typeinf!`, +# which (on 1.12+) recursively walks callees so their CIs carry stored source for +# `CompilerCaching.get_codeinfos` to read back into the `jl_emit_native` payload. Returns +# the root `CodeInstance` (or `nothing` if inference failed). +@static if HAS_INTEGRATED_CACHE + drive_inference!(interp::GPUInterpreter, mi::MethodInstance) = + CompilerCaching.typeinf!(interp, mi) +else + # 1.10: an inline copy that talks to the per-interpreter `CodeCache` instead of the + # integrated one. Returns `nothing` (no integrated CI to hand back; the caller + # fetches via `code_cache(interp)`). + function drive_inference!(interp::GPUInterpreter, mi::MethodInstance) src = CC.typeinf_ext_toplevel(interp, mi) + @assert src !== nothing "Inference of $mi failed" - # inference should have populated our cache - wvc = WorldView(cache, min_world, max_world) - - @assert CC.haskey(wvc, mi) "GPUCompiler: Failed to compile method for $mi, between worlds $min_world and $max_world" - ci = CC.getindex(wvc, mi) - - # if ci is rettype_const, the inference result won't have been cached - # (because it is normally not supposed to be used ever again). - # to avoid the need to re-infer, set that field here. - if ci.inferred === nothing - @atomic ci.inferred = src + # For const-return CIs the inference result wasn't recorded — set it from the + # returned source so callers re-using the CI don't need to re-infer. + wvc = WorldView(CC.code_cache(interp), interp.world, interp.world) + if CC.haskey(wvc, mi) + ci = CC.getindex(wvc, mi) + if ci.inferred === nothing + @atomic ci.inferred = src + end end - end - - return codeinfos -end - -@static if VERSION >= v"1.14-" -function ci_cache_lookup(cache, mi, min_world, max_world) - # In Julia 1.14+, WorldView was replaced by InternalCodeCache with WorldRange - # cache is OverlayCodeCache{InternalCodeCache}, extract owner from globalcache - owner = cache.globalcache.owner - wvc = CC.InternalCodeCache(owner, CC.WorldRange(min_world, max_world)) - ci = CC.get(wvc, mi, nothing) - return ci -end -else -function ci_cache_lookup(cache, mi, min_world, max_world) - wvc = WorldView(cache, min_world, max_world) - ci = CC.get(wvc, mi, nothing) - if VERSION < v"1.12.0-DEV.1434" && ci !== nothing && ci.inferred === nothing - # if for some reason we did end up with a codeinfo without inferred source, e.g., - # because of calling `Base.return_types` which only sets rettyp, pretend we didn't - # run inference so that we re-infer now and not during codegen (which is disallowed) return nothing end - return ci end -end # @static if -## interface +## codegen/inference integration -# for platforms without @cfunction-with-closure support -const _method_instances = Ref{Any}() -const _cache = Ref{Any}() -function _lookup_fun(mi, min_world, max_world) - push!(_method_instances[], mi) - ci_cache_lookup(_cache[], mi, min_world, max_world) -end +const HAS_LLVM_GET_CIS = ( + VERSION >= v"1.13.0-DEV.1120" || ( + Libdl.dlsym( + unsafe_load(cglobal(:jl_libjulia_handle, Ptr{Cvoid})), :jl_get_llvm_cis, throw_error = false + ) !== nothing + ) +) @enum CompilationPolicy::Cint begin CompilationPolicyDefault = 0 @@ -698,11 +303,8 @@ function Base.precompile(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - - # populate the cache interp = get_interpreter(job) - cache = CC.code_cache(interp) - ci_cache_populate(interp, cache, job.source, job.world, job.world) + drive_inference!(interp, job.source) return true end @@ -755,29 +357,66 @@ function record_coverage(mi::MethodInstance, src::CodeInfo) return end -const HAS_LLVM_GET_CIS = ( - VERSION >= v"1.13.0-DEV.1120" || ( - Libdl.dlsym( - unsafe_load(cglobal(:jl_libjulia_handle, Ptr{Cvoid})), :jl_get_llvm_cis, throw_error = false - ) !== nothing - ) -) +# for platforms without @cfunction-with-closure support, used pre-1.12 (and on 1.12 paths +# that still require `cgparams.lookup`). +const _method_instances = Ref{Any}() +const _lookup_cache = Ref{Any}() +function _lookup_fun(mi, min_world, max_world) + push!(_method_instances[], mi) + lookup_ci(_lookup_cache[], mi, min_world, max_world) +end + +# Resolve a `(MI, world)` to a CodeInstance, on whatever cache shape we have: +# `WorldView{CodeCache}` on 1.10, an `owner` token on 1.11+. +@static if HAS_INTEGRATED_CACHE +function lookup_ci(owner, mi::MethodInstance, min_world::UInt, max_world::UInt) + @static if VERSION >= v"1.14-" + wvc = CC.InternalCodeCache(owner, CC.WorldRange(min_world, max_world)) + else + wvc = WorldView(CC.InternalCodeCache(owner), CC.WorldRange(min_world, max_world)) + end + return CC.get(wvc, mi, nothing) +end +else +function lookup_ci(cache::CodeCache, mi::MethodInstance, min_world::UInt, max_world::UInt) + wvc = WorldView(cache, min_world, max_world) + ci = CC.get(wvc, mi, nothing) + if ci !== nothing && ci.inferred === nothing + # rettype-only CI without source — pretend we don't have it so we re-infer + return nothing + end + return ci +end +end function compile_method_instance(@nospecialize(job::CompilerJob)) if job.source.def.primary_world > job.world error("Cannot compile $(job.source) for world $(job.world); method is only valid from world $(job.source.def.primary_world) onwards") end - # populate the cache + # drive inference interp = get_interpreter(job) - cache = CC.code_cache(interp) - populated = ci_cache_populate(interp, cache, job.source, job.world, job.world) + root_ci = drive_inference!(interp, job.source) + + # the cache handle for callback lookups: an owner token on 1.11+, a CodeCache on 1.10 + cache_handle = @static if HAS_INTEGRATED_CACHE + cache_owner(job) + else + interp.code_cache + end + + # gather (CI, CodeInfo) pairs for jl_emit_native (1.12+) + codeinfo_pairs = if VERSION >= v"1.12.0-DEV.1823" && root_ci !== nothing + CompilerCaching.get_codeinfos(root_ci) + else + nothing + end # record line coverage of all compiled code (on older versions of Julia, where - # `ci_cache_populate` does not return sources, this happens after codegen instead) - if Base.JLOptions().code_coverage != 0 - for (ci, src) in populated - record_coverage(ci.def::MethodInstance, src) + # inference does not return sources, this happens after codegen instead) + if Base.JLOptions().code_coverage != 0 && codeinfo_pairs !== nothing + for (ci′, src) in codeinfo_pairs + record_coverage(ci′.def::MethodInstance, src::CodeInfo) end end @@ -787,11 +426,11 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if Sys.ARCH == :x86 || Sys.ARCH == :x86_64 function lookup_fun(mi, min_world, max_world) push!(method_instances, mi) - ci_cache_lookup(cache, mi, min_world, max_world) + lookup_ci(cache_handle, mi, min_world, max_world) end lookup_cb = @cfunction($lookup_fun, Any, (Any, UInt, UInt)) else - _cache[] = cache + _lookup_cache[] = cache_handle _method_instances[] = method_instances lookup_cb = @cfunction(_lookup_fun, Any, (Any, UInt, UInt)) end @@ -831,10 +470,10 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) native_code = if VERSION >= v"1.12.0-DEV.1823" codeinfos = Any[] - for (ci, src) in populated - # each item in the list should be a CodeInstance followed by a CodeInfo + for (ci′, src) in codeinfo_pairs + # each item in the list is a CodeInstance followed by a CodeInfo # indicating something to compile - push!(codeinfos, ci::CodeInstance) + push!(codeinfos, ci′::CodeInstance) push!(codeinfos, src::CodeInfo) end @ccall jl_emit_native(codeinfos::Vector{Any}, ts_mod::LLVM.API.LLVMOrcThreadSafeModuleRef, Ref(params)::Ptr{Base.CodegenParams}, #=extern linkage=# false::Cint)::Ptr{Cvoid} @@ -856,22 +495,17 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # XXX: this is wrong; we can't expose the underlying LLVM module, but should # instead always go through the callback in order to unlock it properly. - # rework this once we depend on Julia 1.9 or later. llvm_ts_mod = LLVM.ThreadSafeModule(llvm_mod_ref) llvm_mod = nothing llvm_ts_mod() do mod llvm_mod = mod end end - if !(Sys.ARCH == :x86 || Sys.ARCH == :x86_64) - cache_gbl = nothing - end # Since Julia 1.13, the caller is responsible for initializing global variables that - # point to global values or bindings with their address in memory. - # Similarly on previous versions when imaging=true, it is also the caller's responsibility - # (see https://github.com/JuliaGPU/GPUCompiler.jl/issues/753), but we can support this on versions - # that have HAS_LLVM_GVS_GLOBALS. + # point to global values or bindings with their address in memory. Similarly on earlier + # versions where `HAS_LLVM_GVS_GLOBALS` is true (see + # https://github.com/JuliaGPU/GPUCompiler.jl/issues/753). gvs = nothing inits = nothing @static if VERSION >= v"1.13.0-DEV.623" @@ -907,19 +541,14 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end end - # Maintain a map from global variables to their initialized Julia values. - # The objects pointed to are perma-rooted, during codegen. - # It is legal to call `Base.unsafe_pointer_to_objref` on `values(gv_to_value)`, - # but x->pointer_from_objref(Base.unsafe_pointer_to_objref(x)) is not idempotent, - # thus we store raw pointers here. - # Currently GVs are privatized, so users may have to handle embedded pointers, - # but this dictionary provides a clear indication that the embedded pointer is - # indeed avalid Julia object. + # Maintain a map from global variables to their initialized Julia values. The + # objects pointed to are perma-rooted during codegen. We *don't* bake these + # addresses into the IR yet, so that we can cache it across sessions. gv_to_value = Dict{String, Ptr{Cvoid}}() - - # On certain version of Julia we have no reliable way to match the `gvs` to their initializers `inits`. if gvs === nothing - # global variables here properly. + # No reliable GV table on this Julia — best-effort discovery from the module. + # On these older versions Julia's own emit may have already baked in absolute + # pointer values; we recover them by reading existing initializers. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -945,22 +574,20 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) for (gv_ref, init) in zip(gvs, inits) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init - # set the initializer - # TODO(vc): To enable full relocation we should actually strip out the initializers here. - if LLVM.isnull(initializer(gv)) - val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) - initializer!(gv, val) - end + # Mirror what `jl_emit_native_impl` does on 1.13+ (aotcompile.cpp:865): + # reset the initializer to null so the IR we hand back is session-portable, + # then `relocate_gvs!` at the toplevel link step writes the session-current + # value in. On 1.13+ this is a no-op (Julia already nulled them); on 1.12, + # where `jl_emit_native_impl` bakes pointers via `literal_static_pointer_val`, + # this is what makes the bitcode we cache safe across sessions. + initializer!(gv, LLVM.null(value_type(initializer(gv)))) end end - code_instances = Core.CodeInstance[] + code_instances = CodeInstance[] if HAS_LLVM_GET_CIS # on sufficiently recent versions of Julia, we can query the CIs compiled. - # this is required after the move to `invoke(::CodeInstance)`, because our - # lookup function (used to populate method_instances) isn't always called then. - num_cis = Ref{Csize_t}(0) @ccall jl_get_llvm_cis(native_code::Ptr{Cvoid}, num_cis::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing @@ -970,7 +597,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) )::Nothing elseif VERSION >= v"1.12.0-DEV.1703" # slightly older versions of Julia used MIs directly - num_mis = Ref{Csize_t}(0) @ccall jl_get_llvm_mis(native_code::Ptr{Cvoid}, num_mis::Ptr{Csize_t}, C_NULL::Ptr{Cvoid})::Nothing @@ -981,53 +607,52 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) if !HAS_LLVM_GET_CIS for mi in method_instances - ci = ci_cache_lookup(cache, mi, job.world, job.world) - ci === nothing && continue + ci′ = lookup_ci(cache_handle, mi, job.world, job.world) + ci′ === nothing && continue llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall( :jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), - native_code, ci, llvm_func_idx, llvm_specfunc_idx + native_code, ci′, llvm_func_idx, llvm_specfunc_idx ) - # Suppose we have two nested interpreters in use at the same time. - # Looking up a ci from the cache is not unique for a given mi. - # Consequently its possible we may not have compiled the ci found - # by the cache (instead having compiled the ci from the other interp). if llvm_func_idx[] == -1 continue end - push!(code_instances, ci) + push!(code_instances, ci′) end else # `jl_get_llvm_cis` can report stale CIs that no longer cover the world # this job was compiled in. The explicit cache lookup path already filters # by world; do the same here before de-duplicating by MI. - filter!(code_instances) do ci - ci.min_world <= job.world <= ci.max_world + filter!(code_instances) do ci′ + ci′.min_world <= job.world <= ci′.max_world end - # To avoid a clash in the compiled cache containing both with an interpreter token (like GPUCompiler.GPUCompilerCacheToken) and native, - # prefer the non-native code-instance. - # TODO: in the future we should migrate compiled to have the ci as the key, not the mi. + # `jl_get_llvm_cis` may return CIs belonging to several foreign interpreters as + # well as a native owner-less CI for the same MI. Keep only this job's owner, and + # retain an owner-less CI only when this job has no owned entry for that method. + # Treating every non-nothing owner as ours breaks when multiple GPU back-ends are + # loaded in the same process and eventually creates duplicate compiled entries. + owner = cache_owner(job) owned_mis = Set{MethodInstance}() - for ci in code_instances - if ci.owner !== nothing - push!(owned_mis, ci.def::MethodInstance) + for ci′ in code_instances + if ci′.owner === owner + push!(owned_mis, ci′.def::MethodInstance) end end - filter!(code_instances) do ci - return ci.owner !== nothing || ci.def ∉ owned_mis + filter!(code_instances) do ci′ + return ci′.owner === owner || + (ci′.owner === nothing && ci′.def ∉ owned_mis) end end - # Avoid redundant code_instances. This is necessary to avoid false positives trying to add the same key'd mi to the compiled Dict. unique!(code_instances) # record line coverage of all compiled code (on newer versions of Julia, this happens - # based on the sources returned by `ci_cache_populate` instead) - if Base.JLOptions().code_coverage != 0 && isempty(populated) + # based on the sources returned by inference instead) + if Base.JLOptions().code_coverage != 0 && codeinfo_pairs === nothing for ci in code_instances src = ci.inferred if src isa String @@ -1040,21 +665,22 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end resize!(method_instances, length(code_instances)) - for (i, ci) in enumerate(code_instances) - method_instances[i] = ci.def::MethodInstance + for (i, ci′) in enumerate(code_instances) + method_instances[i] = ci′.def::MethodInstance end # process all compiled method instances compiled = Dict() - for (ci, mi) in zip(code_instances, method_instances) - - # get the function index + function compiled_entry(ci′::CodeInstance; required::Bool=true) llvm_func_idx = Ref{Int32}(-1) llvm_specfunc_idx = Ref{Int32}(-1) ccall(:jl_get_function_id, Nothing, (Ptr{Cvoid}, Any, Ptr{Int32}, Ptr{Int32}), - native_code, ci, llvm_func_idx, llvm_specfunc_idx) - @assert llvm_func_idx[] != -1 || llvm_specfunc_idx[] != -1 "Static compilation failed" + native_code, ci′, llvm_func_idx, llvm_specfunc_idx) + if llvm_func_idx[] == -1 && llvm_specfunc_idx[] == -1 + required && error("Static compilation failed") + return nothing + end # get the function llvm_func = if llvm_func_idx[] >= 1 @@ -1075,11 +701,24 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) nothing end + (; ci=ci′, func=llvm_func, specfunc=llvm_specfunc) + end + + for (ci′, mi) in zip(code_instances, method_instances) @assert !haskey(compiled, mi) # NOTE: it's not safe to store raw LLVM functions here, since those may get # removed or renamed during optimization, so we store their name instead. - compiled[mi] = (; ci, func=llvm_func, specfunc=llvm_specfunc) + compiled[mi] = compiled_entry(ci′) + end + + if !haskey(compiled, job.source) && root_ci !== nothing + # On Julia 1.14-dev, `jl_get_llvm_cis` may omit the root CI. That root + # can also be less-specific than `job.source` when Julia deliberately + # reuses unspecialized code, so recover the entry that codegen actually + # emitted and expose it under the requested source. + entry = compiled_entry(root_ci; required=false) + entry !== nothing && (compiled[job.source] = entry) end # ensure that the requested method instance was compiled @@ -1088,7 +727,30 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end -# partially revert JuliaLangjulia#49391 +""" + relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + +Bake absolute pointer values into the initializers of `julia.constgv`-tagged +global variables, matching them by name against `gv_to_value`. Only GVs whose +initializer is currently null are touched: Preserving existing initializers +covers the older-Julia path where Julia itself emits pointer values directly. + +GVs present in `mod` but missing from `gv_to_value` keep a null initializer. +""" +function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + mod_gvs = globals(mod) + for (name, init) in gv_to_value + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + if LLVM.isnull(initializer(gv)) + val = const_inttoptr(ConstantInt(Int64(init)), value_type(initializer(gv))) + initializer!(gv, val) + end + end + return +end + +# partially revert JuliaLang/julia#49391 — see #527 @static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master !(v"1.11-beta2" <= VERSION < v"1.12") # reverted on 1.11-beta2 function CC.typeinf(interp::GPUInterpreter, frame::CC.InferenceState) diff --git a/src/metal.jl b/src/metal.jl index 0c7dd6aa..413e4a5b 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -106,12 +106,6 @@ have_fma(@nospecialize(target::MetalCompilerTarget), T::Type) = true ## job -# the debug-info level is part of the slug (as on PTX): it changes the debug metadata emitted -# into the runtime library, and the device-exception reporters branch on it, so a library -# built at one level must not be reused at another. -runtime_slug(job::CompilerJob{MetalCompilerTarget}) = - "metal-macos$(job.config.target.macos)-debuginfo=$(Int(llvm_debug_info(job)))" - isintrinsic(@nospecialize(job::CompilerJob{MetalCompilerTarget}), fn::String) = return startswith(fn, "air.") diff --git a/src/native.jl b/src/native.jl index fdd880ec..536c31d4 100644 --- a/src/native.jl +++ b/src/native.jl @@ -33,6 +33,5 @@ end ## job -runtime_slug(job::CompilerJob{NativeCompilerTarget}) = "native_$(job.config.target.cpu)-$(hash(job.config.target.features))$(job.config.target.jlruntime ? "-jlrt" : "")" uses_julia_runtime(job::CompilerJob{NativeCompilerTarget}) = job.config.target.jlruntime can_vectorize(job::CompilerJob{NativeCompilerTarget}) = true diff --git a/src/optim.jl b/src/optim.jl index 7a07d200..b0b339a3 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -358,6 +358,8 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op end add!(fpm, GCInvariantVerifierPass()) add!(fpm, LateLowerGCPass()) + # FinalLowerGCPass moved from a module pass to a function pass in + # JuliaLang/julia#51081 (1.11.0-DEV.208). if uses_julia_runtime(job) && VERSION >= v"1.11.0-DEV.208" add!(fpm, FinalLowerGCPass()) end @@ -501,13 +503,7 @@ function (self::LinkLibraries)(mod::LLVM.Module) # references after the normal library-linking phase has run. Give back-ends a # second chance to resolve those references before they lower their own runtime # intrinsics. This is a no-op for back-ends without a link_libraries! override. - if has_legacy_link_libraries(self.job) - undefined_fns = [LLVM.name(f) for f in functions(mod) - if isdeclaration(f) && !LLVM.isintrinsic(f)] - link_libraries!(self.job, mod, undefined_fns) - else - link_libraries!(self.job, mod) - end + link_libraries!(self.job, mod) return true end GPULinkLibrariesPass(job) = NewPMModulePass("GPULinkLibraries", LinkLibraries(job)) diff --git a/src/ptx.jl b/src/ptx.jl index dfde75ad..3c22f5c9 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -138,21 +138,14 @@ isintrinsic(@nospecialize(job::CompilerJob{PTXCompilerTarget}), fn::String) = # libdevice's __CUDA_ARCH dispatch, are still supported by the external back-end. startswith(fn, "llvm.nvvm.") -# XXX: the debuginfo part should be handled by GPUCompiler as it applies to all back-ends. -runtime_slug(@nospecialize(job::CompilerJob{PTXCompilerTarget})) = - "ptx$(job.config.target.ptx.major)$(job.config.target.ptx.minor)" * - "-$(cpu_name(job.config.target))" * - "-debuginfo=$(Int(llvm_debug_info(job)))" - function finish_module!(@nospecialize(job::CompilerJob{PTXCompilerTarget}), mod::LLVM.Module, entry::LLVM.Function) # tell NVVMReflect whether to flush denormals; this mirrors what Clang does # for `-fcuda-flush-denormals-to-zero` and is the only `__nvvm_reflect` key # LLVM's NVVMReflectPass honors besides `__CUDA_ARCH`. only emit it on the # toplevel module that runs through `optimize!`, as sub-modules (the cached - # runtime, deferred jobs) don't need it, and the cached runtime in - # particular would otherwise conflict on link if it was built with a - # different `fastmath` setting (which isn't part of `runtime_slug`). + # runtime, deferred jobs) don't need it and should not get module-level + # flags that can collide when linked into the toplevel module. if job.config.toplevel flags(mod)["nvvm-reflect-ftz", LLVM.API.LLVMModuleFlagBehaviorOverride] = Metadata(ConstantInt(Int32(job.config.target.fastmath ? 1 : 0))) diff --git a/src/rtlib.jl b/src/rtlib.jl index 85cb1249..3089d026 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,10 +60,36 @@ end ## functionality to build the runtime library -function emit_function!(mod, config::CompilerConfig, f, method) - tt = Base.to_tuple_type(method.types) - source = generic_methodinstance(f, tt) - new_mod, meta = compile_unhooked(:llvm, CompilerJob(source, config)) +# Per-function compilation results for the GPU runtime library, cached through the +# same `cached_results` mechanism back-ends use for kernels. On 1.11+ the bitcode +# thus lives on the runtime function's `CodeInstance` — possibly alongside a +# back-end's own results struct — and persists through precompilation, so sessions +# loading a back-end that compiled its runtime during precompile skip codegen +# entirely. On 1.10 it is cached for the duration of the session. +mutable struct RuntimeFunctionResults + bitcode::Union{Nothing,Vector{UInt8}} + RuntimeFunctionResults() = new(nothing) +end + +# Compile a single runtime function and link it into `mod`. The renamed bitcode is +# memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache +# below additionally avoids repeating the parse-and-link work within a session. +function emit_function!(mod, config::CompilerConfig, source::MethodInstance, method, + world::UInt) + name = method.llvm_name + rt_job = CompilerJob(source, config, world) + + # On 1.11+, don't run a standalone inference walk on a miss: `compile_unhooked` below + # drives inference itself. The 1.10 implementation returns its session-local result + # directly, without touching inference. + ci, res = runtime_function_results(rt_job) + if res !== nothing && res.bitcode !== nothing + link!(mod, parse(LLVM.Module, MemoryBuffer(res.bitcode))) + ci === nothing && (ci = runtime_code_instance(rt_job)) + return ci::CodeInstance + end + + new_mod, meta = compile_unhooked(:llvm, rt_job) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) if return_type(ft) != return_type(expected_ft) @@ -73,37 +99,95 @@ function emit_function!(mod, config::CompilerConfig, f, method) # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - temp_name = LLVM.name(meta.entry) - link!(mod, new_mod) - entry = functions(mod)[temp_name] + # runtime functions may reference Julia objects through `julia.constgv` globals (e.g. + # `box_bool` returning the `jl_true`/`jl_false` singletons). Kernels get theirs + # relocated when the fully-linked toplevel module is finalized, but the runtime's + # mappings would be dropped along with the rest of its per-function metadata: bake + # the session-absolute addresses into the cached bitcode instead, and keep such + # functions out of package images (their pointers don't survive into other sessions). + if !isempty(meta.gv_to_value) + relocate_gvs!(new_mod, meta.gv_to_value) + mark_session_dependent!(rt_job) + end - # if a declaration already existed, replace it with the function to avoid aliasing - # (and getting function names like gpu_signal_exception1) - name = method.llvm_name - if haskey(functions(mod), name) - decl = functions(mod)[name] - @assert value_type(decl) == value_type(entry) - replace_uses!(decl, entry) + # rename to the final `gpu_*` name on the per-function module, so the cached bitcode + # is immediately link-ready (no per-session rename pass on a cache hit). + if haskey(functions(new_mod), name) && functions(new_mod)[name] !== meta.entry + decl = functions(new_mod)[name] + @assert value_type(decl) == value_type(meta.entry) + replace_uses!(decl, meta.entry) erase!(decl) end - LLVM.name!(entry, name) + LLVM.name!(meta.entry, name) + + io = IOBuffer() + write(io, new_mod) + ci === nothing && (ci = runtime_code_instance(rt_job)) + res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) + res.bitcode = take!(io) + + link!(mod, new_mod) + return ci::CodeInstance end -function build_runtime(@nospecialize(job::CompilerJob)) - mod = LLVM.Module("GPUCompiler run-time library") +function runtime_function_results(@nospecialize(job::CompilerJob)) + @static if HAS_INTEGRATED_CACHE + ci = job_code_instance(job) + ci === nothing && return nothing, nothing + return ci, job_results(RuntimeFunctionResults, ci, job.config) + else + # The 1.10 results store is independent of the CodeCache. Avoid querying the latter + # until the caller actually needs the contributing CI. + return nothing, cached_results(RuntimeFunctionResults, job) + end +end - # the compiler job passed into here is identifies the job that requires the runtime. - # derive a job that represents the runtime itself (notably with kernel=false). - config = CompilerConfig(job.config; kernel=false, toplevel=false, only_entry=false, strip=false) +function runtime_method_instance(@nospecialize(job::CompilerJob), method) + def = if isa(method.def, Symbol) + isdefined(runtime_module(job), method.def) || return nothing + getfield(runtime_module(job), method.def) + else + method.def + end + # Resolve at the requesting job's explicit world, rather than this task's TLS world. + # Runtime methods may be redefined while a long-lived compilation task is running. + return generic_methodinstance( + typeof(def), Base.to_tuple_type(method.types), job.world) +end + +function runtime_code_instance(@nospecialize(job::CompilerJob)) + ci = @static if HAS_INTEGRATED_CACHE + job_code_instance(job) + else + cache = WorldView(get_code_cache(job), job.world, job.world) + CC.get(cache, job.source, nothing) + end + ci === nothing && error("Missing CodeInstance after compiling $(job.source)") + return ci::CodeInstance +end + +# the compiler job passed into here identifies the job that requires the runtime. +# derive a config that represents the runtime itself (notably with kernel=false). +# Fields that identify or optimize only the *kernel* job are reset so runtime artifacts are +# keyed identically for all kernels sharing the remaining codegen-relevant settings. Runtime +# functions always use the specfunc ABI and are deliberately left unoptimized until linked +# into the toplevel module, making the kernel's entry ABI and LLVM opt level irrelevant. +function runtime_config(@nospecialize(job::CompilerJob)) + CompilerConfig(job.config; kernel=false, entry_abi=:specfunc, opt_level=0, + toplevel=false, only_entry=false, strip=false, name=nothing) +end + +function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) + mod = LLVM.Module("GPUCompiler run-time library") + sources = MethodInstance[] + code_instances = CodeInstance[] for method in values(Runtime.methods) - def = if isa(method.def, Symbol) - isdefined(runtime_module(job), method.def) || continue - getfield(runtime_module(job), method.def) - else - method.def - end - emit_function!(mod, config, typeof(def), method) + resolved = runtime_method_instance(job, method) + resolved === nothing && continue + source = resolved + push!(sources, source) + push!(code_instances, emit_function!(mod, config, source, method, job.world)) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -111,65 +195,65 @@ function build_runtime(@nospecialize(job::CompilerJob)) # removes Julia address spaces, which would then lead to type mismatches when using # functions from the runtime library from IR that has not been stripped of AS info. - mod + return mod, sources, code_instances end -@static if VERSION >= v"1.11.0" - import Core.Compiler: is_asserts -else - is_asserts() = false +# Session-local cache of assembled runtime libraries, keyed by +# `(runtime_config(job), opaque_pointers)`: the derived runtime config covers every +# codegen-relevant setting (e.g. the debug level, which is baked into the runtime IR +# as a constant), while cosmetic kernel-job fields are normalized away. Cross-session +# persistence happens at the per-function level (see `RuntimeFunctionResults`): +# reassemble on first use of each session, then reuse within the session. +# +# Keep both the selected MethodInstances and their contributing CodeInstances with the +# assembled bytes. Re-resolving methods after the world changes detects direct method-table +# changes; clipped CI ranges detect invalidated transitive callees. This makes the assembled +# cache follow Julia's validity model without a manual `reset_runtime` hook. +mutable struct RuntimeLibrary + bytes::Vector{UInt8} + sources::Vector{MethodInstance} + code_instances::Vector{CodeInstance} + validated_world::UInt end -@locked function load_runtime(@nospecialize(job::CompilerJob)) - global compile_cache - if compile_cache === nothing # during precompilation - return build_runtime(job) - end - - slug = runtime_slug(job) - if !supports_typed_pointers(context()) - slug *= "-opaque" - end +function runtime_library_valid(lib::RuntimeLibrary, @nospecialize(job::CompilerJob)) + # Method-table changes and CI invalidations always advance the world counter. A runtime + # already validated for this exact world therefore needs no per-function scan on the + # common cache-hit path. + job.world == lib.validated_world && return true - # Julia codegen changes metadata in modules when `FORCE_ASSERTIONS=1` - if is_asserts() - slug *= "-asserts" + i = 0 + for method in values(Runtime.methods) + resolved = runtime_method_instance(job, method) + resolved === nothing && continue + i += 1 + i <= length(lib.sources) || return false + resolved === lib.sources[i] || return false end + i == length(lib.sources) || return false + all(ci -> ci.min_world <= job.world <= ci.max_world, lib.code_instances) || return false + lib.validated_world = job.world + return true +end - name = "runtime_$(slug).bc" - path = joinpath(compile_cache, name) +const runtime_libs = Dict{Tuple{CompilerConfig, Bool}, RuntimeLibrary}() +const runtime_libs_lock = ReentrantLock() - # the cache is shared across processes and may disappear at any point - # (e.g. `reset_runtime()` in another process), so treat it as best-effort - if ispath(path) - try - return parse(LLVM.Module, MemoryBufferFile(path); lazy=true) - catch err - @debug "Failed to load cached GPU runtime library; rebuilding" exception=(err, catch_backtrace()) - end - end +@locked function load_runtime(@nospecialize(job::CompilerJob)) + config = runtime_config(job) + key = (config, !supports_typed_pointers(context())) - @debug "Building the GPU runtime library at $path" - lib = build_runtime(job) - - try - # atomic write to disk - mkpath(compile_cache) - temp_path, io = mktemp(compile_cache; cleanup=false) - write(io, lib) - close(io) - @static if VERSION >= v"1.12.0-DEV.1023" - mv(temp_path, path; force=true) - else - Base.rename(temp_path, path, force=true) + cached = Base.@lock runtime_libs_lock begin + cached = get(runtime_libs, key, nothing) + if cached === nothing || !runtime_library_valid(cached, job) + lib, sources, code_instances = build_runtime(job, config) + io = IOBuffer() + write(io, lib) + cached = RuntimeLibrary(take!(io), sources, code_instances, job.world) + runtime_libs[key] = cached end - catch err - @warn "Failed to cache GPU runtime library" exception=(err, catch_backtrace()) maxlog=1 + cached end - return lib + return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true) end - -# remove the existing cache -# NOTE: call this function from global scope, so any change triggers recompilation. -reset_runtime() = rm(compile_cache; recursive=true, force=true) diff --git a/src/spirv.jl b/src/spirv.jl index 3400fef6..97b3a15b 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -24,7 +24,12 @@ export SPIRVCompilerTarget Base.@kwdef struct SPIRVCompilerTarget <: AbstractCompilerTarget version::Union{Nothing,VersionNumber} = nothing - extensions::Vector{String} = [] + # SPIR-V extensions, as the comma-separated specifier string passed verbatim to the + # translator/back-end via `--spirv-ext`, e.g. "+SPV_EXT_shader_atomic_float_add,+SPV_KHR_expect_assume" + # (LLVM feature-string style, cf. `GCNCompilerTarget.features`). Kept as a plain + # `String` -- not a `Vector` -- so `jl_egal`-based owner/config lookups can match + # structurally equivalent targets after package-image deserialization. + extensions::String = "" supports_fp16::Bool = true supports_fp64::Bool = true supports_bfloat16::Bool = false @@ -58,11 +63,6 @@ llvm_datalayout(::SPIRVCompilerTarget) = Int===Int64 ? ## job -# TODO: encode debug build or not in the compiler job -# https://github.com/JuliaGPU/CUDAnative.jl/issues/368 -runtime_slug(job::CompilerJob{SPIRVCompilerTarget}) = - "spirv-" * String(job.config.target.backend) - function finish_module!(job::CompilerJob{SPIRVCompilerTarget}, mod::LLVM.Module, entry::LLVM.Function) # update calling convention @@ -154,8 +154,7 @@ end cmd = `$(SPIRV_LLVM_Backend_jll.llc()) $input -filetype=obj -o $translated` if !isempty(job.config.target.extensions) - str = join(map(ext->"+$ext", job.config.target.extensions), ",") - cmd = `$(cmd) -spirv-ext=$str` + cmd = `$(cmd) -spirv-ext=$(job.config.target.extensions)` end elseif job.config.target.backend === :khronos translator = if isavailable(SPIRV_LLVM_Translator_jll) @@ -168,8 +167,7 @@ end cmd = `$translator -o $translated $input --spirv-debug-info-version=ocl-100` if !isempty(job.config.target.extensions) - str = join(map(ext->"+$ext", job.config.target.extensions), ",") - cmd = `$(cmd) --spirv-ext=$str` + cmd = `$(cmd) --spirv-ext=$(job.config.target.extensions)` end if job.config.target.version !== nothing diff --git a/src/utils.jl b/src/utils.jl index e8938a8f..db2fc7d7 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -6,6 +6,28 @@ else __has_internal_julia_change(version_or::VersionNumber, feature::Symbol) = false end + + +## `public` keyword compat + +""" + @public foo, bar + +Declare `foo, bar` as public API. Lowers to `public foo, bar` on 1.11+ (where `public` +is keyword syntax) and to a no-op on 1.10. +""" +macro public(symbols_expr) + syms = symbols_expr isa Symbol ? [symbols_expr] : + symbols_expr.head === :tuple ? [a isa Symbol ? a : a.args[1] for a in symbols_expr.args] : + [symbols_expr.args[1]] + if VERSION >= v"1.11.0-DEV.469" + esc(Expr(:public, syms...)) + else + nothing + end +end + + ## debug verification should_verify() = ccall(:jl_is_debugbuild, Cint, ()) == 1 || diff --git a/src/validation.jl b/src/validation.jl index c1a86549..5760afc7 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -14,18 +14,14 @@ function method_matches(@nospecialize(tt::Type{<:Tuple}); world::Integer) end function typeinf_type(mi::MethodInstance; interp::CC.AbstractInterpreter) - @static if VERSION < v"1.11.0" - code = Core.Compiler.get(Core.Compiler.code_cache(interp), mi, nothing) - if code isa Core.Compiler.CodeInstance - return code.rettype - end - result = Core.Compiler.InferenceResult(mi, Core.Compiler.typeinf_lattice(interp)) - Core.Compiler.typeinf(interp, result, :global) - Core.Compiler.is_inferred(result) || return Any - Core.Compiler.widenconst(Core.Compiler.ignorelimited(result.result)) + @static if hasmethod(Core.Compiler.typeinf_type, Tuple{CC.AbstractInterpreter, MethodInstance}) + rt = Core.Compiler.typeinf_type(interp, mi) else - something(Core.Compiler.typeinf_type(interp, mi), Any) + # Julia 1.10: only the 4-arg form exists; reconstruct it from the MI. + method = mi.def::Method + rt = Core.Compiler.typeinf_type(interp, method, mi.specTypes, mi.sparam_vals) end + return something(rt, Any) end function check_method(@nospecialize(job::CompilerJob)) diff --git a/test/Project.toml b/test/Project.toml index d82dab81..c6fbed19 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,7 @@ [deps] AMDGPU_LLVM_Backend_jll = "cc5c0156-bd05-5a77-8a68-bb0aafb29019" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CompilerCaching = "9db33cc3-5358-4881-8759-fa4194144afd" FileCheck = "4e644321-382b-4b05-b0b6-5d23c3d944fb" GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" diff --git a/test/gcn.jl b/test/gcn.jl index e91c7579..4d035351 100644 --- a/test/gcn.jl +++ b/test/gcn.jl @@ -20,12 +20,14 @@ sink_gcn(i) = sink(i, Val(5)) kernel() = return end - # the backend participates in the runtime slug, so different back-ends don't share a cache + # the backend participates in the cache owner, so different back-ends don't share a cache job_ext, _ = GCN.create_job(mod.kernel, Tuple{}; backend=:external) job_inp, _ = GCN.create_job(mod.kernel, Tuple{}; backend=:inprocess) - @test endswith(GPUCompiler.runtime_slug(job_ext), "-external") - @test endswith(GPUCompiler.runtime_slug(job_inp), "-inprocess") - @test GPUCompiler.runtime_slug(job_ext) != GPUCompiler.runtime_slug(job_inp) + owner_ext = GPUCompiler.cache_owner(job_ext) + owner_inp = GPUCompiler.cache_owner(job_inp) + @test owner_ext.target.backend === :external + @test owner_inp.target.backend === :inprocess + @test owner_ext != owner_inp # the explicit :external backend generates machine code through the external llc @test (GCN.code_native(devnull, mod.kernel, Tuple{}; backend=:external); true) diff --git a/test/helpers/metal.jl b/test/helpers/metal.jl index 0000a022..8bea2a25 100644 --- a/test/helpers/metal.jl +++ b/test/helpers/metal.jl @@ -25,8 +25,6 @@ end ThreadedRuntimeCompilerJob = CompilerJob{MetalCompilerTarget,ThreadedRuntimeCompilerParams} GPUCompiler.runtime_module(::ThreadedRuntimeCompilerJob) = ThreadedRuntime -GPUCompiler.runtime_slug(job::ThreadedRuntimeCompilerJob) = - "metal-threadedruntime-macos$(job.config.target.macos)-debuginfo=$(Int(GPUCompiler.llvm_debug_info(job)))" function create_job(@nospecialize(func), @nospecialize(types); kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 656028f4..63d8a0f3 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -69,22 +69,4 @@ function code_execution(@nospecialize(func), @nospecialize(types); kwargs...) end end -const runtime_cache = Dict{Any, Any}() - -function compiler(job) - JuliaContext() do ctx - GPUCompiler.compile(:asm, job) - end -end - -function linker(job, asm) - asm -end - -# simulates cached codegen -function cached_execution(@nospecialize(func), @nospecialize(types); kwargs...) - job, kwargs = create_job(func, types; validate=false, kwargs...) - GPUCompiler.cached_compilation(runtime_cache, job.source, job.config, compiler, linker) -end - end diff --git a/test/native.jl b/test/native.jl index 6e242de0..bb9abeee 100644 --- a/test/native.jl +++ b/test/native.jl @@ -1,5 +1,8 @@ @testset "reflection" begin - job, _ = Native.create_job(identity, (Int,)) + mod = @eval module $(gensym()) + f(x::Int) = x + end + job, _ = Native.create_job(mod.f, (Int,)) @test only(GPUCompiler.code_lowered(job)) isa Core.CodeInfo @@ -7,17 +10,17 @@ @test rt === Int @test @filecheck begin - @check "MethodInstance for identity" + @check "MethodInstance for {{.*}}f" GPUCompiler.code_warntype(job) end @test @filecheck begin - @check "@{{(julia|j)_identity_[0-9]+}}" + @check "@{{(julia|j)_f_[0-9]+}}" GPUCompiler.code_llvm(job) end @test @filecheck begin - @check "@{{(julia|j)_identity_[0-9]+}}" + @check "@{{(julia|j)_f_[0-9]+}}" GPUCompiler.code_native(job) end end @@ -118,97 +121,169 @@ end @check "add i64 %{{[0-9]+}}, 2" GPUCompiler.code_llvm(job) end + end - # cached_compilation interface - invocations = Ref(0) - function compiler(job) - invocations[] += 1 - JuliaContext() do ctx - ir, ir_meta = GPUCompiler.compile(:llvm, job) - string(ir) + @testset "cached results" begin + mod = @eval module $(gensym()) + Base.Experimental.@MethodTable(other_method_table) + + mutable struct Results + asm::Union{Nothing,String} + Results() = new(nothing) + end + mutable struct OtherResults + data::Any + OtherResults() = new(nothing) end - end - linker(job, compiled) = compiled - cache = Dict() - ft = typeof(mod.kernel) - tt = Tuple{Int64} - # initial compilation - source = methodinstance(ft, tt, Base.get_world_counter()) - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 2" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) + @noinline child(i) = i + kernel(i) = child(i)+1 end - @test invocations[] == 1 - # cached compilation - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 2" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 1 + job, _ = Native.create_job(mod.kernel, (Int64,)) - # redefinition - @eval mod kernel(i) = child(i)+3 - source = methodinstance(ft, tt, Base.get_world_counter()) - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # before any code exists for the job, the lookup comes up empty + @test GPUCompiler.cached_results(mod.Results, job) === nothing + end + + # get-or-create: first access after inference yields an empty struct, later + # accesses return the same one + precompile(job) + res = GPUCompiler.cached_results(mod.Results, job) + @test res isa mod.Results + @test res.asm === nothing + res.asm = "compiled" + @test GPUCompiler.cached_results(mod.Results, job) === res + + # independent consumers get independent structs for the same job + other = GPUCompiler.cached_results(mod.OtherResults, job) + @test other isa mod.OtherResults + @test GPUCompiler.cached_results(mod.Results, job) === res + + # results are keyed by the full config: a job differing only in codegen-level + # settings (here: the kernel name) must not share artifacts + named_job, _ = Native.create_job(mod.kernel, (Int64,); name="custom") + @test named_job.source === job.source + named_res = GPUCompiler.cached_results(mod.Results, named_job) + @test named_res !== res + @test named_res.asm === nothing + + # ... but an equal config constructed from scratch resolves to the same struct + job2, _ = Native.create_job(mod.kernel, (Int64,)) + @test GPUCompiler.cached_results(mod.Results, job2) === res + + # vararg kernels: on 1.14+, inference caches these under the compilable + # (vararg-widened) MethodInstance rather than the fully-specialized job.source, + # which the lookup needs to chase + vmod = @eval module $(gensym()) + kernel(args...) = nothing + end + vjob, _ = Native.create_job(vmod.kernel, (Int64, Int64)) + precompile(vjob) + @test GPUCompiler.cached_results(mod.Results, vjob) isa mod.Results + + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # The compiler may report CIs for several foreign owners when the same MI has + # been inferred through multiple interpreters. Codegen must select this job's + # owner rather than treating every non-native CI as interchangeable. + other_owner_job, _ = Native.create_job( + mod.kernel, (Int64,); method_table=mod.other_method_table) + precompile(other_owner_job) + other_owner_res = GPUCompiler.cached_results(mod.Results, other_owner_job) + @test other_owner_res !== res + JuliaContext() do ctx + _, meta = GPUCompiler.compile(:llvm, job) + @test meta.compiled[job.source].ci.owner === GPUCompiler.cache_owner(job) + end end - @test invocations[] == 2 - # cached compilation - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - end - @test invocations[] == 2 + # redefinition invalidates: a job in the new world gets a fresh struct + @eval mod kernel(i) = child(i)+2 + new_job, _ = Native.create_job(mod.kernel, (Int64,)) + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # ... after first showing up empty, as the old CodeInstance no longer covers + # the new world + @test GPUCompiler.cached_results(mod.Results, new_job) === nothing + end + precompile(new_job) + new_res = GPUCompiler.cached_results(mod.Results, new_job) + @test new_res !== res + @test new_res.asm === nothing + + @static if GPUCompiler.HAS_INTEGRATED_CACHE + # session-dependent results (e.g. artifacts with relocated GVs) are wiped + # before image serialization; emulate the atexit-driven wipe directly + new_res.asm = "session-dependent" + other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") + other_res = GPUCompiler.cached_results(mod.Results, other_job) + push!(GPUCompiler.session_dependent_jobs, new_job) + GPUCompiler.wipe_session_dependent_results() + @test isempty(GPUCompiler.session_dependent_jobs) + wiped_res = GPUCompiler.cached_results(mod.Results, new_job) + @test wiped_res !== new_res + @test wiped_res.asm === nothing + # ... without affecting other configs on the same CI + @test GPUCompiler.cached_results(mod.Results, other_job) === other_res + end + end + + @testset "runtime cache invalidation" begin + # The assembled runtime cache must follow Julia's CodeInstance invalidation. Runtime + # functions are ordinary Julia methods and can be redefined during a session. + @eval Native.Runtime signal_exception() = nothing + job, _ = Native.create_job(identity, (Nothing,)) + + func_job, _ = Native.create_job(identity, (Nothing,); entry_abi=:func, opt_level=3) + rt_config = GPUCompiler.runtime_config(func_job) + @test rt_config.entry_abi === :specfunc + @test rt_config.opt_level == 0 - # redefinition of an unrelated function - @eval mod unrelated(i) = 42 - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 2 + JuliaContext() do ctx + empty!(GPUCompiler.runtime_libs) + GPUCompiler.load_runtime(job) - # redefining child functions - @eval mod @noinline child(i) = i+1 - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 3 + key = (GPUCompiler.runtime_config(job), !GPUCompiler.supports_typed_pointers(ctx)) + old = GPUCompiler.runtime_libs[key] + @test GPUCompiler.runtime_library_valid(old, job) - # cached compilation - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test invocations[] == 3 + @eval Native.Runtime signal_exception() = return + new_job, _ = Native.create_job(identity, (Nothing,)) + new_job = CompilerJob(new_job.source, new_job.config, Base.get_world_counter()) + @test !GPUCompiler.runtime_library_valid(old, new_job) - # change in configuration - config = CompilerConfig(job.config; name="foobar") - @test @filecheck begin - @check "define i64 @foobar" - Base.invokelatest(GPUCompiler.cached_compilation, cache, source, config, compiler, linker) - end - @test invocations[] == 4 - - # tasks running in the background should keep on using the old version - c1, c2 = Condition(), Condition() - function background(job) - local_source = methodinstance(ft, tt, Base.get_world_counter()) - notify(c1) - wait(c2) # wait for redefinition - GPUCompiler.cached_compilation(cache, local_source, job.config, compiler, linker) - end - t = @async Base.invokelatest(background, job) - wait(c1) # make sure the task has started - @eval mod kernel(i) = child(i)+4 - source = methodinstance(ft, tt, Base.get_world_counter()) - ir = Base.invokelatest(GPUCompiler.cached_compilation, cache, source, job.config, compiler, linker) - @test contains(ir, r"add i64 %\d+, 4") - notify(c2) # wake up the task - @test @filecheck begin - @check_label "define i64 @{{(julia|j)_kernel_[0-9]+}}" - @check "add i64 %{{[0-9]+}}, 3" - fetch(t) + GPUCompiler.load_runtime(new_job) + new = GPUCompiler.runtime_libs[key] + @test new !== old + @test GPUCompiler.runtime_library_valid(new, new_job) + end + end + + @testset "runtime constgv relocation" begin + # runtime functions like `box_bool` may reference Julia singletons through + # `julia.constgv` globals. Their session-absolute addresses must be baked into + # the cached runtime bitcode when it is built: only kernel modules go through + # `relocate_gvs!`, so a slot left null here would stay null on the device. + job, _ = Native.create_job(identity, (Nothing,)) + JuliaContext() do ctx + GPUCompiler.load_runtime(job) + key = (GPUCompiler.runtime_config(job), + !GPUCompiler.supports_typed_pointers(ctx)) + lib = Base.@lock GPUCompiler.runtime_libs_lock GPUCompiler.runtime_libs[key] + # NOTE: parse eagerly; a lazily-parsed module doesn't expose uses + rt = parse(LLVM.Module, MemoryBuffer(lib.bytes)) + used = 0 + for gv in globals(rt) + haskey(metadata(gv), "julia.constgv") || continue + isempty(uses(gv)) && continue + used += 1 + init = LLVM.initializer(gv) + @test init !== nothing && !LLVM.isnull(init) + end + @static if VERSION >= v"1.12-" + # on older versions, Julia bakes addresses without tagging globals + @test used > 0 + end end end @@ -216,12 +291,14 @@ end # when types have no fields, we should always allow them mod = @eval module $(gensym()) struct Empty end + accept_empty(::Empty) = nothing + accept_symbol(::Symbol) = nothing end - Native.code_execution(Returns(nothing), (mod.Empty,)) + Native.code_execution(mod.accept_empty, (mod.Empty,)) # this also applies to Symbols - Native.code_execution(Returns(nothing), (Symbol,)) + Native.code_execution(mod.accept_symbol, (Symbol,)) end @testset "code coverage" begin @@ -419,19 +496,23 @@ end end @testset "function entry safepoint emission" begin + mod = @eval module $(gensym()) + f(::Nothing) = nothing + end + @test @filecheck begin - @check_label "define void @{{(julia|j)_identity_[0-9]+}}" + @check_label "define void @{{(julia|j)_f_[0-9]+}}" @check_not "%safepoint" - Native.code_llvm(identity, Tuple{Nothing}; entry_safepoint=false, optimize=false, dump_module=true) + Native.code_llvm(mod.f, Tuple{Nothing}; entry_safepoint=false, optimize=false, dump_module=true) end # XXX: broken by JuliaLang/julia#57010, # see https://github.com/JuliaLang/julia/pull/57010/files#r2079576894 if VERSION < v"1.13.0-DEV.533" @test @filecheck begin - @check_label "define void @{{(julia|j)_identity_[0-9]+}}" + @check_label "define void @{{(julia|j)_f_[0-9]+}}" @check "%safepoint" - Native.code_llvm(identity, Tuple{Nothing}; entry_safepoint=true, optimize=false, dump_module=true) + Native.code_llvm(mod.f, Tuple{Nothing}; entry_safepoint=true, optimize=false, dump_module=true) end end end diff --git a/test/native/precompile.jl b/test/native/precompile.jl index f08a561f..eb9abc4f 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -14,6 +14,9 @@ precompile_test_harness("Inference caching") do load_path return end + # a kernel that makes no calls, so its CodeInstance has no inference edges + leaf_kernel(A) = nothing + function kernel_w_global(A, x, sym) if sym == :A A[1] = x @@ -29,11 +32,39 @@ precompile_test_harness("Inference caching") do load_path return Int(x) end + mutable struct Results + artifact::Union{Nothing,String} + Results() = new(nothing) + end + + portable_kernel(x) = x + 1 + session_kernel(x) = x + 2 + + # Attach representative back-end artifacts while the package image is built. The + # portable entry should survive serialization; the session-dependent one should be + # removed by GPUCompiler's pre-output atexit hook. + let + job, _ = NativeCompiler.Native.create_job(portable_kernel, (Int,)) + precompile(job) + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "portable" + end + let + job, _ = NativeCompiler.Native.create_job(session_kernel, (Int,)) + precompile(job) + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "session" + NativeCompiler.GPUCompiler.mark_session_dependent!(job) + end + let job, _ = NativeCompiler.Native.create_job(kernel, (Vector{Int}, Int)) precompile(job) end + let + job, _ = NativeCompiler.Native.create_job(leaf_kernel, (Vector{Int},)) + precompile(job) + end + let job, _ = NativeCompiler.Native.create_job(kernel_w_global, (Vector{Int}, Int, Symbol)) precompile(job) @@ -66,16 +97,30 @@ precompile_test_harness("Inference caching") do load_path import NativeCompiler # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + identity_mis = Any[ + GPUCompiler.methodinstance(typeof(identity), Tuple{Int}), + GPUCompiler.CompilerCaching.method_instance(identity, (Int,)), + ] + unique!(identity_mis) token = let job, _ = NativeCompiler.Native.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) + GPUCompiler.cache_owner(job) end - @test !check_presence(identity_mi, token) + @test all(!check_presence(mi, token) for mi in identity_mis) using NativeBackend + portable_job, _ = NativeCompiler.Native.create_job(NativeBackend.portable_kernel, (Int,)) + portable_res = GPUCompiler.cached_results(NativeBackend.Results, portable_job) + @test portable_res !== nothing + @test portable_res.artifact == "portable" + + session_job, _ = NativeCompiler.Native.create_job(NativeBackend.session_kernel, (Int,)) + session_res = GPUCompiler.cached_results(NativeBackend.Results, session_job) + @test session_res !== nothing + @test session_res.artifact === nothing + # Check that kernel survived kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel), Tuple{Vector{Int}, Int}) @test check_presence(kernel_mi, token) @@ -83,11 +128,28 @@ precompile_test_harness("Inference caching") do load_path kernel_w_global_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel_w_global), Tuple{Vector{Int}, Int, Symbol}) @test check_presence(kernel_w_global_mi, token) + # a CodeInstance without inference edges (a kernel making no calls) is + # serialized with the revalidation sentinel but excluded from the edge + # verification list on Julia 1.12 (jl_record_edges skips empty-edge CIs), + # so it deserializes permanently invalid. Fixed by the serialization + # rework in 1.13; 1.11 uses the old scheme and is unaffected. + leaf_edges_lost = v"1.12-" <= VERSION < v"1.13-" + leaf_kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.leaf_kernel), Tuple{Vector{Int}}) + @test check_presence(leaf_kernel_mi, token) broken=leaf_edges_lost + square_mi = GPUCompiler.methodinstance(typeof(NativeBackend.square), Tuple{Float64}) @test check_presence(square_mi, token) # check that identity survived - @test check_presence(identity_mi, token) broken=(v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.13.0-" <= VERSION < v"1.13.0-beta3"|| v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843") + # NOTE: external CIs from the workload survive only flakily on 1.13 + # (the 1.13.0-beta3 backport did not fully fix this), so skip the + # check there. + ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || + v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" + ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" + if !ext_cis_flaky + @test any(mi -> check_presence(mi, token), identity_mis) broken=ext_cis_lost + end # Recompiling a foreign method after loading precompiled owner-token CIs # may also surface a native owner-less CI for the same MethodInstance. @@ -103,27 +165,5 @@ precompile_test_harness("Inference caching") do load_path _, meta = GPUCompiler.compile(:llvm, job) @test haskey(meta.compiled, job.source) end - - GPUCompiler.clear_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == false - - GPUCompiler.enable_disk_cache!() - @test GPUCompiler.disk_cache_enabled() == true - - job, _ = NativeCompiler.Native.create_job(NativeBackend.kernel, (Vector{Int}, Int); validate=false) - @assert job.source == kernel_mi - ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world) - @assert ci !== nothing - @assert ci.inferred !== nothing - path = GPUCompiler.cache_file(ci, job.config) - @test path !== nothing - @test !ispath(path) - NativeCompiler.Native.cached_execution(NativeBackend.kernel, (Vector{Int}, Int)) - @test ispath(path) - GPUCompiler.clear_disk_cache!() - @test !ispath(path) - - GPUCompiler.enable_disk_cache!(false) - @test GPUCompiler.disk_cache_enabled() == false end end diff --git a/test/ptx.jl b/test/ptx.jl index f76717c4..6b82db5e 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -10,56 +10,24 @@ end end -@testset "kernel state survives a runtime rebuild during optimization" begin - # Clearing the cache forces the relink pass inside `optimize!` to rebuild the runtime - # (nested compilation); the kernel must still get its state argument afterwards. +@testset "kernel state survives a runtime rebuild" begin + # Clearing the runtime cache forces the library link inside `emit_llvm` to rebuild + # the runtime (nested compilation); the kernel must still get its state argument + # afterwards. mod = @eval module $(gensym()) function kernel(x) x < 1 && throw(DivideError()) return end end - old_cache = GPUCompiler.compile_cache - ir = try - GPUCompiler.compile_cache = nothing - sprint() do io - PTX.code_llvm(io, mod.kernel, Tuple{Int}; kernel=true, dump_module=true) - end - finally - GPUCompiler.compile_cache = old_cache + Base.@lock GPUCompiler.runtime_libs_lock empty!(GPUCompiler.runtime_libs) + ir = sprint() do io + PTX.code_llvm(io, mod.kernel, Tuple{Int}; kernel=true, dump_module=true) end @test occursin("gpu_report_exception", ir) @test occursin("[1 x i64] %state", ir) end -@testset "runtime cache disappearing" begin - # the cache may be deleted or unusable at any point; compilation should keep working - mod = @eval module $(gensym()) - kernel() = return - end - old_cache = GPUCompiler.compile_cache - try - GPUCompiler.compile_cache = mktempdir() - PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true) - @test !isempty(readdir(GPUCompiler.compile_cache)) - - # remove the cache directory altogether - GPUCompiler.reset_runtime() - PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true) - @test !isempty(readdir(GPUCompiler.compile_cache)) - - # make the cache unwritable; compilation should proceed with only a warning - GPUCompiler.reset_runtime() - write(GPUCompiler.compile_cache, "not a directory") - @test_logs (:warn, r"Failed to cache GPU runtime library") match_mode=:any begin - PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true) - end - rm(GPUCompiler.compile_cache) - finally - GPUCompiler.compile_cache = old_cache - end -end - @testset "kernel functions" begin @testset "kernel argument attributes" begin mod = @eval module $(gensym()) @@ -554,8 +522,8 @@ end @test GPUCompiler.cpu_name(PTXCompilerTarget(cap=v"10.0", feature_set=:family)) == "sm_100f" @test_throws ErrorException GPUCompiler.cpu_name(PTXCompilerTarget(cap=v"9.0", feature_set=:bogus)) - # hash must discriminate, otherwise two configs differing only on feature_set - # would share the same on-disk runtime slug and collide in the compiler cache. + # hash must discriminate, otherwise two targets differing only on feature_set + # could compare equal inside cache-owner keys. @test hash(PTXCompilerTarget(cap=v"9.0", feature_set=:baseline)) != hash(PTXCompilerTarget(cap=v"9.0", feature_set=:arch)) diff --git a/test/ptx/precompile.jl b/test/ptx/precompile.jl index 290c60be..84c10abd 100644 --- a/test/ptx/precompile.jl +++ b/test/ptx/precompile.jl @@ -30,16 +30,25 @@ precompile_test_harness("Inference caching") do load_path import PTXCompiler # Check that no cached entry is present - identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int}) + identity_mis = Any[ + GPUCompiler.methodinstance(typeof(identity), Tuple{Int}), + GPUCompiler.CompilerCaching.method_instance(identity, (Int,)), + ] + unique!(identity_mis) + # NOTE: use the standalone package's module to construct the token — the + # sandbox's own PTX helper module defines a distinct CompilerParams + # type, whose token can never match the precompiled CIs. token = let - job, _ = PTX.create_job(identity, (Int,)) - GPUCompiler.ci_cache_token(job) + job, _ = PTXCompiler.PTX.create_job(identity, (Int,)) + GPUCompiler.cache_owner(job) end - ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing - while ci !== nothing - @test ci.owner !== token - ci = isdefined(ci, :next) ? ci.next : nothing + for identity_mi in identity_mis + ci = isdefined(identity_mi, :cache) ? identity_mi.cache : nothing + while ci !== nothing + @test ci.owner !== token + ci = isdefined(ci, :next) ? ci.next : nothing + end end using PTXBackend @@ -49,6 +58,14 @@ precompile_test_harness("Inference caching") do load_path @test check_presence(kernel_mi, token) # check that identity survived - @test check_presence(identity_mi, token) broken=(v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || v"1.13.0-" <= VERSION < v"1.13.0-beta3"|| v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843") + # NOTE: external CIs from the workload survive only flakily on 1.13 + # (the 1.13.0-beta3 backport did not fully fix this), so skip the + # check there. + ext_cis_lost = v"1.12.0-DEV.1268" <= VERSION < v"1.12.5" || + v"1.14.0-" <= VERSION < v"1.14.0-DEV.1843" + ext_cis_flaky = v"1.13.0-" <= VERSION < v"1.14-" + if !ext_cis_flaky + @test any(mi -> check_presence(mi, token), identity_mis) broken=ext_cis_lost + end end end diff --git a/test/spirv.jl b/test/spirv.jl index b5006194..a100f4f1 100644 --- a/test/spirv.jl +++ b/test/spirv.jl @@ -182,7 +182,7 @@ end """, NTuple{N, Core.VecElement{Float32}}, NTuple{2, NTuple{N, Core.VecElement{Float32}}}, x.data, y.data)) end end - kernel(x...) = @noinline fadd(x...) + kernel(x, y) = @noinline fadd(x, y) end @test @filecheck begin @@ -191,7 +191,8 @@ end @check_label "define void @{{(julia|j)_kernel_[0-9]+}}" @check cond=(v"1.12" <= VERSION < v"1.12.5") "alloca <2 x i64>, align 16" @check cond=(VERSION >= v"1.12.5") "alloca [2 x i64], align 16" - SPIRV.code_llvm(mod.kernel, NTuple{2, mod.Vec{4, Float32}}; backend, dump_module=true) + SPIRV.code_llvm(mod.kernel, Tuple{mod.Vec{4, Float32}, mod.Vec{4, Float32}}; + backend, dump_module=true) end @test @filecheck begin @@ -199,7 +200,8 @@ end @check_label "define void @{{(julia|j)_kernel_[0-9]+}}" @check cond=(v"1.12" <= VERSION < v"1.12.5") "alloca [2 x <2 x i64>], align 16" @check cond=(VERSION >= v"1.12.5") "alloca [4 x i64], align 16" - SPIRV.code_llvm(mod.kernel, NTuple{2, mod.Vec{8, Float32}}; backend, dump_module=true) + SPIRV.code_llvm(mod.kernel, Tuple{mod.Vec{8, Float32}, mod.Vec{8, Float32}}; + backend, dump_module=true) end end