Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/actions/setup-llvm22/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -165,3 +165,7 @@ runs:
}

"LLVM_SYS_221_PREFIX=C:\llvm" | Out-File -FilePath $env:GITHUB_ENV -Append
# Windows links LLVM-C dynamically to avoid the release archive's
# /MT CRT and bundled-rpmalloc collision with Rust's /MD binaries.
# Put the matching DLL beside every later cargo test/build process.
"C:\llvm\bin" | Out-File -FilePath $env:GITHUB_PATH -Append
12 changes: 10 additions & 2 deletions benchmarks/public_baseline.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def _is_resolved_path(value: Any) -> bool:
absolute path — :func:`portable_path` deliberately emits repo-relative paths.
"""
text = str(value or "")
return bool(text) and (os.path.isabs(text) or os.sep in text)
return bool(text) and (os.path.isabs(text) or "/" in text or "\\" in text)


def _cargo_profile_tables(data: bytes) -> bytes:
Expand Down Expand Up @@ -176,8 +176,16 @@ def _cargo_profile_tables(data: bytes) -> bytes:
return b"".join(kept)


def _normalize_checkout_newlines(data: bytes) -> bytes:
"""Make text fingerprints independent of Git's checkout policy."""
return data.replace(b"\r\n", b"\n")


def _fingerprint_bytes(name: str) -> bytes:
data = (ROOT / name).read_bytes()
# Every fingerprint input is text. Git may materialize it as CRLF on a
# Windows checkout even though the canonical blob (and Linux/macOS
# checkouts) uses LF; that transport detail must not invalidate evidence.
data = _normalize_checkout_newlines((ROOT / name).read_bytes())
if name == "Cargo.toml":
# The version normalization stays: `[workspace.package] version` is not
# in a profile table, but keeping the substitution makes the intent
Expand Down
1 change: 1 addition & 0 deletions crates/perry-codegen-arkts/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ pub(crate) fn empty_module() -> Module {
closure_display_names: std::collections::HashMap::new(),
class_display_names: std::collections::HashMap::new(),
closure_source_text: std::collections::HashMap::new(),
local_source_spans: std::collections::HashMap::new(),
async_generator_funcs: std::collections::HashSet::new(),
gen_param_prologue_len: std::collections::HashMap::new(),
}
Expand Down
1 change: 1 addition & 0 deletions crates/perry-codegen-arkts/tests/phase2_full_app_smoke.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ fn empty_module() -> Module {
closure_display_names: std::collections::HashMap::new(),
class_display_names: std::collections::HashMap::new(),
closure_source_text: std::collections::HashMap::new(),
local_source_spans: std::collections::HashMap::new(),
async_generator_funcs: std::collections::HashSet::new(),
gen_param_prologue_len: std::collections::HashMap::new(),
}
Expand Down
16 changes: 15 additions & 1 deletion crates/perry-codegen/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,23 @@ log.workspace = true
serde.workspace = true
serde_json.workspace = true

inkwell = { version = "0.9.0", features = ["llvm22-1"], optional = true }

# Keep these target lists aligned with `inprocess::global_init`. Inkwell's
# `target-all` default references backends Perry cannot emit and which the
# official Windows LLVM archive does not build.
[target.'cfg(not(windows))'.dependencies]
inkwell = { version = "0.9.0", default-features = false, features = ["llvm22-1", "target-x86", "target-aarch64"], optional = true }
llvm-sys = { version = "221", optional = true }

# LLVM's official Windows static archives use a private /MT CRT and bundle
# rpmalloc, while Rust's MSVC target uses /MD. Passing LLVM-owned allocations
# across that boundary crashes even when the static link succeeds (#7985).
# The same release ships a complete LLVM-C.dll/import-library pair; link that
# on Windows and let build.rs locate its import library under the pinned prefix.
[target.'cfg(windows)'.dependencies]
inkwell = { version = "0.9.0", default-features = false, features = ["llvm22-1-no-llvm-linking", "target-x86", "target-aarch64"], optional = true }
llvm-sys = { version = "221", features = ["no-llvm-linking"], optional = true }

# Self dev-dependency (#7493). This is the whole mechanism by which the
# integration suites under `tests/` — which link this crate as an ordinary
# external consumer — can see `perry_codegen::testing`. Cargo supports
Expand Down
18 changes: 18 additions & 0 deletions crates/perry-codegen/build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
fn main() {
println!("cargo:rerun-if-env-changed=LLVM_SYS_221_PREFIX");

if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("windows") {
return;
}

let prefix = std::env::var_os("LLVM_SYS_221_PREFIX").unwrap_or_else(|| {
panic!("LLVM_SYS_221_PREFIX must point to the LLVM 22 development archive on Windows")
});
let lib_dir = std::path::PathBuf::from(prefix).join("lib");
if !lib_dir.join("LLVM-C.lib").is_file() {
panic!("{} does not contain LLVM-C.lib", lib_dir.display());
}

println!("cargo:rustc-link-search=native={}", lib_dir.display());
println!("cargo:rustc-link-lib=dylib=LLVM-C");
}
48 changes: 37 additions & 11 deletions crates/perry-codegen/src/codegen/artifacts.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
//! Tail of `compile_module`: emits closure bodies, class methods +
//! ctors + statics, wrapper symbols (function-value wrappers,
//! ExternFuncRef closure wrappers, export-rename aliases + stubs,
//! method closure-call wrappers), namespace globals + extern declares,
//! the module entry function, and finally the string-pool init.
//!
//! Split out of `codegen/mod.rs` purely to keep mod.rs under the
//! 2000-line LOC budget. No behavior changes — the function body
//! below is a verbatim move of the original inline block.
//! Emits `compile_module` artifacts: closures, classes, wrappers, namespace
//! globals, the module entry function, and string-pool initialization.
//! Split from `codegen/mod.rs` to keep the compiler pipeline navigable.

use std::collections::HashMap;
use std::time::Instant;

use anyhow::{Context, Result};
use perry_hir::Module as HirModule;
Expand Down Expand Up @@ -54,6 +49,7 @@ use super::string_pool::emit_string_pool;
/// in-prelude local names so the moved block reads unchanged once
/// destructured.
pub(super) struct ModuleArtifactsCtx<'a> {
pub progress: &'a super::CompileProgress,
pub llmod: &'a mut LlModule,
pub target_triple: &'a str,
pub strings: &'a mut StringPool,
Expand Down Expand Up @@ -181,6 +177,7 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
// (auto-reborrowed on each per-function call site below); the
// rest are shared borrows.
let ModuleArtifactsCtx {
progress,
llmod,
target_triple,
strings,
Expand Down Expand Up @@ -230,8 +227,11 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
};

let module_reassigned_locals = crate::collectors::reassigned_locals_in_module(hir);
progress.checkpoint("reassigned-local analysis");

for (func_id, closure_expr) in closures {
let closure_started = Instant::now();
let closure_progress_step = (closures.len() / 20).max(1);
for (closure_index, (func_id, closure_expr)) in closures.iter().enumerate() {
if cross_module.typed_f64_closures.contains(func_id) {
compile_typed_f64_closure(
llmod,
Expand Down Expand Up @@ -295,13 +295,20 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
cross_module,
)
.with_context(|| format!("lowering closure func_id={}", func_id))?;
let done = closure_index + 1;
if done == closures.len() || done % closure_progress_step == 0 {
progress.items("closure bodies", done, closures.len(), closure_started);
}
}
progress.checkpoint("closure bodies");

// Lower each class method as `perry_method_<modprefix>__<class>__<name>(
// this_box, arg0, arg1, ...) -> double`. Methods are emitted as
// standalone LLVM functions; the dispatch in `lower_call` calls
// them directly.
for class in &hir.classes {
let classes_started = Instant::now();
let class_progress_step = (hir.classes.len() / 20).max(1);
for (class_index, class) in hir.classes.iter().enumerate() {
for method in &class.methods {
let typed_public_trampoline = if cross_module
.typed_f64_methods
Expand Down Expand Up @@ -765,8 +772,19 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
)
})?;
}
let done = class_index + 1;
if done == hir.classes.len() || done % class_progress_step == 0 {
progress.items(
"classes (methods, constructors, and statics)",
done,
hir.classes.len(),
classes_started,
);
}
}

progress.checkpoint("class methods, constructors, and statics");

// Emit FuncRef-as-value wrappers. For each user function, generate
// a thin wrapper `__perry_wrap_<name>` whose signature matches the
// closure-call ABI: `double(i64 this_closure, double arg0, double
Expand Down Expand Up @@ -1147,6 +1165,8 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
}
}

progress.checkpoint("top-level function value wrappers and aliases");

// Issue #774: emit closure-call wrappers for class instance methods
// so `Expr::SuperPropertyGet` (value-form `super.<method>`) can
// materialize them via `js_closure_alloc_singleton(@__perry_wrap_<method>)`.
Expand Down Expand Up @@ -1395,6 +1415,8 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
}
}

progress.checkpoint("method, fallback, and imported function wrappers");

// Issue #100: emit the per-module `@__perry_ns_<prefix>` global iff
// this module is the target of at least one dynamic `import()` site
// anywhere in the program. Defined here with external linkage so the
Expand Down Expand Up @@ -1518,6 +1540,7 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
&namespace_key_globals,
)
.with_context(|| format!("lowering entry of module '{}'", hir.name))?;
progress.checkpoint("namespace setup and module entry body");

// Issue #392: pre-intern every user-class method name into the
// string pool so `emit_string_pool` (which takes `&strings`) can
Expand Down Expand Up @@ -1941,6 +1964,8 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
})
.collect();

progress.checkpoint("runtime registration metadata");

emit_string_pool(
llmod,
strings,
Expand Down Expand Up @@ -1968,6 +1993,7 @@ pub(super) fn emit_module_artifacts(c: ModuleArtifactsCtx<'_>) -> Result<()> {
&user_fn_display_names,
&user_fn_source,
);
progress.checkpoint("string pool and registration initializer");

Ok(())
}
41 changes: 31 additions & 10 deletions crates/perry-codegen/src/codegen/closure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -544,6 +544,21 @@ pub(super) fn compile_closure(
_ => return Err(anyhow!("compile_closure: expected Expr::Closure")),
};

// A LocalId is module-unique, but a closure can only observe ids referenced
// or declared in its own body (plus its parameters/capture list). Older
// code cloned the complete module-wide boxed/type/reassignment tables into
// every closure's FnCtx. Generated bundles contain thousands of closures,
// making that O(closures * module locals) in both time and retained memory.
// Build the precise key set once and project each global oracle through it.
let mut closure_referenced_ids: HashSet<u32> = HashSet::new();
collect_ref_ids_in_stmts(body, &mut closure_referenced_ids);
let mut closure_declared_ids: HashSet<u32> = HashSet::new();
collect_let_ids(body, &mut closure_declared_ids);
let mut closure_relevant_ids = closure_referenced_ids.clone();
closure_relevant_ids.extend(closure_declared_ids.iter().copied());
closure_relevant_ids.extend(params.iter().map(|p| p.id));
closure_relevant_ids.extend(captures.iter().copied());

let public_llvm_name = format!("perry_closure_{}__{}", module_prefix, func_id);
let typed_public_trampoline = if cross_module.typed_f64_closures.contains(&func_id) {
Some(TypedFunctionTrampolineKind::F64)
Expand Down Expand Up @@ -616,7 +631,11 @@ pub(super) fn compile_closure(

let _ = lf.create_block("entry");

let mut closure_boxed_vars = module_boxed_vars.clone();
let mut closure_boxed_vars: HashSet<u32> = closure_relevant_ids
.iter()
.filter(|id| module_boxed_vars.contains(id))
.copied()
.collect();
super::arguments::add_arguments_mapped_boxes(params, &mut closure_boxed_vars);

// Allocate slots for the closure's own params (captures don't get
Expand Down Expand Up @@ -645,8 +664,10 @@ pub(super) fn compile_closure(
// typed fast path and return undefined.
let mut local_types: HashMap<u32, perry_hir::types::Type> =
params.iter().map(|p| (p.id, p.ty.clone())).collect();
for (id, ty) in module_receiver_types.iter() {
local_types.entry(*id).or_insert_with(|| ty.clone());
for id in &closure_relevant_ids {
if let Some(ty) = module_receiver_types.get(id) {
local_types.entry(*id).or_insert_with(|| ty.clone());
}
}

// Build the capture map: each captured LocalId gets the index it
Expand All @@ -668,17 +689,13 @@ pub(super) fn compile_closure(
.filter(|id| !module_globals.contains_key(id))
.collect();
{
let mut referenced: std::collections::HashSet<u32> = std::collections::HashSet::new();
collect_ref_ids_in_stmts(body, &mut referenced);
let mut inner_lets: std::collections::HashSet<u32> = std::collections::HashSet::new();
collect_let_ids(body, &mut inner_lets);
let param_ids: std::collections::HashSet<u32> = params.iter().map(|p| p.id).collect();
let already: std::collections::HashSet<u32> = auto_captures.iter().copied().collect();
let mut sorted: Vec<u32> = referenced.into_iter().collect();
let mut sorted: Vec<u32> = closure_referenced_ids.iter().copied().collect();
sorted.sort();
for id in sorted {
if !param_ids.contains(&id)
&& !inner_lets.contains(&id)
&& !closure_declared_ids.contains(&id)
&& !already.contains(&id)
&& !module_globals.contains_key(&id)
{
Expand Down Expand Up @@ -837,7 +854,11 @@ pub(super) fn compile_closure(
std::collections::HashSet::new()
};

let mut reassigned_locals = module_reassigned_locals.clone();
let mut reassigned_locals: HashSet<u32> = closure_relevant_ids
.iter()
.filter(|id| module_reassigned_locals.contains(id))
.copied()
.collect();
reassigned_locals.extend(crate::collectors::reassigned_locals(body));

// #7055: spill the closure's own `%this_closure` pointer into a
Expand Down
60 changes: 49 additions & 11 deletions crates/perry-codegen/src/codegen/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -491,27 +491,65 @@ pub(crate) fn decide_full_outline_ic(callable_count: usize) -> bool {
/// 13MB bundle); splitting bounds peak compiler memory to roughly whole/N.
///
/// `PERRY_CODEGEN_UNITS=N` forces exactly N units (1 disables splitting).
/// Otherwise auto: 1 unit until the module's callable count crosses a floor,
/// then `ceil(callables / target_per_unit)`, capped — so ordinary per-file
/// modules stay on the single-unit path (default 1, zero behavior change).
/// `PERRY_CODEGEN_UNIT_SIZE` overrides the target callables-per-unit.
pub(crate) fn decide_codegen_units(callable_count: usize) -> usize {
/// Otherwise auto: choose the larger of the callable-count estimate and the
/// post-lowering generated-IR estimate. The latter matters for generated and
/// minified bundles: one HIR callable can expand into many large helper/wrapper
/// bodies, so callable count alone left 100+ MiB LLVM modules unsplit.
///
/// `PERRY_CODEGEN_UNIT_SIZE` overrides callables/unit;
/// `PERRY_CODEGEN_UNIT_BYTES` overrides generated IR bytes/unit.
pub(crate) fn decide_codegen_units(callable_count: usize, estimated_ir_bytes: usize) -> usize {
if let Ok(v) = std::env::var("PERRY_CODEGEN_UNITS") {
if let Ok(n) = v.parse::<usize>() {
return n.max(1);
}
}
const MIN_CALLABLES_TO_SPLIT: usize = 8000;
const MAX_UNITS: usize = 48;
let target = std::env::var("PERRY_CODEGEN_UNIT_SIZE")
const MIN_CALLABLES_TO_SPLIT: usize = 8_000;
// Real-app calibration (OpenCode's split CLI): the estimator reports only
// function bodies, while LLVM also receives globals, declarations,
// attributes, and metadata. Modules estimated just below the old 48 MiB
// gate therefore reached LLVM as 44--46 MiB single units. Late in a large
// build, with the collected program HIR and cached-object bookkeeping
// resident, two such units expanded the process/pagefile until C: had less
// than 1 GiB free. Start splitting at 16 MiB of estimated function IR and
// require at least two units once the gate is crossed; the 20 MiB target
// remains the balancing goal for larger modules.
const MIN_IR_BYTES_TO_SPLIT: usize = 16 * 1024 * 1024;
const DEFAULT_IR_BYTES_PER_UNIT: usize = 20 * 1024 * 1024;
const MAX_UNITS: usize = 128;
let target_callables = std::env::var("PERRY_CODEGEN_UNIT_SIZE")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.filter(|&n| n > 0)
.unwrap_or(6000);
if callable_count < MIN_CALLABLES_TO_SPLIT {
return 1;
let target_ir_bytes = std::env::var("PERRY_CODEGEN_UNIT_BYTES")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.filter(|&n| n > 0)
.unwrap_or(DEFAULT_IR_BYTES_PER_UNIT);
let by_callables = if callable_count >= MIN_CALLABLES_TO_SPLIT {
callable_count.div_ceil(target_callables)
} else {
1
};
let by_ir = if estimated_ir_bytes >= MIN_IR_BYTES_TO_SPLIT {
estimated_ir_bytes.div_ceil(target_ir_bytes).max(2)
} else {
1
};
by_callables.max(by_ir).clamp(1, MAX_UNITS)
}

#[cfg(test)]
mod codegen_unit_tests {
use super::decide_codegen_units;

#[test]
fn splits_medium_generated_modules_before_the_llvm_memory_cliff() {
assert_eq!(decide_codegen_units(800, 15 * 1024 * 1024), 1);
assert_eq!(decide_codegen_units(800, 16 * 1024 * 1024), 2);
assert_eq!(decide_codegen_units(800, 45 * 1024 * 1024), 3);
}
callable_count.div_ceil(target).clamp(1, MAX_UNITS)
}

pub(super) fn scoped_fn_name(module_prefix: &str, hir_name: &str) -> String {
Expand Down
Loading
Loading