From b2bbb47a1867250b75398d5eae6f28a7408eefa4 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Sun, 28 Jun 2026 18:56:09 +0800 Subject: [PATCH 01/23] ssa: emit DCE-safe function metadata --- cl/cltest/cltest.go | 12 ++- cl/compile.go | 8 ++ cl/funcinfo_metadata_test.go | 142 +++++++++++++++++++++++++++++++++++ ssa/funcinfo.go | 63 ++++++++++++++++ ssa/package.go | 2 + ssa/ssa_test.go | 98 ++++++++++++++++++++++++ 6 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 cl/funcinfo_metadata_test.go create mode 100644 ssa/funcinfo.go diff --git a/cl/cltest/cltest.go b/cl/cltest/cltest.go index 51f199bf94..d70ab47672 100644 --- a/cl/cltest/cltest.go +++ b/cl/cltest/cltest.go @@ -540,7 +540,7 @@ func filterRunOutput(in []byte) []byte { return out.Bytes() } -func TestCompileEx(t *testing.T, src any, fname, expected string, dbg bool) { +func CompileIREx(t *testing.T, src any, fname string, dbg bool, configure func(llssa.Program)) string { t.Helper() fset := token.NewFileSet() f, err := parser.ParseFile(fset, fname, src, parser.ParseComments) @@ -563,13 +563,21 @@ func TestCompileEx(t *testing.T, src any, fname, expected string, dbg bool) { foo.WriteTo(os.Stderr) prog := ssatest.NewProgramEx(t, nil, imp) prog.TypeSizes(types.SizesFor("gc", runtime.GOARCH)) + if configure != nil { + configure(prog) + } ret, err := cl.NewPackage(prog, foo, files) if err != nil { t.Fatal("cl.NewPackage failed:", err) } + return ret.String() +} - if v := ret.String(); llssa.StripModuleTarget(v) != expected && expected != ";" { // expected == ";" means skipping out.ll +func TestCompileEx(t *testing.T, src any, fname, expected string, dbg bool) { + t.Helper() + v := CompileIREx(t, src, fname, dbg, nil) + if llssa.StripModuleTarget(v) != expected && expected != ";" { // expected == ";" means skipping out.ll t.Fatalf("\n==> got:\n%s\n==> expected:\n%s\n", v, expected) } } diff --git a/cl/compile.go b/cl/compile.go index 434a25d773..4de05558d0 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -476,6 +476,14 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun p.funcs[f] = fn isCgo := isCgoExternSymbol(f) if nblk := len(f.Blocks); nblk > 0 { + if p.prog.FuncInfoMetadataEnabled() { + goName := fn.Name() + if pkgTypes != nil { + goName = funcName(pkgTypes, f, false) + } + pos := p.goProg.Fset.Position(f.Pos()) + pkg.EmitFuncInfo(fn.Name(), goName, pos.Filename, pos.Line, pos.Column) + } var childInits []func() if len(f.AnonFuncs) > 0 { parentInits := p.inits diff --git a/cl/funcinfo_metadata_test.go b/cl/funcinfo_metadata_test.go new file mode 100644 index 0000000000..5b39e3a2cc --- /dev/null +++ b/cl/funcinfo_metadata_test.go @@ -0,0 +1,142 @@ +//go:build !llgo +// +build !llgo + +/* + * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cl_test + +import ( + "regexp" + "strconv" + "strings" + "testing" + + "github.com/goplus/llgo/cl/cltest" + llssa "github.com/goplus/llgo/ssa" +) + +type funcInfoRecord struct { + symbol string + name string + file string + line int + column int +} + +func TestFuncInfoMetadataEmission(t *testing.T) { + const src = `package foo + +type T struct{} + +func top() { + _ = func() int { return leaf() }() +} + +func leaf() int { return 1 } + +func (T) method() {} +` + ir := cltest.CompileIREx(t, src, "foo.go", false, func(prog llssa.Program) { + prog.EnableFuncInfoMetadata(true) + }) + + for _, want := range []string{ + `!llgo.funcinfo = !{!`, + `!"foo.top"`, + `!"foo.top$1"`, + `!"foo.T.method"`, + `!"foo.go"`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("missing funcinfo metadata %s:\n%s", want, ir) + } + } + if strings.Contains(ir, "llvm.compiler.used") { + t.Fatalf("funcinfo metadata should not add llvm.compiler.used:\n%s", ir) + } + if strings.Contains(ir, `ptr @"foo.top"`) || strings.Contains(ir, `ptr @foo.top`) { + t.Fatalf("funcinfo metadata should use symbol strings, not function pointers:\n%s", ir) + } + + records := parseFuncInfoRecords(t, ir) + stackSymbols := []string{"foo.leaf", "foo.top$1", "foo.top"} + for _, symbol := range stackSymbols { + record, ok := records[symbol] + if !ok { + t.Fatalf("stack symbol %q not found in funcinfo metadata: %#v", symbol, records) + } + if record.name == "" || record.file != "foo.go" || record.line <= 0 || record.column <= 0 { + t.Fatalf("bad funcinfo for stack symbol %q: %#v", symbol, record) + } + } + if got := records["foo.leaf"].name; got != "foo.leaf" { + t.Fatalf("leaf stack frame name = %q, want foo.leaf", got) + } + if got := records["foo.top$1"].name; got != "foo.top$1" { + t.Fatalf("closure stack frame name = %q, want foo.top$1", got) + } + if got := records["foo.top"].name; got != "foo.top" { + t.Fatalf("caller stack frame name = %q, want foo.top", got) + } +} + +func parseFuncInfoRecords(t *testing.T, ir string) map[string]funcInfoRecord { + t.Helper() + + listRE := regexp.MustCompile(`!llgo\.funcinfo = !\{([^}]*)\}`) + listMatch := listRE.FindStringSubmatch(ir) + if listMatch == nil { + t.Fatalf("missing funcinfo metadata list:\n%s", ir) + } + refRE := regexp.MustCompile(`!(\d+)`) + refs := refRE.FindAllStringSubmatch(listMatch[1], -1) + if len(refs) == 0 { + t.Fatalf("empty funcinfo metadata list:\n%s", ir) + } + wantRefs := make(map[string]bool, len(refs)) + for _, ref := range refs { + wantRefs[ref[1]] = true + } + + rowRE := regexp.MustCompile(`^!(\d+) = !\{i32 1, !"([^"]+)", !"([^"]+)", !"([^"]*)", i32 ([0-9]+), i32 ([0-9]+)\}$`) + records := make(map[string]funcInfoRecord) + for _, line := range strings.Split(ir, "\n") { + row := rowRE.FindStringSubmatch(line) + if row == nil || !wantRefs[row[1]] { + continue + } + lineNo, err := strconv.Atoi(row[5]) + if err != nil { + t.Fatalf("bad funcinfo line in %q: %v", line, err) + } + column, err := strconv.Atoi(row[6]) + if err != nil { + t.Fatalf("bad funcinfo column in %q: %v", line, err) + } + records[row[2]] = funcInfoRecord{ + symbol: row[2], + name: row[3], + file: row[4], + line: lineNo, + column: column, + } + } + if len(records) != len(wantRefs) { + t.Fatalf("parsed %d funcinfo records, want %d:\n%s", len(records), len(wantRefs), ir) + } + return records +} diff --git a/ssa/funcinfo.go b/ssa/funcinfo.go new file mode 100644 index 0000000000..734399093d --- /dev/null +++ b/ssa/funcinfo.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ssa + +import "github.com/xgo-dev/llvm" + +const ( + FuncInfoMetadataName = "llgo.funcinfo" + funcInfoVersion = 1 +) + +// EnableFuncInfoMetadata controls emission of DCE-safe function source +// metadata. The metadata intentionally stores symbol names as strings instead +// of function pointer operands, so it can be consumed before materializing a +// final runtime line/func table without keeping otherwise-dead functions alive. +func (p Program) EnableFuncInfoMetadata(enable bool) { + p.enableFuncInfoMetadata = enable +} + +func (p Program) FuncInfoMetadataEnabled() bool { + return p.enableFuncInfoMetadata +} + +// EmitFuncInfo records a function's linker symbol, Go name, and declaration +// source position as LLVM named metadata. The row layout is: +// +// !{i32 version, !"symbol", !"go.name", !"file", i32 line, i32 column} +func (p Package) EmitFuncInfo(symbol, name, file string, line, column int) { + if symbol == "" { + return + } + if line < 0 { + line = 0 + } + if column < 0 { + column = 0 + } + i32 := p.Prog.Int32().ll + p.mod.AddNamedMetadataOperand(FuncInfoMetadataName, + p.Prog.ctx.MDNode([]llvm.Metadata{ + llvm.ConstInt(i32, funcInfoVersion, false).ConstantAsMetadata(), + p.Prog.ctx.MDString(symbol), + p.Prog.ctx.MDString(name), + p.Prog.ctx.MDString(file), + llvm.ConstInt(i32, uint64(line), false).ConstantAsMetadata(), + llvm.ConstInt(i32, uint64(column), false).ConstantAsMetadata(), + }), + ) +} diff --git a/ssa/package.go b/ssa/package.go index 74256eddcc..f0f118d639 100644 --- a/ssa/package.go +++ b/ssa/package.go @@ -233,6 +233,8 @@ type aProgram struct { is32Bits bool enableGoGlobalDCE bool + + enableFuncInfoMetadata bool } type AbiSymbol struct { diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 74d4242286..378bde70ff 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -200,6 +200,104 @@ func TestNewFuncExLLVMUsed(t *testing.T) { } } +func TestFuncInfoMetadataDoesNotPreserveFunctions(t *testing.T) { + testFuncInfoMetadataDoesNotPreserveFunctions(t) +} + +func testFuncInfoMetadataDoesNotPreserveFunctions(t *testing.T) { + t.Helper() + + prog := NewProgram(nil) + if prog.FuncInfoMetadataEnabled() { + t.Fatal("funcinfo metadata should be disabled by default") + } + prog.EnableFuncInfoMetadata(true) + if !prog.FuncInfoMetadataEnabled() { + t.Fatal("funcinfo metadata should be enabled") + } + + pkg := prog.NewPackage("main", "main") + sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) + + pkg.NewFunc("main.unused", sig, InGo) + pkg.EmitFuncInfo("", "ignored", "ignored.go", -1, -1) + if ir := pkg.String(); strings.Contains(ir, FuncInfoMetadataName) { + t.Fatalf("empty symbol should not emit funcinfo metadata:\n%s", ir) + } + + pkg.EmitFuncInfo("main.unused", "main.unused", "unused.go", 7, 1) + pkg.EmitFuncInfo("main.negative", "main.negative", "negative.go", -7, -1) + ir := pkg.String() + + if !strings.Contains(ir, `!llgo.funcinfo = !{!`) { + t.Fatalf("missing %s metadata:\n%s", FuncInfoMetadataName, ir) + } + for _, want := range []string{`!"main.unused"`, `!"unused.go"`, `i32 7`, `!"main.negative"`, `!"negative.go"`, `i32 0`} { + if !strings.Contains(ir, want) { + t.Fatalf("missing funcinfo field %s:\n%s", want, ir) + } + } + if strings.Contains(ir, "llvm.compiler.used") { + t.Fatalf("funcinfo metadata must not preserve symbols with llvm.compiler.used:\n%s", ir) + } + if strings.Contains(ir, `ptr @"main.unused"`) || strings.Contains(ir, `ptr @main.unused`) { + t.Fatalf("funcinfo metadata must not contain function pointer operands:\n%s", ir) + } +} + +func TestFuncInfoMetadataDoesNotBlockGlobalDCE(t *testing.T) { + testFuncInfoMetadataDoesNotBlockGlobalDCE(t) +} + +func testFuncInfoMetadataDoesNotBlockGlobalDCE(t *testing.T) { + t.Helper() + + prog := NewProgram(nil) + pkg := prog.NewPackage("main", "main") + sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) + + live := pkg.NewFunc("main.main", sig, InGo) + lb := live.MakeBody(1) + lb.Return() + lb.EndBuild() + + unused := pkg.NewFuncEx("main.unused", sig, InGo, false, true) + ub := unused.MakeBody(1) + ub.Return() + ub.EndBuild() + pkg.EmitFuncInfo(unused.Name(), unused.Name(), "unused.go", 7, 1) + + mod := pkg.Module() + if mod.NamedFunction("main.unused").IsNil() { + t.Fatal("missing main.unused before DCE") + } + mod.SetDataLayout(prog.DataLayout()) + mod.SetTarget(prog.Target().Spec().Triple) + pbo := llvm.NewPassBuilderOptions() + defer pbo.Dispose() + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("verify module before DCE: %v", err) + } + if err := mod.RunPasses("globaldce", prog.TargetMachine(), pbo); err != nil { + t.Fatalf("run globaldce: %v", err) + } + if !mod.NamedFunction("main.unused").IsNil() { + t.Fatalf("funcinfo metadata kept main.unused alive:\n%s", mod.String()) + } + if mod.NamedFunction("main.main").IsNil() { + t.Fatalf("globaldce removed externally visible live function:\n%s", mod.String()) + } + if ir := mod.String(); !strings.Contains(ir, `!"main.unused"`) { + t.Fatalf("funcinfo metadata should remain available for later materialization:\n%s", ir) + } +} + +func TestDevLTOGlobalDCEFuncInfoMetadata(t *testing.T) { + requireGoGlobalDCE(t) + testFuncInfoMetadataDoesNotPreserveFunctions(t) + testFuncInfoMetadataDoesNotBlockGlobalDCE(t) +} + func requireGoGlobalDCE(t *testing.T) { t.Helper() } From 94b2350786a7fcef327238249d7f86f0a71a59d3 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Mon, 29 Jun 2026 16:08:50 +0800 Subject: [PATCH 02/23] runtime: add line info for stack frames --- cl/cltest/cltest.go | 35 ++- cl/compile.go | 40 ++- cl/funcinfo_metadata_test.go | 17 ++ internal/build/build.go | 23 +- internal/build/build_test.go | 43 +++ internal/build/collect.go | 1 + internal/build/funcinfo/funcinfo.go | 156 ++++++++++ internal/build/funcinfo/funcinfo_test.go | 134 +++++++++ internal/build/funcinfo_table.go | 223 ++++++++++++++ internal/build/funcinfo_table_test.go | 182 +++++++++++ internal/build/main_module.go | 2 + runtime/internal/clite/debug/_wrap/debug.c | 17 +- runtime/internal/lib/runtime/extern.go | 19 +- .../lib/runtime/pprof_runtime_stub_llgo.go | 14 +- runtime/internal/lib/runtime/runtime2.go | 50 ++- runtime/internal/lib/runtime/symtab.go | 284 +++++++++++++++++- ssa/decl.go | 5 + test/go/runtime_lineinfo_stack_test.go | 187 ++++++++++++ test/goroot/xfail.yaml | 4 - 19 files changed, 1403 insertions(+), 33 deletions(-) create mode 100644 internal/build/funcinfo/funcinfo.go create mode 100644 internal/build/funcinfo/funcinfo_test.go create mode 100644 internal/build/funcinfo_table.go create mode 100644 internal/build/funcinfo_table_test.go create mode 100644 test/go/runtime_lineinfo_stack_test.go diff --git a/cl/cltest/cltest.go b/cl/cltest/cltest.go index d70ab47672..6a151f67ef 100644 --- a/cl/cltest/cltest.go +++ b/cl/cltest/cltest.go @@ -242,7 +242,10 @@ func testFrom(t *testing.T, pkgDir, sel string) { if spec.Mode == littest.ModeSkip { return } - v := llgen.GenFrom(pkgDir) + var v string + withFuncInfoDisabled(func() { + v = llgen.GenFrom(pkgDir) + }) if spec.Mode == littest.ModeFileCheck { if err := littest.Check(spec, v); err != nil { _ = os.WriteFile(pkgDir+"/result.txt", []byte(v), 0644) @@ -294,7 +297,14 @@ func testRunAndTestFrom(t *testing.T, pkgDir, relPkg, sel string, opts runOption } } - output, err := runWithConf(relPkg, pkgDir, conf) + var output []byte + if checkIR { + withFuncInfoDisabled(func() { + output, err = runWithConf(relPkg, pkgDir, conf) + }) + } else { + output, err = runWithConf(relPkg, pkgDir, conf) + } if err != nil { t.Logf("raw output:\n%s", string(output)) t.Fatalf("run failed: %v\noutput: %s", err, string(output)) @@ -509,6 +519,20 @@ func readIRSpec(pkgDir string) (littest.Spec, bool, error) { return spec, true, nil } +func withFuncInfoDisabled(fn func()) { + const key = "LLGO_FUNCINFO" + old, ok := os.LookupEnv(key) + _ = os.Setenv(key, "0") + defer func() { + if ok { + _ = os.Setenv(key, old) + } else { + _ = os.Unsetenv(key) + } + }() + fn() +} + func filterRunOutput(in []byte) []byte { // Tests compare output with expect.txt. Some toolchain/environment warnings are // inherently machine-specific and should not be part of the golden output. @@ -542,6 +566,13 @@ func filterRunOutput(in []byte) []byte { func CompileIREx(t *testing.T, src any, fname string, dbg bool, configure func(llssa.Program)) string { t.Helper() + // Build.Do configures cl debug globals for full-package builds. Keep the + // single-file compiler assertions independent from any prior build test. + cl.EnableDebug(dbg) + cl.EnableDbgSyms(dbg) + defer cl.EnableDebug(false) + defer cl.EnableDbgSyms(false) + fset := token.NewFileSet() f, err := parser.ParseFile(fset, fname, src, parser.ParseComments) if err != nil { diff --git a/cl/compile.go b/cl/compile.go index 4de05558d0..fe1dbfc8b1 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -469,9 +469,14 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun } if fn == nil { fn = pkg.NewFuncEx(name, sig, llssa.Background(ftype), hasCtx, p.needsLinkOnce(f)) - if disableInline { - fn.Inline(llssa.NoInline) - } + } + noInlineDirective := hasNoInlineDirective(f) + runtimeStackNoInline := needsRuntimeStackNoInline(pkgTypes, f) + if disableInline || noInlineDirective || runtimeStackNoInline { + fn.Inline(llssa.NoInline) + } + if noInlineDirective || runtimeStackNoInline { + fn.DisableTailCalls() } p.funcs[f] = fn isCgo := isCgoExternSymbol(f) @@ -568,6 +573,35 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun return fn, nil, goFunc } +func hasNoInlineDirective(f *ssa.Function) bool { + decl, _ := f.Syntax().(*ast.FuncDecl) + if decl == nil || decl.Doc == nil { + return false + } + for _, c := range decl.Doc.List { + if c.Text == "//go:noinline" { + return true + } + } + return false +} + +func needsRuntimeStackNoInline(pkg *types.Package, f *ssa.Function) bool { + if pkg == nil || f == nil || f.Signature.Recv() != nil { + return false + } + switch pkg.Path() { + case "runtime", "github.com/goplus/llgo/runtime/internal/lib/runtime": + switch f.Name() { + case "Caller", "Callers", "callers": + return true + } + case "github.com/goplus/llgo/runtime/internal/clite/debug": + return f.Name() == "StackTrace" + } + return false +} + func (p *context) getFuncBodyPos(f *ssa.Function) token.Position { if f.Object() != nil { if fn, ok := f.Object().(*types.Func); ok && fn.Scope() != nil { diff --git a/cl/funcinfo_metadata_test.go b/cl/funcinfo_metadata_test.go index 5b39e3a2cc..5319b16751 100644 --- a/cl/funcinfo_metadata_test.go +++ b/cl/funcinfo_metadata_test.go @@ -94,6 +94,23 @@ func (T) method() {} } } +func TestNoInlineDirectiveDisablesTailCalls(t *testing.T) { + const src = `package foo + +func caller() { callee() } + +//go:noinline +func callee() {} +` + ir := cltest.CompileIREx(t, src, "foo.go", false, nil) + if !strings.Contains(ir, `define void @foo.callee()`) { + t.Fatalf("missing callee function:\n%s", ir) + } + if !strings.Contains(ir, `noinline`) || !strings.Contains(ir, `"disable-tail-calls"="true"`) { + t.Fatalf("callee should disable inlining and tail calls:\n%s", ir) + } +} + func parseFuncInfoRecords(t *testing.T, ir string) map[string]funcInfoRecord { t.Helper() diff --git a/internal/build/build.go b/internal/build/build.go index 149411815f..93041b7406 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -318,6 +318,7 @@ func Do(args []string, conf *Config) ([]Package, error) { prog := llssa.NewProgram(target) prog.EnableGoGlobalDCE(conf.goGlobalDCEEnabled()) + prog.EnableFuncInfoMetadata(conf.Mode != ModeGen && IsFuncInfoEnabled()) sizes := func(sizes types.Sizes, compiler, arch string) types.Sizes { if arch == "wasm" { sizes = &types.StdSizes{WordSize: 4, MaxAlign: 4} @@ -1050,6 +1051,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa methodByIndex: methodByIndex, methodByName: methodByName, abiSymbols: linkedModuleGlobals(linkedOrder), + funcInfo: prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil), }) entryObjFile, err := exportObject(ctx, "entry_main", entryPkg.ExportFile, entryPkg.LPkg) if err != nil { @@ -1130,9 +1132,9 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose if needsLinuxNoPIE(ctx, linkArgs) { buildArgs = append(buildArgs, "-no-pie") } + buildArgs = append(buildArgs, linuxExportDynamicArgs(ctx)...) } - // Add common linker arguments based on target OS and architecture if IsDbgSymsEnabled() { buildArgs = append(buildArgs, "-gdwarf-4") } @@ -1178,6 +1180,20 @@ func needsLinuxNoPIE(ctx *context, linkArgs []string) bool { return true } +func needsLinuxExportDynamic(ctx *context) bool { + return ctx.buildConf.Target == "" && ctx.buildConf.Goos == "linux" && IsFuncInfoEnabled() +} + +func linuxExportDynamicArgs(ctx *context) []string { + if !needsLinuxExportDynamic(ctx) { + return nil + } + return []string{ + "-Wl,--export-dynamic-symbol=main.*", + "-Wl,--export-dynamic-symbol=command-line-arguments.*", + } +} + // archiver returns the archiving tool to use for the current context. // For wasm targets and LTO builds, it prefers llvm-ar because linkers need // LLVM-aware archive indexes for wasm objects and bitcode members. @@ -1796,6 +1812,7 @@ var ( const llgoDebug = "LLGO_DEBUG" const llgoDbgSyms = "LLGO_DEBUG_SYMBOLS" +const llgoFuncInfo = "LLGO_FUNCINFO" const llgoTrace = "LLGO_TRACE" const llgoOptimize = "LLGO_OPTIMIZE" const llgoWasmRuntime = "LLGO_WASM_RUNTIME" @@ -1843,6 +1860,10 @@ func IsDbgEnabled() bool { return isEnvOn(llgoDebug, false) || isEnvOn(llgoDbgSyms, false) } +func IsFuncInfoEnabled() bool { + return isEnvOn(llgoFuncInfo, true) +} + func IsDbgSymsEnabled() bool { return isEnvOn(llgoDbgSyms, false) } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index 51401c2131..bc6f89d785 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -55,6 +55,49 @@ func TestNeedsLinuxNoPIE(t *testing.T) { } } +func TestNeedsLinuxExportDynamic(t *testing.T) { + t.Setenv(llgoFuncInfo, "") + ctx := &context{buildConf: &Config{Goos: "linux"}} + if !needsLinuxExportDynamic(ctx) { + t.Fatal("linux funcinfo executable should export dynamic symbols") + } + if got := linuxExportDynamicArgs(ctx); strings.Join(got, " ") != "-Wl,--export-dynamic-symbol=main.* -Wl,--export-dynamic-symbol=command-line-arguments.*" { + t.Fatalf("linuxExportDynamicArgs = %v", got) + } + t.Setenv(llgoFuncInfo, "0") + if needsLinuxExportDynamic(ctx) { + t.Fatal("LLGO_FUNCINFO=0 should disable dynamic symbol export") + } + if got := linuxExportDynamicArgs(ctx); got != nil { + t.Fatalf("disabled linuxExportDynamicArgs = %v, want nil", got) + } + t.Setenv(llgoFuncInfo, "1") + ctx.buildConf.Goos = "darwin" + if needsLinuxExportDynamic(ctx) { + t.Fatal("non-linux executable should not export dynamic symbols for funcinfo") + } + ctx.buildConf.Goos = "linux" + ctx.buildConf.Target = "wasi" + if needsLinuxExportDynamic(ctx) { + t.Fatal("named targets should not force host linux dynamic symbol export") + } +} + +func TestIsFuncInfoEnabled(t *testing.T) { + t.Setenv(llgoFuncInfo, "") + if !IsFuncInfoEnabled() { + t.Fatal("funcinfo should be enabled by default") + } + t.Setenv(llgoFuncInfo, "0") + if IsFuncInfoEnabled() { + t.Fatal("LLGO_FUNCINFO=0 should disable funcinfo") + } + t.Setenv(llgoFuncInfo, "1") + if !IsFuncInfoEnabled() { + t.Fatal("LLGO_FUNCINFO=1 should enable funcinfo") + } +} + func mockRun(args []string, cfg *Config) { defer mockable.DisableMock() mockable.EnableMock() diff --git a/internal/build/collect.go b/internal/build/collect.go index ab6d7076f6..dd30daf72b 100644 --- a/internal/build/collect.go +++ b/internal/build/collect.go @@ -82,6 +82,7 @@ func (c *context) collectEnvInputs(m *manifestBuilder) { envVars := []string{ llgoDebug, llgoDbgSyms, + llgoFuncInfo, llgoTrace, llgoOptimize, llgoWasmRuntime, diff --git a/internal/build/funcinfo/funcinfo.go b/internal/build/funcinfo/funcinfo.go new file mode 100644 index 0000000000..c092b606e9 --- /dev/null +++ b/internal/build/funcinfo/funcinfo.go @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package funcinfo + +import ( + "fmt" + "math" + "sort" + "strings" +) + +type Record struct { + Symbol string + Name string + File string + Line uint32 + Column uint32 +} + +type EncodedRecord struct { + Symbol uint32 + Name uint32 + File uint32 + Line uint32 + Column uint32 +} + +type Table struct { + Records []EncodedRecord + Strings []byte + Hash []uint32 +} + +func Encode(records []Record) (Table, error) { + if len(records) == 0 { + return Table{}, nil + } + pool := stringPool{ + offsets: map[string]uint32{"": 0}, + data: []byte{0}, + text: "\x00", + } + for _, s := range collectStrings(records) { + if _, err := pool.offset(s); err != nil { + return Table{}, err + } + } + out := Table{ + Records: make([]EncodedRecord, 0, len(records)), + } + for _, rec := range records { + out.Records = append(out.Records, EncodedRecord{ + Symbol: pool.offsets[rec.Symbol], + Name: pool.offsets[rec.Name], + File: pool.offsets[rec.File], + Line: rec.Line, + Column: rec.Column, + }) + } + out.Strings = pool.data + out.Hash = buildHash(records) + return out, nil +} + +func collectStrings(records []Record) []string { + seen := make(map[string]bool) + for _, rec := range records { + seen[rec.Symbol] = true + seen[rec.Name] = true + seen[rec.File] = true + } + delete(seen, "") + out := make([]string, 0, len(seen)) + for s := range seen { + out = append(out, s) + } + sort.Slice(out, func(i, j int) bool { + if len(out[i]) != len(out[j]) { + return len(out[i]) > len(out[j]) + } + return out[i] < out[j] + }) + return out +} + +type stringPool struct { + offsets map[string]uint32 + data []byte + text string +} + +func (p *stringPool) offset(s string) (uint32, error) { + if off, ok := p.offsets[s]; ok { + return off, nil + } + if off := strings.Index(p.text, s+"\x00"); off >= 0 { + uoff := uint32(off) + p.offsets[s] = uoff + return uoff, nil + } + if len(p.data)+len(s)+1 > math.MaxUint32 { + return 0, fmt.Errorf("funcinfo string table exceeds 4 GiB") + } + off := uint32(len(p.data)) + p.data = append(p.data, s...) + p.data = append(p.data, 0) + p.text = string(p.data) + p.offsets[s] = off + return off, nil +} + +func buildHash(records []Record) []uint32 { + if len(records) == 0 { + return nil + } + buckets := 1 + for buckets*3 < len(records)*4 { + buckets <<= 1 + } + hash := make([]uint32, buckets) + for i, rec := range records { + slot := int(HashString(rec.Symbol) & uint32(buckets-1)) + for hash[slot] != 0 { + slot = (slot + 1) & (buckets - 1) + } + hash[slot] = uint32(i + 1) + } + return hash +} + +func HashString(s string) uint32 { + const ( + offset = uint32(2166136261) + prime = uint32(16777619) + ) + h := offset + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= prime + } + return h +} diff --git a/internal/build/funcinfo/funcinfo_test.go b/internal/build/funcinfo/funcinfo_test.go new file mode 100644 index 0000000000..7bc92ec8b1 --- /dev/null +++ b/internal/build/funcinfo/funcinfo_test.go @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package funcinfo + +import "testing" + +func TestEncodePoolsStringsAndBuildsHash(t *testing.T) { + table, err := Encode([]Record{ + {Symbol: "example.com/p.a", Name: "example.com/p.A", File: "/src/p/shared.go", Line: 10, Column: 1}, + {Symbol: "example.com/p.b", Name: "example.com/p.B", File: "shared.go", Line: 20, Column: 2}, + }) + if err != nil { + t.Fatal(err) + } + if len(table.Records) != 2 { + t.Fatalf("encoded records = %d, want 2", len(table.Records)) + } + if table.Records[0].File == table.Records[1].File { + t.Fatalf("suffix sharing should not collapse distinct file strings to the same offset") + } + if got := cstring(table.Strings, table.Records[1].File); got != "shared.go" { + t.Fatalf("suffix file string = %q, want shared.go", got) + } + if len(table.Hash) == 0 || len(table.Hash)&(len(table.Hash)-1) != 0 { + t.Fatalf("hash bucket count = %d, want power-of-two non-zero", len(table.Hash)) + } + if idx, ok := lookup(table, "example.com/p.a"); !ok || idx != 0 { + t.Fatalf("lookup a = %d, %v; want 0, true", idx, ok) + } + if idx, ok := lookup(table, "example.com/p.b"); !ok || idx != 1 { + t.Fatalf("lookup b = %d, %v; want 1, true", idx, ok) + } + if _, ok := lookup(table, "missing"); ok { + t.Fatalf("lookup missing succeeded") + } +} + +func TestEncodeUsesUint32Records(t *testing.T) { + table, err := Encode([]Record{{Symbol: "s", Name: "n", File: "f", Line: 1, Column: 2}}) + if err != nil { + t.Fatal(err) + } + if got, want := len(table.Records), 1; got != want { + t.Fatalf("records = %d, want %d", got, want) + } + rec := table.Records[0] + if got, want := cstring(table.Strings, rec.Symbol), "s"; got != want { + t.Fatalf("symbol = %q, want %q", got, want) + } + if got, want := cstring(table.Strings, rec.Name), "n"; got != want { + t.Fatalf("name = %q, want %q", got, want) + } + if got, want := cstring(table.Strings, rec.File), "f"; got != want { + t.Fatalf("file = %q, want %q", got, want) + } + if rec.Line != 1 || rec.Column != 2 { + t.Fatalf("source position = %d:%d, want 1:2", rec.Line, rec.Column) + } +} + +func TestEncodeHashHandlesCollisions(t *testing.T) { + a, b := collisionPair(t) + table, err := Encode([]Record{ + {Symbol: a, Name: a, File: "a.go"}, + {Symbol: b, Name: b, File: "b.go"}, + }) + if err != nil { + t.Fatal(err) + } + if idx, ok := lookup(table, a); !ok || idx != 0 { + t.Fatalf("lookup collision a = %d, %v; want 0, true", idx, ok) + } + if idx, ok := lookup(table, b); !ok || idx != 1 { + t.Fatalf("lookup collision b = %d, %v; want 1, true", idx, ok) + } +} + +func collisionPair(t *testing.T) (string, string) { + t.Helper() + const mask = uint32(3) + seen := make(map[uint32]string) + for i := 0; i < 100; i++ { + s := string(rune('a' + i)) + slot := HashString(s) & mask + if prev, ok := seen[slot]; ok { + return prev, s + } + seen[slot] = s + } + t.Fatal("failed to find hash collision") + return "", "" +} + +func cstring(data []byte, off uint32) string { + end := int(off) + for end < len(data) && data[end] != 0 { + end++ + } + return string(data[off:end]) +} + +func lookup(table Table, symbol string) (int, bool) { + if len(table.Hash) == 0 { + return 0, false + } + mask := uint32(len(table.Hash) - 1) + slot := HashString(symbol) & mask + for probes := 0; probes < len(table.Hash); probes++ { + idx := table.Hash[slot] + if idx == 0 { + return 0, false + } + rec := table.Records[idx-1] + if cstring(table.Strings, rec.Symbol) == symbol { + return int(idx - 1), true + } + slot = (slot + 1) & mask + } + return 0, false +} diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go new file mode 100644 index 0000000000..30c63c5fe4 --- /dev/null +++ b/internal/build/funcinfo_table.go @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "sort" + + "github.com/xgo-dev/llvm" + + buildfuncinfo "github.com/goplus/llgo/internal/build/funcinfo" + llssa "github.com/goplus/llgo/ssa" +) + +const ( + funcInfoTableSymbol = "__llgo_funcinfo_table" + funcInfoCountSymbol = "__llgo_funcinfo_count" + funcInfoStringsSymbol = "__llgo_funcinfo_strings" + funcInfoHashSymbol = "__llgo_funcinfo_hash" + funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" + funcInfoDataSymbol = "__llgo_funcinfo_table$data" + funcInfoStringsDataSymbol = "__llgo_funcinfo_strings$data" + funcInfoHashDataSymbol = "__llgo_funcinfo_hash$data" +) + +type funcInfoRecord struct { + symbol string + name string + file string + line uint32 + column uint32 +} + +func collectFuncInfo(pkgs []Package) []funcInfoRecord { + seen := make(map[string]funcInfoRecord) + for _, pkg := range pkgs { + if pkg == nil || pkg.LPkg == nil { + continue + } + for _, rec := range readFuncInfo(pkg.LPkg.Module()) { + if rec.symbol == "" { + continue + } + if _, ok := seen[rec.symbol]; !ok { + seen[rec.symbol] = rec + } + } + } + if len(seen) == 0 { + return nil + } + out := make([]funcInfoRecord, 0, len(seen)) + for _, rec := range seen { + out = append(out, rec) + } + sort.Slice(out, func(i, j int) bool { + return out[i].symbol < out[j].symbol + }) + return out +} + +func prepareFuncInfoTableRecords(records []funcInfoRecord, liveSymbols map[string]none) []funcInfoRecord { + if len(records) == 0 { + return nil + } + // A nil liveSymbols means no post-DCE live symbol set is available yet. + // The current table is still DCE-compatible because it stores only strings, + // never function pointers or llvm.compiler.used references. Once the linker + // or an LTO hook exposes a live-symbol set, pass it here to drop metadata for + // functions removed by global DCE before materializing the runtime table. + if liveSymbols == nil { + return records + } + out := records[:0] + for _, rec := range records { + if _, ok := liveSymbols[rec.symbol]; ok { + out = append(out, rec) + } + } + if len(out) == 0 { + return nil + } + return out +} + +func readFuncInfo(mod llvm.Module) []funcInfoRecord { + rows := mod.NamedMetadataOperands(llssa.FuncInfoMetadataName) + if len(rows) == 0 { + return nil + } + out := make([]funcInfoRecord, 0, len(rows)) + for _, row := range rows { + fields := row.MDNodeOperands() + if len(fields) != 6 || fields[0].ZExtValue() != 1 { + continue + } + if !fields[1].IsAMDString() || !fields[2].IsAMDString() || !fields[3].IsAMDString() { + continue + } + out = append(out, funcInfoRecord{ + symbol: fields[1].MDString(), + name: fields[2].MDString(), + file: fields[3].MDString(), + line: uint32(fields[4].ZExtValue()), + column: uint32(fields[5].ZExtValue()), + }) + } + return out +} + +func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord) { + mod := pkg.Module() + llvmCtx := mod.Context() + i8Type := llvmCtx.Int8Type() + i32Type := llvmCtx.Int32Type() + countType := llvmCtx.IntType(ctx.prog.PointerSize() * 8) + recordType := llvmCtx.StructType([]llvm.Type{ + i32Type, + i32Type, + i32Type, + i32Type, + i32Type, + }, false) + + tablePtr := llvm.AddGlobal(mod, llvm.PointerType(recordType, 0), funcInfoTableSymbol) + stringsPtr := llvm.AddGlobal(mod, llvm.PointerType(i8Type, 0), funcInfoStringsSymbol) + hashPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoHashSymbol) + count := llvm.AddGlobal(mod, countType, funcInfoCountSymbol) + hashMask := llvm.AddGlobal(mod, countType, funcInfoHashMaskSymbol) + if len(records) == 0 { + tablePtr.SetInitializer(llvm.ConstPointerNull(tablePtr.GlobalValueType())) + stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) + hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) + count.SetInitializer(llvm.ConstInt(countType, 0, false)) + hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) + return + } + + encoded, err := buildfuncinfo.Encode(toFuncInfoRecords(records)) + if err != nil { + panic(err) + } + + values := make([]llvm.Value, 0, len(encoded.Records)) + for _, rec := range encoded.Records { + values = append(values, llvm.ConstNamedStruct(recordType, []llvm.Value{ + llvm.ConstInt(i32Type, uint64(rec.Symbol), false), + llvm.ConstInt(i32Type, uint64(rec.Name), false), + llvm.ConstInt(i32Type, uint64(rec.File), false), + llvm.ConstInt(i32Type, uint64(rec.Line), false), + llvm.ConstInt(i32Type, uint64(rec.Column), false), + })) + } + arrayType := llvm.ArrayType(recordType, len(values)) + data := llvm.AddGlobal(mod, arrayType, funcInfoDataSymbol) + data.SetInitializer(llvm.ConstArray(recordType, values)) + data.SetLinkage(llvm.PrivateLinkage) + data.SetGlobalConstant(true) + data.SetUnnamedAddr(true) + data.SetAlignment(4) + + stringArrayType := llvm.ArrayType(i8Type, len(encoded.Strings)) + stringData := llvm.AddGlobal(mod, stringArrayType, funcInfoStringsDataSymbol) + stringData.SetInitializer(llvmCtx.ConstString(string(encoded.Strings), false)) + stringData.SetLinkage(llvm.PrivateLinkage) + stringData.SetGlobalConstant(true) + stringData.SetUnnamedAddr(true) + stringData.SetAlignment(1) + + hashValues := make([]llvm.Value, 0, len(encoded.Hash)) + for _, idx := range encoded.Hash { + hashValues = append(hashValues, llvm.ConstInt(i32Type, uint64(idx), false)) + } + hashArrayType := llvm.ArrayType(i32Type, len(hashValues)) + hashData := llvm.AddGlobal(mod, hashArrayType, funcInfoHashDataSymbol) + hashData.SetInitializer(llvm.ConstArray(i32Type, hashValues)) + hashData.SetLinkage(llvm.PrivateLinkage) + hashData.SetGlobalConstant(true) + hashData.SetUnnamedAddr(true) + hashData.SetAlignment(4) + + tablePtr.SetInitializer(llvm.ConstInBoundsGEP(arrayType, data, []llvm.Value{ + llvm.ConstInt(countType, 0, false), + llvm.ConstInt(countType, 0, false), + })) + stringsPtr.SetInitializer(llvm.ConstInBoundsGEP(stringArrayType, stringData, []llvm.Value{ + llvm.ConstInt(countType, 0, false), + llvm.ConstInt(countType, 0, false), + })) + hashPtr.SetInitializer(llvm.ConstInBoundsGEP(hashArrayType, hashData, []llvm.Value{ + llvm.ConstInt(countType, 0, false), + llvm.ConstInt(countType, 0, false), + })) + count.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Records)), false)) + hashMask.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Hash)-1), false)) +} + +func toFuncInfoRecords(records []funcInfoRecord) []buildfuncinfo.Record { + out := make([]buildfuncinfo.Record, len(records)) + for i, rec := range records { + out[i] = buildfuncinfo.Record{ + Symbol: rec.symbol, + Name: rec.name, + File: rec.file, + Line: rec.line, + Column: rec.column, + } + } + return out +} diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go new file mode 100644 index 0000000000..f7367ebf56 --- /dev/null +++ b/internal/build/funcinfo_table_test.go @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package build + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" + + "github.com/goplus/llgo/internal/packages" + llssa "github.com/goplus/llgo/ssa" +) + +func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) { + prog := llssa.NewProgram(nil) + src := prog.NewPackage("example.com/p", "example.com/p") + src.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "live.go", 17, 3) + src.EmitFuncInfo("example.com/p.live", "example.com/p.LiveDuplicate", "dup.go", 19, 1) + + records := collectFuncInfo([]Package{{LPkg: src}}) + if len(records) != 1 { + t.Fatalf("collectFuncInfo returned %d records, want 1", len(records)) + } + if got := records[0]; got.symbol != "example.com/p.live" || got.name != "example.com/p.Live" || got.file != "live.go" || got.line != 17 || got.column != 3 { + t.Fatalf("unexpected record: %+v", got) + } + + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records}) + ir := entry.LPkg.String() + for _, want := range []string{ + "@__llgo_funcinfo_table = global ptr", + "@__llgo_funcinfo_strings = global ptr", + "@__llgo_funcinfo_hash = global ptr", + "@__llgo_funcinfo_count = global i64 1", + "@__llgo_funcinfo_hash_mask = global i64 1", + `@"__llgo_funcinfo_table$data" = private unnamed_addr constant [1 x { i32, i32, i32, i32, i32 }]`, + `@"__llgo_funcinfo_strings$data" = private unnamed_addr constant [47 x i8]`, + `@"__llgo_funcinfo_hash$data" = private unnamed_addr constant [2 x i32]`, + `example.com/p.live\00`, + `example.com/p.Live\00`, + `live.go\00`, + "i32 17", + "i32 3", + } { + if !strings.Contains(ir, want) { + t.Fatalf("funcinfo table IR missing %q:\n%s", want, ir) + } + } + if strings.Contains(ir, `ptr @"example.com/p.live"`) { + t.Fatalf("funcinfo table must not reference function pointers:\n%s", ir) + } +} + +func TestPrepareFuncInfoTableRecordsFiltersLiveSymbols(t *testing.T) { + records := []funcInfoRecord{ + {symbol: "dead", name: "dead"}, + {symbol: "live", name: "live"}, + } + if got := prepareFuncInfoTableRecords(nil, nil); got != nil { + t.Fatalf("empty records = %+v, want nil", got) + } + if got := prepareFuncInfoTableRecords(records, nil); len(got) != 2 { + t.Fatalf("nil live set kept %d records, want 2", len(got)) + } + got := prepareFuncInfoTableRecords(records, map[string]none{"live": {}}) + if len(got) != 1 || got[0].symbol != "live" { + t.Fatalf("filtered records = %+v, want live only", got) + } + if got := prepareFuncInfoTableRecords(records, map[string]none{}); got != nil { + t.Fatalf("empty live set = %+v, want nil", got) + } +} + +func TestFuncInfoTablePoolsRepeatedStrings(t *testing.T) { + prog := llssa.NewProgram(nil) + records := []funcInfoRecord{ + {symbol: "example.com/p.a", name: "example.com/p.A", file: "shared.go", line: 10}, + {symbol: "example.com/p.b", name: "example.com/p.B", file: "shared.go", line: 20}, + } + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records}) + if got := strings.Count(entry.LPkg.String(), `shared.go\00`); got != 1 { + t.Fatalf("shared file string emitted %d times, want 1", got) + } +} + +func TestFuncInfoTableEmptyDefinitions(t *testing.T) { + prog := llssa.NewProgram(nil) + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{}) + ir := entry.LPkg.String() + for _, want := range []string{ + "@__llgo_funcinfo_table = global ptr null", + "@__llgo_funcinfo_strings = global ptr null", + "@__llgo_funcinfo_hash = global ptr null", + "@__llgo_funcinfo_count = global i64 0", + "@__llgo_funcinfo_hash_mask = global i64 0", + } { + if !strings.Contains(ir, want) { + t.Fatalf("empty funcinfo table IR missing %q:\n%s", want, ir) + } + } +} + +func TestFuncInfoTableIgnoresInvalidMetadata(t *testing.T) { + prog := llssa.NewProgram(nil) + pkg := prog.NewPackage("example.com/p", "example.com/p") + mod := pkg.Module() + ctx := mod.Context() + i32 := ctx.Int32Type() + mdstr := func(s string) llvm.Metadata { return ctx.MDString(s) } + mdint := func(v uint64) llvm.Metadata { + return llvm.ConstInt(i32, v, false).ConstantAsMetadata() + } + add := func(fields ...llvm.Metadata) { + mod.AddNamedMetadataOperand(llssa.FuncInfoMetadataName, ctx.MDNode(fields)) + } + + add(mdstr("short")) + add(mdint(2), mdstr("bad.version"), mdstr("bad.version"), mdstr("bad.go"), mdint(1), mdint(1)) + add(mdint(1), mdint(0), mdstr("bad.symbol"), mdstr("bad.go"), mdint(1), mdint(1)) + add(mdint(1), mdstr(""), mdstr("empty.symbol"), mdstr("empty.go"), mdint(1), mdint(1)) + + if got := readFuncInfo(mod); len(got) != 1 || got[0].symbol != "" { + t.Fatalf("readFuncInfo invalid rows = %+v, want one empty-symbol row", got) + } + if got := collectFuncInfo([]Package{nil, {}, {LPkg: pkg}}); len(got) != 0 { + t.Fatalf("collectFuncInfo invalid rows = %+v, want none", got) + } + + empty := ctx.NewModule("empty") + defer empty.Dispose() + if got := readFuncInfo(empty); got != nil { + t.Fatalf("readFuncInfo(empty) = %+v, want nil", got) + } +} diff --git a/internal/build/main_module.go b/internal/build/main_module.go index d5ac73671e..9f68a976ac 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -43,6 +43,7 @@ type genConfig struct { methodByIndex map[int]none methodByName map[string]none abiSymbols map[string]none + funcInfo []funcInfoRecord } // genMainModule generates the main entry module for an llgo program. @@ -60,6 +61,7 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g argvValueType := prog.Pointer(prog.CStr()) argvVar := mainPkg.NewVarEx("__llgo_argv", prog.Pointer(argvValueType)) argvVar.InitNil() + emitFuncInfoTable(ctx, mainPkg, cfg.funcInfo) exportFile := pkg.ExportFile if exportFile == "" { diff --git a/runtime/internal/clite/debug/_wrap/debug.c b/runtime/internal/clite/debug/_wrap/debug.c index 32d87903bf..cf050c8848 100644 --- a/runtime/internal/clite/debug/_wrap/debug.c +++ b/runtime/internal/clite/debug/_wrap/debug.c @@ -7,6 +7,7 @@ #endif #include +#include #include void *llgo_address() { @@ -14,10 +15,14 @@ void *llgo_address() { } int llgo_addrinfo(void *addr, Dl_info *info) { - return dladdr(addr, info); + int saved_errno = errno; + int ret = dladdr(addr, info); + errno = saved_errno; + return ret; } void llgo_stacktrace(int skip, void *ctx, int (*fn)(void *ctx, void *pc, void *offset, void *sp, char *name)) { + int saved_errno = errno; unw_cursor_t cursor; unw_context_t context; unw_word_t offset, pc, sp; @@ -31,11 +36,17 @@ void llgo_stacktrace(int skip, void *ctx, int (*fn)(void *ctx, void *pc, void *o continue; } if (unw_get_reg(&cursor, UNW_REG_IP, &pc) == 0) { - unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); + fname[0] = 0; + offset = 0; + if (unw_get_proc_name(&cursor, fname, sizeof(fname), &offset) == 0) { + fname[sizeof(fname) - 1] = 0; + } unw_get_reg(&cursor, UNW_REG_SP, &sp); if (fn(ctx, (void*)pc, (void*)offset, (void*)sp, fname) == 0) { + errno = saved_errno; return; } } } -} \ No newline at end of file + errno = saved_errno; +} diff --git a/runtime/internal/lib/runtime/extern.go b/runtime/internal/lib/runtime/extern.go index 1fb397dd8a..377c973876 100644 --- a/runtime/internal/lib/runtime/extern.go +++ b/runtime/internal/lib/runtime/extern.go @@ -9,16 +9,26 @@ import ( ) func Caller(skip int) (pc uintptr, file string, line int, ok bool) { - // llgo currently doesn't have reliable source file/line mapping from PC. - // Return a stable placeholder location so stdlib log/testing can proceed. var pcs [1]uintptr - if Callers(skip+1, pcs[:]) < 1 { + if Callers(skip+2, pcs[:]) < 1 { return 0, "", 0, false } - return pcs[0], "???", 1, true + sym := frameSymbol(pcs[0]) + file, line = sym.file, sym.line + if file == "" { + file = "???" + } + if line == 0 { + line = 1 + } + return pcs[0], file, line, true } func Callers(skip int, pc []uintptr) int { + return callers(skip+1, pc) +} + +func callers(skip int, pc []uintptr) int { if len(pc) == 0 { return 0 } @@ -28,6 +38,7 @@ func Callers(skip int, pc []uintptr) int { return false } pc[n] = fr.PC + recordFrameSymbol(fr.PC, fr.Offset, fr.Name) n++ return true }) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 5b8155d0e8..bd131c9a25 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -85,5 +85,17 @@ func NumGoroutine() int { func SetCPUProfileRate(hz int) {} func FuncForPC(pc uintptr) *Func { - return nil + sym := frameSymbol(pc) + if !sym.ok && sym.function == "" { + return &Func{entry: pc, name: unknownFunctionName(pc)} + } + name := sym.function + if name == "" { + name = unknownFunctionName(pc) + } + entry := sym.entry + if entry == 0 { + entry = pc + } + return &Func{entry: entry, name: name} } diff --git a/runtime/internal/lib/runtime/runtime2.go b/runtime/internal/lib/runtime/runtime2.go index 7327f9892c..8bf049e087 100644 --- a/runtime/internal/lib/runtime/runtime2.go +++ b/runtime/internal/lib/runtime/runtime2.go @@ -17,7 +17,55 @@ type _func struct { } func Stack(buf []byte, all bool) int { - return 0 + var pcs [64]uintptr + n := Callers(0, pcs[:]) + out := make([]byte, 0, 1024) + out = append(out, "goroutine 1 [running]:\n"...) + frames := CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if frame.Function == "" { + frame.Function = unknownFunctionName(frame.PC) + } + out = append(out, frame.Function...) + out = append(out, "()\n\t"...) + if frame.File == "" { + out = append(out, "???"...) + } else { + out = append(out, frame.File...) + } + out = append(out, ':') + out = appendInt(out, frame.Line) + out = append(out, ' ') + out = append(out, "+0x0\n"...) + if !more { + break + } + } + if len(out) > len(buf) { + copy(buf, out[:len(buf)]) + return len(buf) + } + copy(buf, out) + return len(out) +} + +func appendInt(out []byte, v int) []byte { + if v == 0 { + return append(out, '0') + } + if v < 0 { + out = append(out, '-') + v = -v + } + var digits [20]byte + i := len(digits) + for v > 0 { + i-- + digits[i] = byte('0' + v%10) + v /= 10 + } + return append(out, digits[i:]...) } type traceError string diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index a4d18d9b30..84e04aa943 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -105,6 +105,249 @@ func unknownFunctionName(pc uintptr) string { return "pc=" + uintptrHex(pc) } +type pcSymbol struct { + pc uintptr + entry uintptr + function string + file string + line int + startLine int + ok bool +} + +type frameSymbolCacheEntry struct { + pc uintptr + offset uintptr + name string +} + +const frameSymbolCacheSize = 128 + +var frameSymbolCache [frameSymbolCacheSize]frameSymbolCacheEntry + +func recordFrameSymbol(pc, offset uintptr, name string) { + if pc == 0 || name == "" { + return + } + i := (pc >> 4) & (frameSymbolCacheSize - 1) + frameSymbolCache[i] = frameSymbolCacheEntry{pc: pc, offset: offset, name: name} +} + +type runtimeFuncInfoRecord struct { + symbol uint32 + name uint32 + file uint32 + line uint32 + column uint32 +} + +//go:linkname runtimeFuncInfoTable __llgo_funcinfo_table +var runtimeFuncInfoTable *runtimeFuncInfoRecord + +//go:linkname runtimeFuncInfoStrings __llgo_funcinfo_strings +var runtimeFuncInfoStrings *c.Char + +//go:linkname runtimeFuncInfoHash __llgo_funcinfo_hash +var runtimeFuncInfoHash *uint32 + +//go:linkname runtimeFuncInfoCount __llgo_funcinfo_count +var runtimeFuncInfoCount uintptr + +//go:linkname runtimeFuncInfoHashMask __llgo_funcinfo_hash_mask +var runtimeFuncInfoHashMask uintptr + +func hasStringPrefix(s, prefix string) bool { + if len(s) < len(prefix) { + return false + } + for i := 0; i < len(prefix); i++ { + if s[i] != prefix[i] { + return false + } + } + return true +} + +func publicFunctionName(name string) string { + const commandLineArguments = "command-line-arguments." + if hasStringPrefix(name, commandLineArguments) { + return "main." + name[len(commandLineArguments):] + } + if len(name) > 0 && name[0] == '_' { + name = name[1:] + } + return name +} + +func cStringEqual(cstr *c.Char, s string) bool { + return cStringCompare(cstr, s) == 0 +} + +func cStringCompare(cstr *c.Char, s string) int { + if cstr == nil { + if s == "" { + return 0 + } + return -1 + } + ptr := unsafe.Pointer(cstr) + for i := 0; ; i++ { + c := *(*byte)(unsafe.Add(ptr, i)) + if i == len(s) { + if c == 0 { + return 0 + } + return 1 + } + if c == 0 { + return -1 + } + if c < s[i] { + return -1 + } + if c > s[i] { + return 1 + } + } +} + +func funcInfoCString(off uint32) *c.Char { + if runtimeFuncInfoStrings == nil { + return nil + } + return (*c.Char)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoStrings), uintptr(off))) +} + +func funcInfoAt(i uintptr) *runtimeFuncInfoRecord { + size := unsafe.Sizeof(*runtimeFuncInfoTable) + return (*runtimeFuncInfoRecord)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoTable), i*size)) +} + +func funcInfoHashString(s string) uintptr { + const ( + offset = uint32(2166136261) + prime = uint32(16777619) + ) + h := offset + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= prime + } + return uintptr(h) +} + +func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { + if symbol == "" || runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 { + return nil + } + if runtimeFuncInfoHash != nil && runtimeFuncInfoHashMask != 0 { + slot := funcInfoHashString(symbol) & runtimeFuncInfoHashMask + for probes := uintptr(0); probes <= runtimeFuncInfoHashMask; probes++ { + idx := *(*uint32)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoHash), slot*unsafe.Sizeof(*runtimeFuncInfoHash))) + if idx == 0 { + return nil + } + if uintptr(idx) <= runtimeFuncInfoCount { + rec := funcInfoAt(uintptr(idx) - 1) + if cStringEqual(funcInfoCString(rec.symbol), symbol) { + return rec + } + } + slot = (slot + 1) & runtimeFuncInfoHashMask + } + return nil + } + for i := uintptr(0); i < runtimeFuncInfoCount; i++ { + rec := funcInfoAt(i) + if cStringEqual(funcInfoCString(rec.symbol), symbol) { + return rec + } + } + return nil +} + +func applyFuncInfo(sym *pcSymbol, rawFunction string) { + rec := funcInfoForSymbol(rawFunction) + if rec == nil { + public := publicFunctionName(rawFunction) + if public != rawFunction { + rec = funcInfoForSymbol(public) + } + } + if rec == nil { + return + } + if name := safeGoString(funcInfoCString(rec.name), ""); name != "" { + sym.function = publicFunctionName(name) + } + if file := safeGoString(funcInfoCString(rec.file), ""); file != "" { + if sym.file == "" { + sym.file = file + } + } + if rec.line != 0 { + sym.startLine = int(rec.line) + if sym.line == 0 { + sym.line = int(rec.line) + } + } + sym.ok = sym.ok || sym.function != "" || sym.file != "" +} + +func cachedFrameSymbol(pc uintptr) pcSymbol { + i := (pc >> 4) & (frameSymbolCacheSize - 1) + entry := frameSymbolCache[i] + if entry.pc != pc || entry.name == "" { + return pcSymbol{pc: pc} + } + rawFn := entry.name + fn := publicFunctionName(rawFn) + sym := pcSymbol{ + pc: pc, + entry: pc - entry.offset, + function: fn, + ok: fn != "" || entry.offset != 0, + } + applyFuncInfo(&sym, rawFn) + return sym +} + +func addrInfoSymbol(pc uintptr) pcSymbol { + var info clitedebug.Info + if clitedebug.Addrinfo(unsafe.Pointer(pc), &info) == 0 { + return cachedFrameSymbol(pc) + } + rawFn := safeGoString(info.Sname, "") + if rawFn == "" { + if sym := cachedFrameSymbol(pc); sym.ok { + return sym + } + } + fn := publicFunctionName(rawFn) + sym := pcSymbol{ + pc: pc, + entry: uintptr(info.Saddr), + function: fn, + ok: fn != "" || info.Saddr != nil, + } + applyFuncInfo(&sym, rawFn) + return sym +} + +func frameSymbol(pc uintptr) pcSymbol { + sym := addrInfoSymbol(pc) + if pc == 0 { + return sym + } + if sym.entry == 0 || pc > sym.entry { + if callSym := addrInfoSymbol(pc - 1); callSym.ok { + callSym.pc = pc + return callSym + } + } + return sym +} + func (ci *Frames) Next() (frame Frame, more bool) { for len(ci.frames) < 2 { // Find the next frame. @@ -119,8 +362,8 @@ func (ci *Frames) Next() (frame Frame, more bool) { } else { pc, ci.callers = ci.callers[0], ci.callers[1:] } - info := &clitedebug.Info{} - if clitedebug.Addrinfo(unsafe.Pointer(pc), info) == 0 { + sym := frameSymbol(pc) + if !sym.ok { ci.frames = append(ci.frames, Frame{ PC: pc, Function: unknownFunctionName(pc), @@ -131,17 +374,22 @@ func (ci *Frames) Next() (frame Frame, more bool) { }) continue } - fn := safeGoString(info.Sname, "") + fn := sym.function if fn == "" { fn = unknownFunctionName(pc) } + var f *Func + if sym.entry != 0 || fn != "" { + f = &Func{entry: sym.entry, name: fn} + } ci.frames = append(ci.frames, Frame{ PC: pc, + Func: f, Function: fn, - File: "", - Line: 0, - startLine: 0, - Entry: uintptr(info.Saddr), + File: sym.file, + Line: sym.line, + startLine: sym.startLine, + Entry: sym.entry, }) } @@ -176,19 +424,27 @@ func CallersFrames(callers []uintptr) *Frames { // A Func represents a Go function in the running binary. type Func struct { - opaque struct{} // unexported field to disallow conversions + entry uintptr + name string } func (f *Func) Name() string { - panic("todo") + if f == nil { + return "" + } + return f.name } -func (f *Func) FileLine(pc uintptr) (file string, line int) { - var info clitedebug.Info - if pc == 0 || clitedebug.Addrinfo(unsafe.Pointer(pc), &info) == 0 { - return "", 0 +func (f *Func) Entry() uintptr { + if f == nil { + return 0 } - return safeGoString(info.Fname, ""), 0 + return f.entry +} + +func (f *Func) FileLine(pc uintptr) (file string, line int) { + sym := frameSymbol(pc) + return sym.file, sym.line } // moduledata records information about the layout of the executable diff --git a/ssa/decl.go b/ssa/decl.go index 115bf28fc2..3e17b40e9f 100644 --- a/ssa/decl.go +++ b/ssa/decl.go @@ -423,4 +423,9 @@ func (p Function) Inline(inline inlineAttr) { p.impl.AddFunctionAttr(inlineAttr) } +func (p Function) DisableTailCalls() { + attr := p.Pkg.mod.Context().CreateStringAttribute("disable-tail-calls", "true") + p.impl.AddFunctionAttr(attr) +} + // ----------------------------------------------------------------------------- diff --git a/test/go/runtime_lineinfo_stack_test.go b/test/go/runtime_lineinfo_stack_test.go new file mode 100644 index 0000000000..4d46601112 --- /dev/null +++ b/test/go/runtime_lineinfo_stack_test.go @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gotest + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const runtimeLineInfoProbe = `package main + +import ( + "strconv" + "runtime" + "runtime/debug" + "strings" + _ "unsafe" +) + +func main() { + checkCaller() + checkCallerSkip() + checkFrames() + checkFuncForPC() + checkFuncInfoRename() + checkRuntimeStack() + checkPanicStack() +} + +//go:noinline +func checkCaller() { + _, file, line, ok := runtime.Caller(0) // CALLER_MARK + if !ok || !strings.HasSuffix(file, "main.go") || line != CALLER_LINE { + panic("bad caller: " + file + ":" + strconv.Itoa(line)) + } +} + +//go:noinline +func checkCallerSkip() { + helperCallerSkip() +} + +//go:noinline +func helperCallerSkip() { + _, file, line, ok := runtime.Caller(1) + if !ok || !strings.HasSuffix(file, "main.go") || line != CALLER_SKIP_LINE { + panic("bad caller skip: " + file + ":" + strconv.Itoa(line)) + } +} + +//go:noinline +func checkFrames() { + var pcs [8]uintptr + n := runtime.Callers(0, pcs[:]) + frames := runtime.CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if frame.Function == "main.checkFrames" { + if !strings.HasSuffix(frame.File, "main.go") || frame.Line == 0 { + panic("bad frame") + } + return + } + if !more { + break + } + } + panic("missing frame") +} + +//go:noinline +func checkFuncForPC() { + pc, _, _, ok := runtime.Caller(0) // FUNC_FILELINE_MARK + if !ok { + panic("missing pc") + } + fn := runtime.FuncForPC(pc) + if fn == nil { + panic("missing func") + } + if name := fn.Name(); name != "main.checkFuncForPC" { + panic("bad func: " + name) + } + if entry := fn.Entry(); entry == 0 { + panic("missing func entry") + } + file, line := fn.FileLine(pc) + if !strings.HasSuffix(file, "main.go") || line != FUNC_FILELINE_LINE { + panic("bad func fileline: " + file + ":" + strconv.Itoa(line)) + } +} + +//go:noinline +func checkFuncInfoRename() { + pc := renamedPC() + if name := runtime.FuncForPC(pc).Name(); name != "main.renamedPC" { + panic("bad renamed func: " + name) + } +} + +//go:linkname renamedPC main.renamedPCSymbol +//go:noinline +func renamedPC() uintptr { + pc, _, _, ok := runtime.Caller(0) + if !ok { + panic("missing renamed pc") + } + return pc +} + +//go:noinline +func checkRuntimeStack() { + var buf [4096]byte + n := runtime.Stack(buf[:], false) // RUNTIME_STACK_MARK + stack := string(buf[:n]) + if !strings.Contains(stack, "main.checkRuntimeStack") || !strings.Contains(stack, "main.go:RUNTIME_STACK_LINE") { + panic("bad runtime stack: " + stack) + } +} + +//go:noinline +func checkPanicStack() { + defer func() { // DEBUG_STACK_MARK + if recover() == nil { + panic("missing panic") + } + stack := string(debug.Stack()) + if !strings.Contains(stack, "main.checkPanicStack") || !strings.Contains(stack, "main.go:DEBUG_STACK_LINE") { + panic("bad stack: " + stack) + } + }() + s := []int{1, 2, 3} + _ = s[3] +} +` + +func TestRuntimeLineInfoAndStack(t *testing.T) { + source := runtimeLineInfoProbe + source = strings.ReplaceAll(source, "CALLER_LINE", strconv.Itoa(markerLine(source, "func checkCaller()"))) + source = strings.ReplaceAll(source, "CALLER_SKIP_LINE", strconv.Itoa(markerLine(source, "func checkCallerSkip()"))) + source = strings.ReplaceAll(source, "FUNC_FILELINE_LINE", strconv.Itoa(markerLine(source, "func checkFuncForPC()"))) + source = strings.ReplaceAll(source, "RUNTIME_STACK_LINE", strconv.Itoa(markerLine(source, "func checkRuntimeStack()"))) + source = strings.ReplaceAll(source, "DEBUG_STACK_LINE", strconv.Itoa(markerLine(source, "DEBUG_STACK_MARK"))) + + dir := t.TempDir() + file := filepath.Join(dir, "main.go") + if err := os.WriteFile(file, []byte(source), 0644); err != nil { + t.Fatal(err) + } + + repoRoot := findStringConversionRepoRoot(t) + t.Setenv("LLGO_ROOT", repoRoot) + cmd := exec.Command("go", "run", "./cmd/llgo", "run", "-a", file) + cmd.Dir = repoRoot + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("llgo lineinfo probe failed: %v\n%s", err, out) + } +} + +func markerLine(source, marker string) int { + line := 1 + for _, part := range strings.SplitAfter(source, "\n") { + if strings.Contains(part, marker) { + return line + } + line++ + } + panic("missing marker " + marker) +} diff --git a/test/goroot/xfail.yaml b/test/goroot/xfail.yaml index d16ea32788..458df947fc 100644 --- a/test/goroot/xfail.yaml +++ b/test/goroot/xfail.yaml @@ -2159,10 +2159,6 @@ xfails: directive: run case: fixedbugs/issue29504.go reason: latest main goroot run failure on darwin/arm64 - - platform: darwin/arm64 - directive: run - case: fixedbugs/issue29735.go - reason: latest main goroot run failure on darwin/arm64 - platform: darwin/arm64 directive: run case: fixedbugs/issue32477.go From a7f038331f9cb2dd1809b6a2dc1646792e1d008f Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 00:10:52 +0800 Subject: [PATCH 03/23] runtime: add statement line caller frames --- cl/caller_frame_test.go | 368 +++++++++++++++++++++++++ cl/compile.go | 105 ++++++- cl/instr.go | 227 +++++++++++++++ runtime/internal/lib/runtime/extern.go | 15 + runtime/internal/lib/runtime/symtab.go | 12 + runtime/internal/runtime/caller.go | 254 +++++++++++++++++ runtime/internal/runtime/z_rt.go | 1 + test/go/runtime_lineinfo_stack_test.go | 14 +- test/go/runtime_statement_line_test.go | 162 +++++++++++ 9 files changed, 1149 insertions(+), 9 deletions(-) create mode 100644 cl/caller_frame_test.go create mode 100644 runtime/internal/runtime/caller.go create mode 100644 test/go/runtime_statement_line_test.go diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go new file mode 100644 index 0000000000..965086fd2d --- /dev/null +++ b/cl/caller_frame_test.go @@ -0,0 +1,368 @@ +//go:build !llgo +// +build !llgo + +package cl + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/goplus/gogen/packages" + llssa "github.com/goplus/llgo/ssa" + gossa "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" +) + +func parseCallerFrameFile(t *testing.T, src string) *ast.File { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), "caller_frame.go", src, 0) + if err != nil { + t.Fatal(err) + } + return file +} + +func TestFilesUseRuntimeCaller(t *testing.T) { + tests := []struct { + name string + src string + want bool + }{ + { + name: "runtime selector", + src: `package foo +import "runtime" +func f() { runtime.Caller(0) } +`, + want: true, + }, + { + name: "runtime alias", + src: `package foo +import rt "runtime" +func f() { rt.Callers(0, nil) } +`, + want: true, + }, + { + name: "runtime debug stack", + src: `package foo +import dbg "runtime/debug" +func f() { _ = dbg.Stack() } +`, + want: true, + }, + { + name: "dot import", + src: `package foo +import . "runtime" +func f() { _ = FuncForPC(0) } +`, + want: true, + }, + { + name: "blank import", + src: `package foo +import _ "runtime" +func f() {} +`, + want: false, + }, + { + name: "non caller runtime selector", + src: `package foo +import "runtime" +func f() { _ = runtime.GOOS } +`, + want: false, + }, + { + name: "caller name without runtime import", + src: `package foo +func f() { Caller(0) } +`, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := filesUseRuntimeCaller([]*ast.File{parseCallerFrameFile(t, tt.src)}); got != tt.want { + t.Fatalf("filesUseRuntimeCaller() = %v, want %v", got, tt.want) + } + }) + } + + badImport := &ast.File{ + Imports: []*ast.ImportSpec{{ + Path: &ast.BasicLit{Kind: token.STRING, Value: "runtime"}, + }}, + } + if filesUseRuntimeCaller([]*ast.File{badImport}) { + t.Fatal("invalid import literal should not enable caller frame tracking") + } +} + +func buildCallerFrameSSAPackage(t *testing.T, pkgPath, src string) (*gossa.Package, []*ast.File) { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "caller_frame_compile.go", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + imp := packages.NewImporter(fset) + mode := gossa.SanityCheckFunctions | gossa.InstantiateGenerics + ssapkg, _, err := ssautil.BuildPackage( + &types.Config{Importer: imp}, + fset, + types.NewPackage(pkgPath, file.Name.Name), + files, + mode, + ) + if err != nil { + t.Fatal(err) + } + return ssapkg, files +} + +func TestRuntimeCallerPackageDetection(t *testing.T) { + ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" +import "runtime/debug" + +func direct() { runtime.Caller(0) } +func stack() { _ = debug.Stack() } +func anonOnly() { func() { runtime.FuncForPC(0) }() } +func plain() {} +`) + if !packageUsesRuntimeCaller(ssapkg) { + t.Fatal("package should report runtime caller usage") + } + if !fnUsesRuntimeCaller(ssapkg.Func("direct")) { + t.Fatal("direct runtime.Caller use should be detected") + } + if !fnUsesRuntimeCaller(ssapkg.Func("stack")) { + t.Fatal("runtime/debug.Stack use should be detected") + } + if !fnUsesRuntimeCaller(ssapkg.Func("anonOnly")) { + t.Fatal("runtime caller use in anonymous functions should be detected") + } + if fnUsesRuntimeCaller(ssapkg.Func("plain")) { + t.Fatal("plain function should not report runtime caller usage") + } + + for _, name := range []string{"Caller", "Callers", "CallersFrames", "FuncForPC", "Stack"} { + if !isRuntimeCallerName(name) { + t.Fatalf("%s should be a runtime caller metadata function", name) + } + } + if isRuntimeCallerName("Version") { + t.Fatal("Version should not be a runtime caller metadata function") + } + + rtpkg, _ := buildCallerFrameSSAPackage(t, "github.com/goplus/llgo/runtime/internal/lib/runtime", `package runtime +func Caller(skip int) (uintptr, string, int, bool) { return 0, "", 0, false } +func FuncForPC(pc uintptr) uintptr { return 0 } +`) + if !isRuntimeCallerFunc(rtpkg.Func("Caller")) || !isRuntimeCallerLookupFunc(rtpkg.Func("Caller")) { + t.Fatal("LLGo runtime lib Caller should be treated as runtime.Caller") + } + if !isRuntimeCallerFunc(rtpkg.Func("FuncForPC")) { + t.Fatal("LLGo runtime lib FuncForPC should be treated as runtime metadata use") + } + if isRuntimeCallerLookupFunc(rtpkg.Func("FuncForPC")) { + t.Fatal("FuncForPC should not consume caller lookup tokens") + } +} + +func TestCallerFrameTrackingEligibility(t *testing.T) { + if (&context{}).shouldTrackCallerFrames() { + t.Fatal("missing compiler state should not track caller frames") + } + var nilContext *context + if nilContext.shouldTrackCallerFrames() { + t.Fatal("nil context should not track caller frames") + } + + tests := []struct { + name string + pkgPath string + track bool + targetName string + goarch string + want bool + }{ + {name: "enabled user package", pkgPath: "example.com/foo", track: true, want: true}, + {name: "disabled flag", pkgPath: "example.com/foo", want: false}, + {name: "named target", pkgPath: "example.com/foo", track: true, targetName: "esp32", want: false}, + {name: "wasm", pkgPath: "example.com/foo", track: true, goarch: "wasm", want: false}, + {name: "stdlib", pkgPath: "fmt", track: true, want: false}, + {name: "runtime", pkgPath: "runtime", track: true, want: false}, + {name: "llgo runtime", pkgPath: llssa.PkgRuntime, track: true, want: false}, + {name: "llgo runtime internal", pkgPath: "github.com/goplus/llgo/runtime/internal/foo", track: true, want: false}, + {name: "command line package", pkgPath: "command-line-arguments", track: true, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prog := llssa.NewProgram(nil) + if tt.targetName != "" { + prog.Target().Target = tt.targetName + } + if tt.goarch != "" { + prog.Target().GOARCH = tt.goarch + } + pkg := prog.NewPackage("foo", tt.pkgPath) + fn := pkg.NewFunc("f", llssa.NoArgsNoRet, llssa.InGo) + ctx := &context{prog: prog, pkg: pkg, fn: fn, trackCallerFrames: tt.track} + if got := ctx.shouldTrackCallerFrames(); got != tt.want { + t.Fatalf("shouldTrackCallerFrames() = %v, want %v", got, tt.want) + } + }) + } + + if canTrackCallerFramesForPackage("net/http") { + t.Fatal("stdlib paths without dots should not track caller frames") + } +} + +func TestRuntimeFrameNameNormalization(t *testing.T) { + tests := []struct { + in string + want string + }{ + {in: "command-line-arguments.main", want: "main.main"}, + {in: "example.com/foo.f$1", want: "example.com/foo.f.func1"}, + {in: "example.com/foo.f", want: "example.com/foo.f"}, + {in: "example.com/foo.f$", want: "example.com/foo.f$"}, + {in: "example.com/foo.f$inner", want: "example.com/foo.f$inner"}, + } + for _, tt := range tests { + if got := runtimeFrameName(tt.in); got != tt.want { + t.Fatalf("runtimeFrameName(%q) = %q, want %q", tt.in, got, tt.want) + } + } + + if got := (*context)(nil).runtimeCallerFrameName(); got != "" { + t.Fatalf("nil context runtimeCallerFrameName() = %q, want empty", got) + } + if got := (&context{}).runtimeCallerFrameName(); got != "" { + t.Fatalf("empty context runtimeCallerFrameName() = %q, want empty", got) + } + prog := newLLSSAProg(t) + pkg := prog.NewPackage("main", "command-line-arguments") + sig := types.NewSignatureType(nil, nil, nil, nil, nil, false) + ctx := &context{fn: pkg.NewFuncEx("command-line-arguments.f$1", sig, llssa.InGo, false, false)} + if got, want := ctx.runtimeCallerFrameName(), "main.f.func1"; got != want { + t.Fatalf("fallback runtimeCallerFrameName() = %q, want %q", got, want) + } +} + +func TestCompileRuntimeCallerFrameInstrumentation(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime/debug" + +func f() { + _ = debug.Stack() +} +`) + prog := newLLSSAProg(t) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + for _, want := range []string{ + "PushCallerFrame", + "SetCallerLookupLine", + "PopCallerFrame", + `c"example.com/foo.f`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("compiled caller-frame IR missing %q:\n%s", want, ir) + } + } +} + +func TestCompileRuntimeCallerFrameUsesGoNameForLinkname(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "command-line-arguments", `package main +import "runtime" + +func renamedPC() uintptr { + pc, _, _, _ := runtime.Caller(0) + return pc +} +`) + prog := newLLSSAProg(t) + prog.SetLinkname("command-line-arguments.renamedPC", "main.renamedPCSymbol") + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + if !strings.Contains(ir, `c"main.renamedPC"`) { + t.Fatalf("compiled caller-frame IR missing source function name:\n%s", ir) + } + if strings.Contains(ir, `c"main.renamedPCSymbol"`) { + t.Fatalf("compiled caller-frame IR used linkname target as runtime function name:\n%s", ir) + } +} + +func TestCompileRuntimeCallerFrameInstrumentationSkipped(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func f() { + runtime.Caller(0) +} +`) + prog := newLLSSAProg(t) + prog.Target().Target = "esp32" + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + if ir := pkg.Module().String(); strings.Contains(ir, "PushCallerFrame") { + t.Fatalf("target builds should not emit caller-frame tracking:\n%s", ir) + } + + ssapkg, files = buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +func f() {} +`) + prog = newLLSSAProg(t) + pkg, err = NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + if ir := pkg.Module().String(); strings.Contains(ir, "PushCallerFrame") || strings.Contains(ir, "SetCallerLine") { + t.Fatalf("packages without runtime stack APIs should not emit caller-frame tracking:\n%s", ir) + } +} + +func TestCompileRuntimeCallerLookupTokenOnlyForRuntimeAPIs(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func helper() {} + +func f() { + helper() + runtime.Caller(0) +} +`) + prog := newLLSSAProg(t) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + if !strings.Contains(ir, "SetCallerLookupLine") { + t.Fatalf("runtime.Caller should enable caller lookup:\n%s", ir) + } + if !strings.Contains(ir, "SetCallerLine") { + t.Fatalf("ordinary calls in an instrumented package should only update the current line:\n%s", ir) + } +} diff --git a/cl/compile.go b/cl/compile.go index fe1dbfc8b1..27282c7461 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -199,6 +199,9 @@ type context struct { rewrites map[string]string embedMap goembed.VarMap embedInits []embedInit + + trackCallerFrames bool + callerFrameMark llssa.Expr } func (p *context) rewriteValue(name string) (string, bool) { @@ -214,6 +217,79 @@ func (p *context) rewriteValue(name string) (string, bool) { return val, ok } +func filesUseRuntimeCaller(files []*ast.File) bool { + for _, file := range files { + imports := make(map[string]string) + dotImports := make(map[string]bool) + for _, imp := range file.Imports { + path, err := strconv.Unquote(imp.Path.Value) + if err != nil { + continue + } + switch path { + case "runtime", "runtime/debug": + default: + continue + } + name := path[strings.LastIndex(path, "/")+1:] + if imp.Name != nil { + switch imp.Name.Name { + case ".": + dotImports[path] = true + continue + case "_": + continue + default: + name = imp.Name.Name + } + } + imports[name] = path + } + if len(imports) == 0 && len(dotImports) == 0 { + continue + } + found := false + ast.Inspect(file, func(n ast.Node) bool { + if found { + return false + } + switch n := n.(type) { + case *ast.SelectorExpr: + ident, ok := n.X.(*ast.Ident) + if !ok { + return true + } + if runtimeCallerSelector(imports[ident.Name], n.Sel.Name) { + found = true + return false + } + case *ast.Ident: + if (dotImports["runtime"] && isRuntimeCallerName(n.Name)) || + (dotImports["runtime/debug"] && n.Name == "Stack") { + found = true + return false + } + } + return true + }) + if found { + return true + } + } + return false +} + +func runtimeCallerSelector(path, name string) bool { + switch path { + case "runtime": + return isRuntimeCallerName(name) + case "runtime/debug": + return name == "Stack" + default: + return false + } +} + // isStringPtrType checks if typ is a pointer to the basic string type (*string). // This is used to validate that -ldflags -X can only rewrite variables of type *string, // not derived string types like "type T string". @@ -513,12 +589,13 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun dbgEnabled := enableDbg && (f == nil || f.Origin() == nil) dbgSymsEnabled := enableDbgSyms && (f == nil || f.Origin() == nil) p.inits = append(p.inits, func() { - oldFn, oldGoFn, oldMethodNilDerefChecks := p.fn, p.goFn, p.methodNilDerefChecks + oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark := p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark p.fn = fn p.goFn = f + p.callerFrameMark = llssa.Nil p.state = state // restore pkgState when compiling funcBody defer func() { - p.fn, p.goFn, p.methodNilDerefChecks = oldFn, oldGoFn, oldMethodNilDerefChecks + p.fn, p.goFn, p.methodNilDerefChecks, p.callerFrameMark = oldFn, oldGoFn, oldMethodNilDerefChecks, oldCallerFrameMark }() p.phis = nil if dbgSymsEnabled { @@ -669,6 +746,9 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do var instrs = block.Instrs[n:] var ret = fn.Block(block.Index) b.SetBlock(ret) + if block.Index == 0 && p.shouldTrackCallerFrames() { + p.pushCallerFrame(b, block.Parent()) + } if block.Index == 0 && enableCallTracing && !strings.HasPrefix(fn.Name(), "github.com/goplus/llgo/runtime/internal/runtime.Print") { b.Printf("call " + fn.Name() + "\n\x00") } @@ -1057,6 +1137,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue if t := p.type_(v.Type(), llssa.InGo); t.RawType() != nil { if p.isLargeNonPointerValue(t) { x := p.compileValue(b, v.X) + p.setCallerLine(b, v.Pos()) p.assertNilDerefBase(b, v.X) b.AssertNilDeref(x) return @@ -1070,6 +1151,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue // Zero-length slice-to-array conversions can leave only // an unused slice deref; preserve its required nil check. x := p.compileValue(b, v.X) + p.setCallerLine(b, v.Pos()) p.assertNilDerefBase(b, v.X) b.AssertNilDeref(x) return @@ -1100,6 +1182,9 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } } x := p.compileValue(b, v.X) + if v.Op != token.ARROW { + p.setCallerLine(b, v.Pos()) + } if shouldAssertDirectNilDeref(v) { b.AssertNilDeref(x) } @@ -1135,6 +1220,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue ret = b.Convert(p.type_(t, llssa.InGo), x) case *ssa.FieldAddr: x := p.compileValue(b, v.X) + p.setCallerLine(b, v.Pos()) if p.isAddressOfFieldAddr(v) { b.AssertNilDeref(x) } @@ -1156,10 +1242,12 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } x := p.compileValue(b, vx) idx := p.compileValue(b, v.Index) + p.setCallerLine(b, v.Pos()) ret = b.IndexAddr(x, idx) case *ssa.Index: x := p.compileValue(b, v.X) idx := p.compileValue(b, v.Index) + p.setCallerLine(b, v.Pos()) ret = b.Index(x, idx, func() (addr llssa.Expr, zero bool) { switch n := v.X.(type) { case *ssa.Const: @@ -1193,6 +1281,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue if v.Max != nil { max = p.compileValue(b, v.Max) } + p.setCallerLine(b, v.Pos()) ret = b.Slice(x, low, high, max) ret.Type = p.type_(v.Type(), llssa.InGo) case *ssa.MakeInterface: @@ -1249,6 +1338,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue case *ssa.TypeAssert: x := p.compileValue(b, v.X) t := p.type_(v.AssertedType, llssa.InGo) + p.setCallerLine(b, v.Pos()) ret = b.TypeAssert(x, t, v.CommaOk) case *ssa.Extract: x := p.compileValue(b, v.Tuple) @@ -1289,6 +1379,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue case *ssa.SliceToArrayPointer: t := p.type_(v.Type(), llssa.InGo) x := p.compileValue(b, v.X) + p.setCallerLine(b, v.Pos()) ret = b.SliceToArrayPointer(x, t) default: panic(fmt.Sprintf("compileInstrAndValue: unknown instr - %T\n", iv)) @@ -1423,8 +1514,12 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { } } if p.returnNeedsImplicitRunDefers(v) { + p.setCallerLine(b, v.Pos()) b.RunDefers() } + if p.shouldTrackCallerFrames() { + p.popCallerFrame(b) + } b.Return(results...) case *ssa.If: fn := p.fn @@ -1437,6 +1532,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { m := p.compileValue(b, v.Map) key := p.compileValue(b, v.Key) val := p.compileValue(b, v.Value) + p.setCallerLine(b, v.Pos()) b.MapUpdate(m, key, val) case *ssa.Defer: if v.DeferStack != nil { @@ -1447,13 +1543,16 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { case *ssa.Go: p.call(b, llssa.Go, &v.Call) case *ssa.RunDefers: + p.setCallerLine(b, v.Pos()) b.RunDefers() case *ssa.Panic: arg := p.compileValue(b, v.X) + p.setCallerLine(b, v.Pos()) b.Panic(arg) case *ssa.Send: ch := p.compileValue(b, v.Chan) x := p.compileValue(b, v.X) + p.setCallerLine(b, v.Pos()) b.Send(ch, x) case *ssa.DebugRef: if enableDbgSyms && v.Parent().Origin() == nil { @@ -1769,6 +1868,8 @@ func newPackageEx(prog llssa.Program, patches Patches, rewrites map[string]strin }, cgoSymbols: make([]string, 0, 128), rewrites: rewrites, + + trackCallerFrames: filesUseRuntimeCaller(files) || packageUsesRuntimeCaller(pkg), } if embedMap != nil { ctx.embedMap = *embedMap diff --git a/cl/instr.go b/cl/instr.go index b7fc52abd3..0c6a2d69a6 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -853,6 +853,232 @@ func (p *context) sourceLine(filename string, line int) (string, bool) { return lines[line-1], true } +func (p *context) shouldTrackCallerFrames() bool { + if p == nil || p.pkg == nil || p.fn == nil || !p.trackCallerFrames { + return false + } + if target := p.prog.Target(); target != nil && (target.Target != "" || target.GOARCH == "wasm") { + return false + } + return canTrackCallerFramesForPackage(p.pkg.Path()) +} + +func canTrackCallerFramesForPackage(pkgPath string) bool { + return pkgPath != llssa.PkgRuntime && + pkgPath != "runtime" && + !isStandardLibraryPackage(pkgPath) && + !strings.HasPrefix(pkgPath, "github.com/goplus/llgo/runtime/internal/") +} + +func isStandardLibraryPackage(pkgPath string) bool { + return pkgPath != "command-line-arguments" && !strings.Contains(pkgPath, ".") +} + +func packageUsesRuntimeCaller(pkg *ssa.Package) bool { + if pkg == nil { + return false + } + for _, member := range pkg.Members { + fn, ok := member.(*ssa.Function) + if ok && fnUsesRuntimeCaller(fn) { + return true + } + } + return false +} + +func fnUsesRuntimeCaller(fn *ssa.Function) bool { + if fn == nil { + return false + } + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + call, ok := instr.(ssa.CallInstruction) + if !ok { + continue + } + if isRuntimeCallerFunc(call.Common().StaticCallee()) { + return true + } + } + } + for _, anon := range fn.AnonFuncs { + if fnUsesRuntimeCaller(anon) { + return true + } + } + return false +} + +func isRuntimeCallerFunc(fn *ssa.Function) bool { + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { + return false + } + switch fn.Pkg.Pkg.Path() { + case "runtime", "github.com/goplus/llgo/runtime/internal/lib/runtime": + return isRuntimeCallerName(fn.Name()) + case "runtime/debug": + return fn.Name() == "Stack" + default: + return false + } +} + +func isRuntimeCallerLookupFunc(fn *ssa.Function) bool { + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { + return false + } + switch fn.Pkg.Pkg.Path() { + case "runtime", "github.com/goplus/llgo/runtime/internal/lib/runtime": + switch fn.Name() { + case "Caller", "Callers", "Stack": + return true + } + case "runtime/debug": + return fn.Name() == "Stack" + } + return false +} + +func isRuntimeCallerName(name string) bool { + switch name { + case "Caller", "Callers", "CallersFrames", "FuncForPC", "Stack": + return true + default: + return false + } +} + +func (p *context) pushCallerFrame(b llssa.Builder, fn *ssa.Function) { + if fn == nil { + return + } + pos := p.fset.Position(fn.Pos()) + entry := b.Convert(p.prog.Uintptr(), p.fn.Expr) + p.callerFrameMark = b.Call( + p.runtimeFunc("PushCallerFrame", pushCallerFrameSig()), + entry, + b.Str(p.runtimeCallerFrameName()), + b.Str(pos.Filename), + p.prog.IntVal(uint64(pos.Line), p.prog.Int()), + ) +} + +func (p *context) runtimeCallerFrameName() string { + if p == nil { + return "" + } + if p.goFn != nil && p.goFn.Pkg != nil && p.goFn.Pkg.Pkg != nil { + return runtimeFrameName(funcName(p.goFn.Pkg.Pkg, p.goFn, false)) + } + if p.fn != nil { + return runtimeFrameName(p.fn.Name()) + } + return "" +} + +func (p *context) setCallerLine(b llssa.Builder, pos token.Pos) { + if !p.shouldTrackCallerFrames() { + return + } + line := p.fset.Position(pos).Line + p.setCallerLineNumber(b, line) +} + +func (p *context) setCallerLineForCall(b llssa.Builder, call *ssa.CallCommon) { + if !p.shouldTrackCallerFrames() { + return + } + line := p.fset.Position(call.Pos()).Line + if line <= 0 { + return + } + fn := "SetCallerLine" + sig := setCallerLineSig() + if isRuntimeCallerLookupFunc(call.StaticCallee()) { + fn = "SetCallerLookupLine" + sig = setCallerLookupLineSig() + } + b.Call(p.runtimeFunc(fn, sig), p.prog.IntVal(uint64(line), p.prog.Int())) +} + +func (p *context) setCallerLineNumber(b llssa.Builder, line int) { + if line <= 0 { + return + } + b.Call(p.runtimeFunc("SetCallerLine", setCallerLineSig()), p.prog.IntVal(uint64(line), p.prog.Int())) +} + +func (p *context) popCallerFrame(b llssa.Builder) { + if p.callerFrameMark.IsNil() { + return + } + b.Call(p.runtimeFunc("PopCallerFrame", popCallerFrameSig()), p.callerFrameMark) +} + +func (p *context) runtimeFunc(name string, sig *types.Signature) llssa.Expr { + p.pkg.NeedRuntime = true + fullName := llssa.PkgRuntime + "." + name + if fn := p.pkg.FuncOf(fullName); fn != nil { + return fn.Expr + } + return p.pkg.NewFuncEx(fullName, sig, llssa.InGo, false, false).Expr +} + +func pushCallerFrameSig() *types.Signature { + return types.NewSignatureType(nil, nil, nil, + types.NewTuple( + types.NewVar(token.NoPos, nil, "entry", types.Typ[types.Uintptr]), + types.NewVar(token.NoPos, nil, "name", types.Typ[types.String]), + types.NewVar(token.NoPos, nil, "file", types.Typ[types.String]), + types.NewVar(token.NoPos, nil, "startLine", types.Typ[types.Int]), + ), + types.NewTuple(types.NewVar(token.NoPos, nil, "", types.Typ[types.Int])), + false, + ) +} + +func setCallerLineSig() *types.Signature { + return types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "line", types.Typ[types.Int])), + nil, + false, + ) +} + +func setCallerLookupLineSig() *types.Signature { + return setCallerLineSig() +} + +func popCallerFrameSig() *types.Signature { + return types.NewSignatureType(nil, nil, nil, + types.NewTuple(types.NewVar(token.NoPos, nil, "mark", types.Typ[types.Int])), + nil, + false, + ) +} + +func runtimeFrameName(name string) string { + const commandLineArguments = "command-line-arguments." + if strings.HasPrefix(name, commandLineArguments) { + name = "main." + name[len(commandLineArguments):] + } + return normalizeRuntimeAnonFuncName(name) +} + +func normalizeRuntimeAnonFuncName(name string) string { + dollar := strings.LastIndexByte(name, '$') + if dollar < 0 || dollar == len(name)-1 { + return name + } + for i := dollar + 1; i < len(name); i++ { + if name[i] < '0' || name[i] > '9' { + return name + } + } + return name[:dollar] + ".func" + name[dollar+1:] +} + // ----------------------------------------------------------------------------- type explicitDeferStack struct { @@ -1049,6 +1275,7 @@ func collectMethodNilDerefChecks(fn *ssa.Function) map[*ssa.UnOp]none { } func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallCommon, ds *explicitDeferStack) (ret llssa.Expr) { + p.setCallerLineForCall(b, call) cv := call.Value if mthd := call.Method; mthd != nil { reflectCheck := p.reflectTypeMethodCheck(call, mthd) diff --git a/runtime/internal/lib/runtime/extern.go b/runtime/internal/lib/runtime/extern.go index 377c973876..d6835b794f 100644 --- a/runtime/internal/lib/runtime/extern.go +++ b/runtime/internal/lib/runtime/extern.go @@ -6,9 +6,21 @@ package runtime import ( clitedebug "github.com/goplus/llgo/runtime/internal/clite/debug" + rtdebug "github.com/goplus/llgo/runtime/internal/runtime" ) func Caller(skip int) (pc uintptr, file string, line int, ok bool) { + if frame, ok := rtdebug.Caller(skip); ok { + file = frame.File + line = frame.Line + if file == "" { + file = "???" + } + if line == 0 { + line = 1 + } + return frame.PC, file, line, true + } var pcs [1]uintptr if Callers(skip+2, pcs[:]) < 1 { return 0, "", 0, false @@ -25,6 +37,9 @@ func Caller(skip int) (pc uintptr, file string, line int, ok bool) { } func Callers(skip int, pc []uintptr) int { + if n := rtdebug.Callers(skip, pc); n > 0 { + return n + } return callers(skip+1, pc) } diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 84e04aa943..bd0a616018 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -9,6 +9,7 @@ import ( c "github.com/goplus/llgo/runtime/internal/clite" clitedebug "github.com/goplus/llgo/runtime/internal/clite/debug" + rtdebug "github.com/goplus/llgo/runtime/internal/runtime" ) // Frames may be used to get function/file/line information for a @@ -335,6 +336,17 @@ func addrInfoSymbol(pc uintptr) pcSymbol { } func frameSymbol(pc uintptr) pcSymbol { + if frame, ok := rtdebug.FrameForPC(pc); ok { + return pcSymbol{ + pc: pc, + entry: frame.Entry, + function: frame.Function, + file: frame.File, + line: frame.Line, + startLine: frame.StartLine, + ok: true, + } + } sym := addrInfoSymbol(pc) if pc == 0 { return sym diff --git a/runtime/internal/runtime/caller.go b/runtime/internal/runtime/caller.go new file mode 100644 index 0000000000..ba2a212fed --- /dev/null +++ b/runtime/internal/runtime/caller.go @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package runtime + +import ( + "unsafe" + + "github.com/goplus/llgo/runtime/internal/clite/tls" +) + +type CallerFrame struct { + PC uintptr + Entry uintptr + Function string + File string + Line int + StartLine int +} + +const ( + callerPCMask = uintptr(3) + callerPCValue = uintptr(1) + callersPCValue = uintptr(3) + callerPCRingSize = 1024 +) + +type callerPCStore struct { + next uintptr + frames [callerPCRingSize]CallerFrame +} + +var ( + callerFrameTLS = tls.Alloc[[]CallerFrame](nil) + callerPCStoreTLS = tls.Alloc[*callerPCStore](nil) + callerLookupTLS = tls.Alloc[bool](nil) + panicCallerFrameTLS = tls.Alloc[[]CallerFrame](nil) +) + +var ( + runtimeCallersFrame = CallerFrame{Function: "runtime.Callers"} + runtimeMainFrame = CallerFrame{Function: "runtime.main"} + runtimeGoexitFrame = CallerFrame{Function: "runtime.goexit"} +) + +func PushCallerFrame(entry uintptr, name, file string, startLine int) int { + frames := callerFrameTLS.Get() + mark := len(frames) + frames = append(frames, CallerFrame{ + PC: entry, + Entry: entry, + Function: name, + File: file, + Line: startLine, + StartLine: startLine, + }) + callerFrameTLS.Set(frames) + return mark +} + +func SetCallerLine(line int) { + frames := callerFrameTLS.Get() + if line <= 0 || len(frames) == 0 { + return + } + frames[len(frames)-1].Line = line + callerFrameTLS.Set(frames) +} + +func SetCallerLookupLine(line int) { + SetCallerLine(line) + callerLookupTLS.Set(true) +} + +func PopCallerFrame(mark int) { + frames := callerFrameTLS.Get() + oldLen := len(frames) + if mark < 0 || mark > oldLen { + return + } + var zero CallerFrame + for i := mark; i < oldLen; i++ { + frames[i] = zero + } + callerFrameTLS.Set(frames[:mark]) + + panicFrames := panicCallerFrameTLS.Get() + if len(panicFrames) > 0 && oldLen >= len(panicFrames) && mark <= len(panicFrames) { + for i := range panicFrames { + panicFrames[i] = zero + } + panicCallerFrameTLS.Clear() + } +} + +func SavePanicCallerFrames() { + frames := callerFrameTLS.Get() + if len(frames) == 0 { + panicCallerFrameTLS.Clear() + return + } + panicFrames := panicCallerFrameTLS.Get() + if cap(panicFrames) < len(frames) { + panicFrames = make([]CallerFrame, len(frames)) + } else { + panicFrames = panicFrames[:len(frames)] + } + copy(panicFrames, frames) + panicCallerFrameTLS.Set(panicFrames) +} + +func Caller(skip int) (CallerFrame, bool) { + if !takeCallerLookup() { + return CallerFrame{}, false + } + if skip < 0 { + return CallerFrame{}, false + } + frames := callerFrameTLS.Get() + panicFrames := panicCallerFrameTLS.Get() + if len(frames) == 0 { + if skip < len(panicFrames) { + return captureFrame(panicFrames[len(panicFrames)-1-skip], callerPCValue), true + } + return CallerFrame{}, false + } + if skip < len(frames) { + return captureFrame(frames[len(frames)-1-skip], callerPCValue), true + } + if len(panicFrames) > len(frames) { + idx := len(panicFrames) - 1 - skip + if idx >= 0 { + return captureFrame(panicFrames[idx], callerPCValue), true + } + } + switch skip - len(frames) { + case 0: + return captureFrame(runtimeMainFrame, callerPCValue), true + case 1: + return captureFrame(runtimeGoexitFrame, callerPCValue), true + default: + return CallerFrame{}, false + } +} + +func Callers(skip int, pcs []uintptr) int { + if !takeCallerLookup() { + return 0 + } + if skip < 0 { + skip = 0 + } + frames := callerFrameTLS.Get() + if len(frames) == 0 { + frames = panicCallerFrameTLS.Get() + } + if len(frames) == 0 { + return 0 + } + n := 0 + add := func(frame CallerFrame) bool { + if skip > 0 { + skip-- + return true + } + if n >= len(pcs) { + return false + } + pcs[n] = captureFrame(frame, callersPCValue).PC + n++ + return true + } + if !add(runtimeCallersFrame) { + return n + } + for i := len(frames) - 1; i >= 0; i-- { + if !add(frames[i]) { + return n + } + } + _ = add(runtimeMainFrame) + _ = add(runtimeGoexitFrame) + return n +} + +func takeCallerLookup() bool { + if !callerLookupTLS.Get() { + return false + } + callerLookupTLS.Set(false) + return true +} + +func FrameForPC(pc uintptr) (CallerFrame, bool) { + if pc&callerPCMask == 0 { + return CallerFrame{}, false + } + store := callerPCStoreTLS.Get() + if store == nil { + return CallerFrame{}, false + } + addr := pc &^ callerPCMask + if !store.contains(addr) { + return CallerFrame{}, false + } + frame := *(*CallerFrame)(unsafe.Pointer(addr)) + return frame, true +} + +func callerPCStoreForThread() *callerPCStore { + store := callerPCStoreTLS.Get() + if store == nil { + store = new(callerPCStore) + callerPCStoreTLS.Set(store) + } + return store +} + +func captureFrame(frame CallerFrame, pcValue uintptr) CallerFrame { + store := callerPCStoreForThread() + idx := store.next & (callerPCRingSize - 1) + store.next++ + store.frames[idx] = frame + rec := &store.frames[idx] + pc := uintptr(unsafe.Pointer(rec)) | pcValue + rec.PC = pc + if rec.Entry == 0 { + rec.Entry = pc + } + return *rec +} + +func (s *callerPCStore) contains(addr uintptr) bool { + start := uintptr(unsafe.Pointer(&s.frames[0])) + size := unsafe.Sizeof(s.frames) + end := start + size + if addr < start || addr >= end { + return false + } + return (addr-start)%unsafe.Sizeof(s.frames[0]) == 0 +} diff --git a/runtime/internal/runtime/z_rt.go b/runtime/internal/runtime/z_rt.go index 3b17c951e1..4cd79f22ac 100644 --- a/runtime/internal/runtime/z_rt.go +++ b/runtime/internal/runtime/z_rt.go @@ -49,6 +49,7 @@ func Recover() (ret any) { // Panic panics with a value. func Panic(v any) { + SavePanicCallerFrames() ptr := c.Malloc(unsafe.Sizeof(v)) *(*any)(ptr) = v excepKey.Set(ptr) diff --git a/test/go/runtime_lineinfo_stack_test.go b/test/go/runtime_lineinfo_stack_test.go index 4d46601112..e9c9bf7334 100644 --- a/test/go/runtime_lineinfo_stack_test.go +++ b/test/go/runtime_lineinfo_stack_test.go @@ -55,7 +55,7 @@ func checkCaller() { //go:noinline func checkCallerSkip() { - helperCallerSkip() + helperCallerSkip() // CALLER_SKIP_MARK } //go:noinline @@ -142,7 +142,7 @@ func checkPanicStack() { if recover() == nil { panic("missing panic") } - stack := string(debug.Stack()) + stack := string(debug.Stack()) // DEBUG_STACK_CALL_MARK if !strings.Contains(stack, "main.checkPanicStack") || !strings.Contains(stack, "main.go:DEBUG_STACK_LINE") { panic("bad stack: " + stack) } @@ -154,11 +154,11 @@ func checkPanicStack() { func TestRuntimeLineInfoAndStack(t *testing.T) { source := runtimeLineInfoProbe - source = strings.ReplaceAll(source, "CALLER_LINE", strconv.Itoa(markerLine(source, "func checkCaller()"))) - source = strings.ReplaceAll(source, "CALLER_SKIP_LINE", strconv.Itoa(markerLine(source, "func checkCallerSkip()"))) - source = strings.ReplaceAll(source, "FUNC_FILELINE_LINE", strconv.Itoa(markerLine(source, "func checkFuncForPC()"))) - source = strings.ReplaceAll(source, "RUNTIME_STACK_LINE", strconv.Itoa(markerLine(source, "func checkRuntimeStack()"))) - source = strings.ReplaceAll(source, "DEBUG_STACK_LINE", strconv.Itoa(markerLine(source, "DEBUG_STACK_MARK"))) + source = strings.ReplaceAll(source, "CALLER_LINE", strconv.Itoa(markerLine(source, "CALLER_MARK"))) + source = strings.ReplaceAll(source, "CALLER_SKIP_LINE", strconv.Itoa(markerLine(source, "CALLER_SKIP_MARK"))) + source = strings.ReplaceAll(source, "FUNC_FILELINE_LINE", strconv.Itoa(markerLine(source, "FUNC_FILELINE_MARK"))) + source = strings.ReplaceAll(source, "RUNTIME_STACK_LINE", strconv.Itoa(markerLine(source, "RUNTIME_STACK_MARK"))) + source = strings.ReplaceAll(source, "DEBUG_STACK_LINE", strconv.Itoa(markerLine(source, "DEBUG_STACK_CALL_MARK"))) dir := t.TempDir() file := filepath.Join(dir, "main.go") diff --git a/test/go/runtime_statement_line_test.go b/test/go/runtime_statement_line_test.go new file mode 100644 index 0000000000..d534b36dc8 --- /dev/null +++ b/test/go/runtime_statement_line_test.go @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2026 The XGo Authors (xgo.dev). All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gotest + +import ( + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const runtimeStatementLineProbe = `package main + +import ( + "runtime" + "runtime/debug" + "strconv" + "strings" +) + +type Wrapper struct { + a []int +} + +func (w Wrapper) Get(i int) int { + return w.a[i] +} + +func main() { + checkCallerStatement() + checkCallersFramesStatement() + checkAdjacentRuntimeStack() + checkRecoveredDebugStackBounds() +} + +//go:noinline +func checkCallerStatement() { + _, file, line, ok := runtime.Caller(0) // CALLER_STMT_MARK + if !ok || !strings.HasSuffix(file, "main.go") || line != CALLER_STMT_LINE { + panic("bad caller statement: " + file + ":" + strconv.Itoa(line)) + } +} + +//go:noinline +func checkCallersFramesStatement() { + var pcs [16]uintptr + n := runtime.Callers(0, pcs[:]) // CALLERS_STMT_MARK + frames := runtime.CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if frame.Function == "main.checkCallersFramesStatement" { + if !strings.HasSuffix(frame.File, "main.go") || frame.Line != CALLERS_STMT_LINE { + panic("bad callers frame: " + frame.File + ":" + strconv.Itoa(frame.Line)) + } + fn := runtime.FuncForPC(frame.PC - 1) + if fn == nil || fn.Name() != "main.checkCallersFramesStatement" { + name := "" + if fn != nil { + name = fn.Name() + } + panic("bad FuncForPC(pc-1): " + name) + } + file, line := fn.FileLine(frame.PC - 1) + if !strings.HasSuffix(file, "main.go") || line != CALLERS_STMT_LINE { + panic("bad Func.FileLine(pc-1): " + file + ":" + strconv.Itoa(line)) + } + return + } + if !more { + break + } + } + panic("missing callers frame") +} + +//go:noinline +func checkAdjacentRuntimeStack() { + var buf1, buf2 [4096]byte + n1 := runtime.Stack(buf1[:], false) // STACK_ONE_MARK + n2 := runtime.Stack(buf2[:], false) // STACK_TWO_MARK + line1 := stackLineFor(string(buf1[:n1]), "main.checkAdjacentRuntimeStack") + line2 := stackLineFor(string(buf2[:n2]), "main.checkAdjacentRuntimeStack") + if line1 != STACK_ONE_LINE || line2 != STACK_TWO_LINE || line1+1 != line2 { + panic("bad adjacent stack lines: " + strconv.Itoa(line1) + "," + strconv.Itoa(line2)) + } +} + +//go:noinline +func checkRecoveredDebugStackBounds() { + defer func() { + if recover() == nil { + panic("missing bounds panic") + } + stack := string(debug.Stack()) + if !strings.Contains(stack, "main.go:BOUNDS_LINE") { + panic("bad recovered stack: " + stack) + } + }() + foo := Wrapper{a: []int{0, 1, 2}} + _ = foo.Get(3) // BOUNDS_MARK +} + +func stackLineFor(stack, fn string) int { + lines := strings.Split(stack, "\n") + for i := 0; i+1 < len(lines); i++ { + if strings.TrimSpace(lines[i]) == fn+"()" { + loc := strings.TrimSpace(lines[i+1]) + colon := strings.LastIndexByte(loc, ':') + if colon < 0 { + return 0 + } + rest := loc[colon+1:] + end := strings.IndexByte(rest, ' ') + if end >= 0 { + rest = rest[:end] + } + n, _ := strconv.Atoi(rest) + return n + } + } + return 0 +} +` + +func TestRuntimeStatementLineInfo(t *testing.T) { + source := runtimeStatementLineProbe + source = strings.ReplaceAll(source, "CALLER_STMT_LINE", strconv.Itoa(markerLine(source, "CALLER_STMT_MARK"))) + source = strings.ReplaceAll(source, "CALLERS_STMT_LINE", strconv.Itoa(markerLine(source, "CALLERS_STMT_MARK"))) + source = strings.ReplaceAll(source, "STACK_ONE_LINE", strconv.Itoa(markerLine(source, "STACK_ONE_MARK"))) + source = strings.ReplaceAll(source, "STACK_TWO_LINE", strconv.Itoa(markerLine(source, "STACK_TWO_MARK"))) + source = strings.ReplaceAll(source, "BOUNDS_LINE", strconv.Itoa(markerLine(source, "BOUNDS_MARK"))) + + dir := t.TempDir() + file := filepath.Join(dir, "main.go") + if err := os.WriteFile(file, []byte(source), 0644); err != nil { + t.Fatal(err) + } + + repoRoot := findStringConversionRepoRoot(t) + t.Setenv("LLGO_ROOT", repoRoot) + cmd := exec.Command("go", "run", "./cmd/llgo", "run", "-a", file) + cmd.Dir = repoRoot + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("llgo statement line probe failed: %v\n%s", err, out) + } +} From 7de6c564c7c4881c344a6b986915305ae29c7c7c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 12:42:11 +0800 Subject: [PATCH 04/23] runtime: compress funcinfo table --- cl/caller_frame_test.go | 56 ++- cl/compile.go | 36 +- cl/instr.go | 178 +++++--- internal/build/funcinfo/funcinfo.go | 264 ++++++++++-- internal/build/funcinfo/funcinfo_test.go | 154 +++++-- internal/build/funcinfo_table.go | 87 ++-- internal/build/funcinfo_table_test.go | 14 +- runtime/internal/lib/runtime/extern.go | 1 + .../lib/runtime/pprof_runtime_stub_llgo.go | 65 ++- runtime/internal/lib/runtime/symtab.go | 128 +++++- runtime/internal/runtime/caller.go | 404 +++++++++++++----- 11 files changed, 1067 insertions(+), 320 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 965086fd2d..4239a062a9 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -135,6 +135,9 @@ import "runtime" import "runtime/debug" func direct() { runtime.Caller(0) } +func indirect() { direct() } +func dynamic(f func()) { f() } +func dynamicCaller() { dynamic(direct) } func stack() { _ = debug.Stack() } func anonOnly() { func() { runtime.FuncForPC(0) }() } func plain() {} @@ -145,6 +148,9 @@ func plain() {} if !fnUsesRuntimeCaller(ssapkg.Func("direct")) { t.Fatal("direct runtime.Caller use should be detected") } + if !fnUsesRuntimeCaller(ssapkg.Func("indirect")) { + t.Fatal("transitive runtime.Caller use should be detected") + } if !fnUsesRuntimeCaller(ssapkg.Func("stack")) { t.Fatal("runtime/debug.Stack use should be detected") } @@ -154,6 +160,15 @@ func plain() {} if fnUsesRuntimeCaller(ssapkg.Func("plain")) { t.Fatal("plain function should not report runtime caller usage") } + runtimeCallerFuncs := runtimeCallerFuncSet(ssapkg) + for _, name := range []string{"dynamic", "dynamicCaller"} { + if !runtimeCallerFuncs[ssapkg.Func(name)] { + t.Fatalf("%s should be tracked because dynamic calls may reach runtime stack APIs", name) + } + } + if runtimeCallerFuncs[ssapkg.Func("plain")] { + t.Fatal("plain function should not be tracked") + } for _, name := range []string{"Caller", "Callers", "CallersFrames", "FuncForPC", "Stack"} { if !isRuntimeCallerName(name) { @@ -208,6 +223,10 @@ func TestCallerFrameTrackingEligibility(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + ssapkg, _ := buildCallerFrameSSAPackage(t, tt.pkgPath, `package foo +import "runtime" +func f() { runtime.Caller(0) } +`) prog := llssa.NewProgram(nil) if tt.targetName != "" { prog.Target().Target = tt.targetName @@ -217,7 +236,15 @@ func TestCallerFrameTrackingEligibility(t *testing.T) { } pkg := prog.NewPackage("foo", tt.pkgPath) fn := pkg.NewFunc("f", llssa.NoArgsNoRet, llssa.InGo) - ctx := &context{prog: prog, pkg: pkg, fn: fn, trackCallerFrames: tt.track} + goFn := ssapkg.Func("f") + ctx := &context{ + prog: prog, + pkg: pkg, + fn: fn, + goFn: goFn, + trackCallerFrames: tt.track, + runtimeCallerFuncs: runtimeCallerFuncSet(ssapkg), + } if got := ctx.shouldTrackCallerFrames(); got != tt.want { t.Fatalf("shouldTrackCallerFrames() = %v, want %v", got, tt.want) } @@ -276,15 +303,18 @@ func f() { } ir := pkg.Module().String() for _, want := range []string{ - "PushCallerFrame", - "SetCallerLookupLine", - "PopCallerFrame", + "RecordCallerLocation", `c"example.com/foo.f`, } { if !strings.Contains(ir, want) { t.Fatalf("compiled caller-frame IR missing %q:\n%s", want, ir) } } + for _, old := range []string{"PushCallerFrame", "SetCallerLine", "PopCallerFrame"} { + if strings.Contains(ir, old) { + t.Fatalf("compiled caller-frame IR still contains old %q instrumentation:\n%s", old, ir) + } + } } func TestCompileRuntimeCallerFrameUsesGoNameForLinkname(t *testing.T) { @@ -325,8 +355,8 @@ func f() { if err != nil { t.Fatal(err) } - if ir := pkg.Module().String(); strings.Contains(ir, "PushCallerFrame") { - t.Fatalf("target builds should not emit caller-frame tracking:\n%s", ir) + if ir := pkg.Module().String(); strings.Contains(ir, "RecordCallerLocation") || strings.Contains(ir, "RecordPanicLocation") { + t.Fatalf("target builds should not emit caller location tracking:\n%s", ir) } ssapkg, files = buildCallerFrameSSAPackage(t, "example.com/foo", `package foo @@ -337,12 +367,12 @@ func f() {} if err != nil { t.Fatal(err) } - if ir := pkg.Module().String(); strings.Contains(ir, "PushCallerFrame") || strings.Contains(ir, "SetCallerLine") { - t.Fatalf("packages without runtime stack APIs should not emit caller-frame tracking:\n%s", ir) + if ir := pkg.Module().String(); strings.Contains(ir, "RecordCallerLocation") || strings.Contains(ir, "RecordPanicLocation") { + t.Fatalf("packages without runtime stack APIs should not emit caller location tracking:\n%s", ir) } } -func TestCompileRuntimeCallerLookupTokenOnlyForRuntimeAPIs(t *testing.T) { +func TestCompileRuntimeCallerLocationOnlyForRuntimePaths(t *testing.T) { ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" @@ -359,10 +389,10 @@ func f() { t.Fatal(err) } ir := pkg.Module().String() - if !strings.Contains(ir, "SetCallerLookupLine") { - t.Fatalf("runtime.Caller should enable caller lookup:\n%s", ir) + if !strings.Contains(ir, "RecordCallerLocation") { + t.Fatalf("runtime.Caller should record caller location:\n%s", ir) } - if !strings.Contains(ir, "SetCallerLine") { - t.Fatalf("ordinary calls in an instrumented package should only update the current line:\n%s", ir) + if strings.Contains(ir, "SetCallerLine") || strings.Contains(ir, "PushCallerFrame") { + t.Fatalf("caller location tracking should not emit old TLS instrumentation:\n%s", ir) } } diff --git a/cl/compile.go b/cl/compile.go index 27282c7461..20531efc4d 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -176,6 +176,7 @@ type context struct { stackDefers map[*ssa.Function]bool anonDefers map[*ssa.Function]bool paramDIVars map[*types.Var]llssa.DIVar + runtimeCallerFuncs map[*ssa.Function]bool patches Patches blkInfos []blocks.Info @@ -747,7 +748,7 @@ func (p *context) compileBlock(b llssa.Builder, block *ssa.BasicBlock, n int, do var ret = fn.Block(block.Index) b.SetBlock(ret) if block.Index == 0 && p.shouldTrackCallerFrames() { - p.pushCallerFrame(b, block.Parent()) + p.pushCallerLocationFrame(b, block.Parent()) } if block.Index == 0 && enableCallTracing && !strings.HasPrefix(fn.Name(), "github.com/goplus/llgo/runtime/internal/runtime.Print") { b.Printf("call " + fn.Name() + "\n\x00") @@ -1137,7 +1138,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue if t := p.type_(v.Type(), llssa.InGo); t.RawType() != nil { if p.isLargeNonPointerValue(t) { x := p.compileValue(b, v.X) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) p.assertNilDerefBase(b, v.X) b.AssertNilDeref(x) return @@ -1151,7 +1152,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue // Zero-length slice-to-array conversions can leave only // an unused slice deref; preserve its required nil check. x := p.compileValue(b, v.X) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) p.assertNilDerefBase(b, v.X) b.AssertNilDeref(x) return @@ -1183,7 +1184,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } x := p.compileValue(b, v.X) if v.Op != token.ARROW { - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) } if shouldAssertDirectNilDeref(v) { b.AssertNilDeref(x) @@ -1220,7 +1221,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue ret = b.Convert(p.type_(t, llssa.InGo), x) case *ssa.FieldAddr: x := p.compileValue(b, v.X) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) if p.isAddressOfFieldAddr(v) { b.AssertNilDeref(x) } @@ -1242,12 +1243,12 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue } x := p.compileValue(b, vx) idx := p.compileValue(b, v.Index) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) ret = b.IndexAddr(x, idx) case *ssa.Index: x := p.compileValue(b, v.X) idx := p.compileValue(b, v.Index) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) ret = b.Index(x, idx, func() (addr llssa.Expr, zero bool) { switch n := v.X.(type) { case *ssa.Const: @@ -1281,7 +1282,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue if v.Max != nil { max = p.compileValue(b, v.Max) } - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) ret = b.Slice(x, low, high, max) ret.Type = p.type_(v.Type(), llssa.InGo) case *ssa.MakeInterface: @@ -1338,7 +1339,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue case *ssa.TypeAssert: x := p.compileValue(b, v.X) t := p.type_(v.AssertedType, llssa.InGo) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) ret = b.TypeAssert(x, t, v.CommaOk) case *ssa.Extract: x := p.compileValue(b, v.Tuple) @@ -1379,7 +1380,7 @@ func (p *context) compileInstrOrValue(b llssa.Builder, iv instrOrValue, asValue case *ssa.SliceToArrayPointer: t := p.type_(v.Type(), llssa.InGo) x := p.compileValue(b, v.X) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) ret = b.SliceToArrayPointer(x, t) default: panic(fmt.Sprintf("compileInstrAndValue: unknown instr - %T\n", iv)) @@ -1514,11 +1515,11 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { } } if p.returnNeedsImplicitRunDefers(v) { - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) b.RunDefers() } if p.shouldTrackCallerFrames() { - p.popCallerFrame(b) + p.popCallerLocationFrame(b) } b.Return(results...) case *ssa.If: @@ -1532,7 +1533,7 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { m := p.compileValue(b, v.Map) key := p.compileValue(b, v.Key) val := p.compileValue(b, v.Value) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) b.MapUpdate(m, key, val) case *ssa.Defer: if v.DeferStack != nil { @@ -1543,16 +1544,16 @@ func (p *context) compileInstr(b llssa.Builder, instr ssa.Instruction) { case *ssa.Go: p.call(b, llssa.Go, &v.Call) case *ssa.RunDefers: - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) b.RunDefers() case *ssa.Panic: arg := p.compileValue(b, v.X) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) b.Panic(arg) case *ssa.Send: ch := p.compileValue(b, v.Chan) x := p.compileValue(b, v.X) - p.setCallerLine(b, v.Pos()) + p.recordPanicLocation(b, v.Pos()) b.Send(ch, x) case *ssa.DebugRef: if enableDbgSyms && v.Parent().Origin() == nil { @@ -1869,7 +1870,8 @@ func newPackageEx(prog llssa.Program, patches Patches, rewrites map[string]strin cgoSymbols: make([]string, 0, 128), rewrites: rewrites, - trackCallerFrames: filesUseRuntimeCaller(files) || packageUsesRuntimeCaller(pkg), + trackCallerFrames: filesUseRuntimeCaller(files) || packageUsesRuntimeCaller(pkg), + runtimeCallerFuncs: runtimeCallerFuncSet(pkg), } if embedMap != nil { ctx.embedMap = *embedMap diff --git a/cl/instr.go b/cl/instr.go index 0c6a2d69a6..357107167b 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -854,7 +854,10 @@ func (p *context) sourceLine(filename string, line int) (string, bool) { } func (p *context) shouldTrackCallerFrames() bool { - if p == nil || p.pkg == nil || p.fn == nil || !p.trackCallerFrames { + if p == nil || p.pkg == nil || p.fn == nil || p.goFn == nil || !p.trackCallerFrames { + return false + } + if !p.runtimeCallerFuncs[p.goFn] { return false } if target := p.prog.Target(); target != nil && (target.Target != "" || target.GOARCH == "wasm") { @@ -875,19 +878,50 @@ func isStandardLibraryPackage(pkgPath string) bool { } func packageUsesRuntimeCaller(pkg *ssa.Package) bool { + return len(runtimeCallerFuncSet(pkg)) != 0 +} + +func fnUsesRuntimeCaller(fn *ssa.Function) bool { + return runtimeCallerFuncSetFor(fn, make(map[*ssa.Function]bool), make(map[*ssa.Function]bool), false) +} + +func runtimeCallerFuncSet(pkg *ssa.Package) map[*ssa.Function]bool { if pkg == nil { - return false + return nil + } + dynamicCallsMayReachRuntimeCaller := packageHasDirectRuntimeCaller(pkg) + if !dynamicCallsMayReachRuntimeCaller { + return nil } + memo := make(map[*ssa.Function]bool) + visiting := make(map[*ssa.Function]bool) for _, member := range pkg.Members { - fn, ok := member.(*ssa.Function) - if ok && fnUsesRuntimeCaller(fn) { + if fn, ok := member.(*ssa.Function); ok { + runtimeCallerFuncSetFor(fn, memo, visiting, dynamicCallsMayReachRuntimeCaller) + } + } + out := make(map[*ssa.Function]bool) + for fn, ok := range memo { + if ok { + out[fn] = true + } + } + if len(out) == 0 { + return nil + } + return out +} + +func packageHasDirectRuntimeCaller(pkg *ssa.Package) bool { + for _, member := range pkg.Members { + if fn, ok := member.(*ssa.Function); ok && fnHasDirectRuntimeCaller(fn) { return true } } return false } -func fnUsesRuntimeCaller(fn *ssa.Function) bool { +func fnHasDirectRuntimeCaller(fn *ssa.Function) bool { if fn == nil { return false } @@ -903,10 +937,50 @@ func fnUsesRuntimeCaller(fn *ssa.Function) bool { } } for _, anon := range fn.AnonFuncs { - if fnUsesRuntimeCaller(anon) { + if fnHasDirectRuntimeCaller(anon) { + return true + } + } + return false +} + +func runtimeCallerFuncSetFor(fn *ssa.Function, memo, visiting map[*ssa.Function]bool, dynamicCallsMayReachRuntimeCaller bool) bool { + if fn == nil { + return false + } + if ok, done := memo[fn]; done { + return ok + } + if visiting[fn] { + return false + } + visiting[fn] = true + defer delete(visiting, fn) + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + call, ok := instr.(ssa.CallInstruction) + if !ok { + continue + } + callee := call.Common().StaticCallee() + if callee == nil && dynamicCallsMayReachRuntimeCaller { + memo[fn] = true + return true + } + if isRuntimeCallerFunc(callee) || + (callee != nil && callee.Pkg == fn.Pkg && runtimeCallerFuncSetFor(callee, memo, visiting, dynamicCallsMayReachRuntimeCaller)) { + memo[fn] = true + return true + } + } + } + for _, anon := range fn.AnonFuncs { + if runtimeCallerFuncSetFor(anon, memo, visiting, dynamicCallsMayReachRuntimeCaller) { + memo[fn] = true return true } } + memo[fn] = false return false } @@ -949,21 +1023,6 @@ func isRuntimeCallerName(name string) bool { } } -func (p *context) pushCallerFrame(b llssa.Builder, fn *ssa.Function) { - if fn == nil { - return - } - pos := p.fset.Position(fn.Pos()) - entry := b.Convert(p.prog.Uintptr(), p.fn.Expr) - p.callerFrameMark = b.Call( - p.runtimeFunc("PushCallerFrame", pushCallerFrameSig()), - entry, - b.Str(p.runtimeCallerFrameName()), - b.Str(pos.Filename), - p.prog.IntVal(uint64(pos.Line), p.prog.Int()), - ) -} - func (p *context) runtimeCallerFrameName() string { if p == nil { return "" @@ -977,43 +1036,63 @@ func (p *context) runtimeCallerFrameName() string { return "" } -func (p *context) setCallerLine(b llssa.Builder, pos token.Pos) { - if !p.shouldTrackCallerFrames() { +func (p *context) pushCallerLocationFrame(b llssa.Builder, fn *ssa.Function) { + if fn == nil { return } - line := p.fset.Position(pos).Line - p.setCallerLineNumber(b, line) + pos := p.fset.Position(fn.Pos()) + entry := b.Convert(p.prog.Uintptr(), p.fn.Expr) + p.callerFrameMark = b.Call( + p.runtimeFunc("PushCallerLocationFrame", pushCallerLocationFrameSig()), + entry, + b.Str(p.runtimeCallerFrameName()), + b.Str(pos.Filename), + p.prog.IntVal(uint64(pos.Line), p.prog.Int()), + ) +} + +func (p *context) recordCallerLocation(b llssa.Builder, pos token.Pos) { + p.recordRuntimeLocation(b, pos, "RecordCallerLocation") } -func (p *context) setCallerLineForCall(b llssa.Builder, call *ssa.CallCommon) { +func (p *context) recordPanicLocation(b llssa.Builder, pos token.Pos) { + p.recordRuntimeLocation(b, pos, "RecordPanicLocation") +} + +func (p *context) recordRuntimeLocation(b llssa.Builder, pos token.Pos, fn string) { if !p.shouldTrackCallerFrames() { return } - line := p.fset.Position(call.Pos()).Line - if line <= 0 { + position := p.fset.Position(pos) + if position.Line <= 0 || position.Filename == "" { return } - fn := "SetCallerLine" - sig := setCallerLineSig() - if isRuntimeCallerLookupFunc(call.StaticCallee()) { - fn = "SetCallerLookupLine" - sig = setCallerLookupLineSig() - } - b.Call(p.runtimeFunc(fn, sig), p.prog.IntVal(uint64(line), p.prog.Int())) + b.Call( + p.runtimeFunc(fn, recordRuntimeLocationSig()), + b.Convert(p.prog.Uintptr(), p.fn.Expr), + b.Str(p.runtimeCallerFrameName()), + b.Str(position.Filename), + p.prog.IntVal(uint64(position.Line), p.prog.Int()), + ) } -func (p *context) setCallerLineNumber(b llssa.Builder, line int) { - if line <= 0 { +func (p *context) recordCallerLocationForCall(b llssa.Builder, call *ssa.CallCommon) { + if !p.shouldTrackCallerFrames() { + return + } + callee := call.StaticCallee() + if isRuntimeCallerLookupFunc(callee) { + p.recordCallerLocation(b, call.Pos()) return } - b.Call(p.runtimeFunc("SetCallerLine", setCallerLineSig()), p.prog.IntVal(uint64(line), p.prog.Int())) + p.recordPanicLocation(b, call.Pos()) } -func (p *context) popCallerFrame(b llssa.Builder) { +func (p *context) popCallerLocationFrame(b llssa.Builder) { if p.callerFrameMark.IsNil() { return } - b.Call(p.runtimeFunc("PopCallerFrame", popCallerFrameSig()), p.callerFrameMark) + b.Call(p.runtimeFunc("PopCallerLocationFrame", popCallerLocationFrameSig()), p.callerFrameMark) } func (p *context) runtimeFunc(name string, sig *types.Signature) llssa.Expr { @@ -1025,7 +1104,7 @@ func (p *context) runtimeFunc(name string, sig *types.Signature) llssa.Expr { return p.pkg.NewFuncEx(fullName, sig, llssa.InGo, false, false).Expr } -func pushCallerFrameSig() *types.Signature { +func pushCallerLocationFrameSig() *types.Signature { return types.NewSignatureType(nil, nil, nil, types.NewTuple( types.NewVar(token.NoPos, nil, "entry", types.Typ[types.Uintptr]), @@ -1038,19 +1117,20 @@ func pushCallerFrameSig() *types.Signature { ) } -func setCallerLineSig() *types.Signature { +func recordRuntimeLocationSig() *types.Signature { return types.NewSignatureType(nil, nil, nil, - types.NewTuple(types.NewVar(token.NoPos, nil, "line", types.Typ[types.Int])), + types.NewTuple( + types.NewVar(token.NoPos, nil, "entry", types.Typ[types.Uintptr]), + types.NewVar(token.NoPos, nil, "name", types.Typ[types.String]), + types.NewVar(token.NoPos, nil, "file", types.Typ[types.String]), + types.NewVar(token.NoPos, nil, "line", types.Typ[types.Int]), + ), nil, false, ) } -func setCallerLookupLineSig() *types.Signature { - return setCallerLineSig() -} - -func popCallerFrameSig() *types.Signature { +func popCallerLocationFrameSig() *types.Signature { return types.NewSignatureType(nil, nil, nil, types.NewTuple(types.NewVar(token.NoPos, nil, "mark", types.Typ[types.Int])), nil, @@ -1275,7 +1355,7 @@ func collectMethodNilDerefChecks(fn *ssa.Function) map[*ssa.UnOp]none { } func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallCommon, ds *explicitDeferStack) (ret llssa.Expr) { - p.setCallerLineForCall(b, call) + p.recordCallerLocationForCall(b, call) cv := call.Value if mthd := call.Method; mthd != nil { reflectCheck := p.reflectTypeMethodCheck(call, mthd) diff --git a/internal/build/funcinfo/funcinfo.go b/internal/build/funcinfo/funcinfo.go index c092b606e9..c6043484c6 100644 --- a/internal/build/funcinfo/funcinfo.go +++ b/internal/build/funcinfo/funcinfo.go @@ -32,56 +32,62 @@ type Record struct { } type EncodedRecord struct { - Symbol uint32 - Name uint32 - File uint32 - Line uint32 - Column uint32 + SymbolPkg uint16 + SymbolName uint16 + NamePkg uint16 + NameName uint16 + FileRoot uint16 + FileName uint16 + Line uint32 } type Table struct { - Records []EncodedRecord - Strings []byte - Hash []uint32 + Records []EncodedRecord + StringOffsets []uint32 + Strings []byte + Hash []uint16 } func Encode(records []Record) (Table, error) { if len(records) == 0 { return Table{}, nil } - pool := stringPool{ - offsets: map[string]uint32{"": 0}, - data: []byte{0}, - text: "\x00", - } - for _, s := range collectStrings(records) { - if _, err := pool.offset(s); err != nil { - return Table{}, err - } + ids, offsets, strings, err := buildStringTable(collectStrings(records)) + if err != nil { + return Table{}, err } out := Table{ - Records: make([]EncodedRecord, 0, len(records)), + Records: make([]EncodedRecord, 0, len(records)), + StringOffsets: offsets, + Strings: strings, } for _, rec := range records { + symPkg, symName := splitQualifiedName(rec.Symbol) + namePkg, nameName := splitQualifiedName(rec.Name) + fileRoot, fileName := splitFileName(rec.File) out.Records = append(out.Records, EncodedRecord{ - Symbol: pool.offsets[rec.Symbol], - Name: pool.offsets[rec.Name], - File: pool.offsets[rec.File], - Line: rec.Line, - Column: rec.Column, + SymbolPkg: ids[symPkg], + SymbolName: ids[symName], + NamePkg: ids[namePkg], + NameName: ids[nameName], + FileRoot: ids[fileRoot], + FileName: ids[fileName], + Line: rec.Line, }) } - out.Strings = pool.data - out.Hash = buildHash(records) + out.Hash, err = buildHash(records) + if err != nil { + return Table{}, err + } return out, nil } func collectStrings(records []Record) []string { seen := make(map[string]bool) for _, rec := range records { - seen[rec.Symbol] = true - seen[rec.Name] = true - seen[rec.File] = true + for _, s := range splitRecordStrings(rec) { + seen[s] = true + } } delete(seen, "") out := make([]string, 0, len(seen)) @@ -97,6 +103,69 @@ func collectStrings(records []Record) []string { return out } +func splitRecordStrings(rec Record) []string { + symPkg, symName := splitQualifiedName(rec.Symbol) + namePkg, nameName := splitQualifiedName(rec.Name) + fileRoot, fileName := splitFileName(rec.File) + return []string{symPkg, symName, namePkg, nameName, fileRoot, fileName} +} + +func buildStringTable(strings []string) (map[string]uint16, []uint32, []byte, error) { + ids := map[string]uint16{"": 0} + values := []string{""} + for _, s := range strings { + if _, ok := ids[s]; ok { + continue + } + if len(values) > math.MaxUint16 { + return nil, nil, nil, fmt.Errorf("funcinfo string id table exceeds 65535 entries") + } + ids[s] = uint16(len(values)) + values = append(values, s) + } + pool := stringPool{ + offsets: map[string]uint32{"": 0}, + data: []byte{0}, + text: "\x00", + } + offsets := make([]uint32, len(values)) + for id, s := range values { + off, err := pool.offset(s) + if err != nil { + return nil, nil, nil, err + } + offsets[id] = off + } + return ids, offsets, pool.data, nil +} + +func splitQualifiedName(name string) (pkg, local string) { + if name == "" { + return "", "" + } + start := strings.LastIndexByte(name, '/') + if start < 0 { + start = 0 + } else { + start++ + } + if dot := strings.IndexByte(name[start:], '.'); dot >= 0 { + idx := start + dot + return name[:idx], name[idx+1:] + } + return "", name +} + +func splitFileName(file string) (root, name string) { + if file == "" { + return "", "" + } + if slash := strings.LastIndexByte(file, '/'); slash >= 0 { + return file[:slash+1], file[slash+1:] + } + return "", file +} + type stringPool struct { offsets map[string]uint32 data []byte @@ -123,23 +192,26 @@ func (p *stringPool) offset(s string) (uint32, error) { return off, nil } -func buildHash(records []Record) []uint32 { +func buildHash(records []Record) ([]uint16, error) { if len(records) == 0 { - return nil + return nil, nil + } + if len(records) > math.MaxUint16 { + return nil, nil } buckets := 1 for buckets*3 < len(records)*4 { buckets <<= 1 } - hash := make([]uint32, buckets) + hash := make([]uint16, buckets) for i, rec := range records { slot := int(HashString(rec.Symbol) & uint32(buckets-1)) for hash[slot] != 0 { slot = (slot + 1) & (buckets - 1) } - hash[slot] = uint32(i + 1) + hash[slot] = uint16(i + 1) } - return hash + return hash, nil } func HashString(s string) uint32 { @@ -154,3 +226,129 @@ func HashString(s string) uint32 { } return h } + +func (t Table) String(id uint16) string { + if int(id) >= len(t.StringOffsets) { + return "" + } + return cstring(t.Strings, t.StringOffsets[id]) +} + +func (t Table) Symbol(rec EncodedRecord) string { + return joinQualified(t.String(rec.SymbolPkg), t.String(rec.SymbolName)) +} + +func (t Table) Name(rec EncodedRecord) string { + return joinQualified(t.String(rec.NamePkg), t.String(rec.NameName)) +} + +func (t Table) File(rec EncodedRecord) string { + return t.String(rec.FileRoot) + t.String(rec.FileName) +} + +func (t Table) LookupSymbol(symbol string) (int, bool) { + if len(t.Hash) == 0 { + return 0, false + } + mask := uint32(len(t.Hash) - 1) + slot := HashString(symbol) & mask + for probes := 0; probes < len(t.Hash); probes++ { + idx := t.Hash[slot] + if idx == 0 { + return 0, false + } + rec := t.Records[idx-1] + if t.Symbol(rec) == symbol { + return int(idx - 1), true + } + slot = (slot + 1) & mask + } + return 0, false +} + +func (t Table) SizeBytes() int { + return len(t.Records)*16 + len(t.StringOffsets)*4 + len(t.Strings) + len(t.Hash)*2 +} + +func joinQualified(pkg, local string) string { + if pkg == "" { + return local + } + if local == "" { + return pkg + } + return pkg + "." + local +} + +func cstring(data []byte, off uint32) string { + end := int(off) + for end < len(data) && data[end] != 0 { + end++ + } + return string(data[off:end]) +} + +type PCIndex struct { + PageShift uint + Base uint64 + Pages []uint32 +} + +const DefaultPCPageShift = 12 + +func BuildPCIndex(entries []uint64) PCIndex { + return BuildPCIndexWithShift(entries, DefaultPCPageShift) +} + +func BuildPCIndexWithShift(entries []uint64, shift uint) PCIndex { + if len(entries) == 0 { + return PCIndex{PageShift: shift} + } + base := entries[0] >> shift + last := entries[len(entries)-1] >> shift + pages := make([]uint32, last-base+2) + next := 0 + for page := range pages { + limit := (base + uint64(page)) << shift + for next < len(entries) && entries[next] < limit { + next++ + } + pages[page] = uint32(next) + } + return PCIndex{ + PageShift: shift, + Base: base, + Pages: pages, + } +} + +func LookupPC(entries []uint64, index PCIndex, pc uint64) int { + if len(entries) == 0 { + return -1 + } + lo, hi := 0, len(entries) + page := pc >> index.PageShift + if len(index.Pages) != 0 && page >= index.Base { + off := page - index.Base + if off < uint64(len(index.Pages)) { + lo = int(index.Pages[off]) + if off+1 < uint64(len(index.Pages)) { + hi = int(index.Pages[off+1]) + } + if lo > 0 { + lo-- + } + if hi < len(entries) { + hi++ + } + } + } + i := sort.Search(hi-lo, func(i int) bool { + return entries[lo+i] > pc + }) + idx := lo + i - 1 + if idx < 0 { + return -1 + } + return idx +} diff --git a/internal/build/funcinfo/funcinfo_test.go b/internal/build/funcinfo/funcinfo_test.go index 7bc92ec8b1..238543d3e9 100644 --- a/internal/build/funcinfo/funcinfo_test.go +++ b/internal/build/funcinfo/funcinfo_test.go @@ -29,27 +29,27 @@ func TestEncodePoolsStringsAndBuildsHash(t *testing.T) { if len(table.Records) != 2 { t.Fatalf("encoded records = %d, want 2", len(table.Records)) } - if table.Records[0].File == table.Records[1].File { - t.Fatalf("suffix sharing should not collapse distinct file strings to the same offset") + if table.Records[0].FileRoot == table.Records[1].FileRoot { + t.Fatalf("distinct file roots should use distinct ids") } - if got := cstring(table.Strings, table.Records[1].File); got != "shared.go" { + if got := table.File(table.Records[1]); got != "shared.go" { t.Fatalf("suffix file string = %q, want shared.go", got) } if len(table.Hash) == 0 || len(table.Hash)&(len(table.Hash)-1) != 0 { t.Fatalf("hash bucket count = %d, want power-of-two non-zero", len(table.Hash)) } - if idx, ok := lookup(table, "example.com/p.a"); !ok || idx != 0 { + if idx, ok := table.LookupSymbol("example.com/p.a"); !ok || idx != 0 { t.Fatalf("lookup a = %d, %v; want 0, true", idx, ok) } - if idx, ok := lookup(table, "example.com/p.b"); !ok || idx != 1 { + if idx, ok := table.LookupSymbol("example.com/p.b"); !ok || idx != 1 { t.Fatalf("lookup b = %d, %v; want 1, true", idx, ok) } - if _, ok := lookup(table, "missing"); ok { + if _, ok := table.LookupSymbol("missing"); ok { t.Fatalf("lookup missing succeeded") } } -func TestEncodeUsesUint32Records(t *testing.T) { +func TestEncodeRoundTripsSingleRecord(t *testing.T) { table, err := Encode([]Record{{Symbol: "s", Name: "n", File: "f", Line: 1, Column: 2}}) if err != nil { t.Fatal(err) @@ -58,17 +58,17 @@ func TestEncodeUsesUint32Records(t *testing.T) { t.Fatalf("records = %d, want %d", got, want) } rec := table.Records[0] - if got, want := cstring(table.Strings, rec.Symbol), "s"; got != want { + if got, want := table.Symbol(rec), "s"; got != want { t.Fatalf("symbol = %q, want %q", got, want) } - if got, want := cstring(table.Strings, rec.Name), "n"; got != want { + if got, want := table.Name(rec), "n"; got != want { t.Fatalf("name = %q, want %q", got, want) } - if got, want := cstring(table.Strings, rec.File), "f"; got != want { + if got, want := table.File(rec), "f"; got != want { t.Fatalf("file = %q, want %q", got, want) } - if rec.Line != 1 || rec.Column != 2 { - t.Fatalf("source position = %d:%d, want 1:2", rec.Line, rec.Column) + if rec.Line != 1 { + t.Fatalf("source line = %d, want 1", rec.Line) } } @@ -81,14 +81,101 @@ func TestEncodeHashHandlesCollisions(t *testing.T) { if err != nil { t.Fatal(err) } - if idx, ok := lookup(table, a); !ok || idx != 0 { + if idx, ok := table.LookupSymbol(a); !ok || idx != 0 { t.Fatalf("lookup collision a = %d, %v; want 0, true", idx, ok) } - if idx, ok := lookup(table, b); !ok || idx != 1 { + if idx, ok := table.LookupSymbol(b); !ok || idx != 1 { t.Fatalf("lookup collision b = %d, %v; want 1, true", idx, ok) } } +func TestEncodeOmitsHashWhenRecordIndexesDoNotFitUint16(t *testing.T) { + records := make([]Record, 1<<16) + for i := range records { + records[i] = Record{Symbol: "example.com/p.f", Name: "example.com/p.F"} + } + table, err := Encode(records) + if err != nil { + t.Fatal(err) + } + if table.Hash != nil { + t.Fatalf("hash buckets = %d, want nil fallback for oversized table", len(table.Hash)) + } + if len(table.Records) != len(records) { + t.Fatalf("records = %d, want %d", len(table.Records), len(records)) + } +} + +func TestEncodeSplitsPackageAndFilePrefixes(t *testing.T) { + records := []Record{ + {Symbol: "example.com/p.alpha", Name: "example.com/p.Alpha", File: "/home/me/mod/p/alpha.go", Line: 10}, + {Symbol: "example.com/p.beta", Name: "example.com/p.Beta", File: "/home/me/mod/p/beta.go", Line: 20}, + {Symbol: "example.com/q.gamma", Name: "example.com/q.Gamma", File: "/home/me/mod/q/gamma.go", Line: 30}, + } + table, err := Encode(records) + if err != nil { + t.Fatal(err) + } + for i, rec := range table.Records { + if got := table.Symbol(rec); got != records[i].Symbol { + t.Fatalf("record %d symbol = %q, want %q", i, got, records[i].Symbol) + } + if got := table.Name(rec); got != records[i].Name { + t.Fatalf("record %d name = %q, want %q", i, got, records[i].Name) + } + if got := table.File(rec); got != records[i].File { + t.Fatalf("record %d file = %q, want %q", i, got, records[i].File) + } + } + if table.Records[0].SymbolPkg != table.Records[1].SymbolPkg { + t.Fatalf("same package prefix got different ids: %d vs %d", table.Records[0].SymbolPkg, table.Records[1].SymbolPkg) + } + if table.Records[0].FileRoot != table.Records[1].FileRoot { + t.Fatalf("same file root got different ids: %d vs %d", table.Records[0].FileRoot, table.Records[1].FileRoot) + } + if got := table.SizeBytes(); got >= legacySizeBytes(records) { + t.Fatalf("compressed table size = %d, want below legacy %d", got, legacySizeBytes(records)) + } +} + +func TestLookupPCUsesPageIndex(t *testing.T) { + entries := []uint64{0x1000, 0x1010, 0x2800, 0x4000, 0x4010} + index := BuildPCIndex(entries) + tests := []struct { + pc uint64 + want int + }{ + {0xfff, -1}, + {0x1000, 0}, + {0x100f, 0}, + {0x1010, 1}, + {0x27ff, 1}, + {0x2800, 2}, + {0x4018, 4}, + } + for _, tt := range tests { + if got := LookupPC(entries, index, tt.pc); got != tt.want { + t.Fatalf("LookupPC(%#x) = %d, want %d", tt.pc, got, tt.want) + } + } +} + +func BenchmarkLookupPCRandom(b *testing.B) { + entries := make([]uint64, 8192) + for i := range entries { + entries[i] = 0x100000 + uint64(i)*37 + } + index := BuildPCIndex(entries) + var sum int + for i := 0; i < b.N; i++ { + pc := entries[(i*1103515245+12345)&(len(entries)-1)] + uint64(i&31) + sum += LookupPC(entries, index, pc) + } + if sum == 0 { + b.Fatal(sum) + } +} + func collisionPair(t *testing.T) (string, string) { t.Helper() const mask = uint32(3) @@ -105,30 +192,21 @@ func collisionPair(t *testing.T) (string, string) { return "", "" } -func cstring(data []byte, off uint32) string { - end := int(off) - for end < len(data) && data[end] != 0 { - end++ - } - return string(data[off:end]) -} - -func lookup(table Table, symbol string) (int, bool) { - if len(table.Hash) == 0 { - return 0, false - } - mask := uint32(len(table.Hash) - 1) - slot := HashString(symbol) & mask - for probes := 0; probes < len(table.Hash); probes++ { - idx := table.Hash[slot] - if idx == 0 { - return 0, false +func legacySizeBytes(records []Record) int { + seen := make(map[string]bool) + stringsBytes := 1 + for _, rec := range records { + for _, s := range []string{rec.Symbol, rec.Name, rec.File} { + if s == "" || seen[s] { + continue + } + seen[s] = true + stringsBytes += len(s) + 1 } - rec := table.Records[idx-1] - if cstring(table.Strings, rec.Symbol) == symbol { - return int(idx - 1), true - } - slot = (slot + 1) & mask } - return 0, false + buckets := 1 + for buckets*3 < len(records)*4 { + buckets <<= 1 + } + return len(records)*20 + stringsBytes + buckets*4 } diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 30c63c5fe4..37402229dd 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -26,14 +26,16 @@ import ( ) const ( - funcInfoTableSymbol = "__llgo_funcinfo_table" - funcInfoCountSymbol = "__llgo_funcinfo_count" - funcInfoStringsSymbol = "__llgo_funcinfo_strings" - funcInfoHashSymbol = "__llgo_funcinfo_hash" - funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" - funcInfoDataSymbol = "__llgo_funcinfo_table$data" - funcInfoStringsDataSymbol = "__llgo_funcinfo_strings$data" - funcInfoHashDataSymbol = "__llgo_funcinfo_hash$data" + funcInfoTableSymbol = "__llgo_funcinfo_table" + funcInfoCountSymbol = "__llgo_funcinfo_count" + funcInfoStringsSymbol = "__llgo_funcinfo_strings" + funcInfoStringOffsetsSymbol = "__llgo_funcinfo_string_offsets" + funcInfoHashSymbol = "__llgo_funcinfo_hash" + funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" + funcInfoDataSymbol = "__llgo_funcinfo_table$data" + funcInfoStringsDataSymbol = "__llgo_funcinfo_strings$data" + funcInfoStringOffsetsDataSymbol = "__llgo_funcinfo_string_offsets$data" + funcInfoHashDataSymbol = "__llgo_funcinfo_hash$data" ) type funcInfoRecord struct { @@ -125,24 +127,29 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord mod := pkg.Module() llvmCtx := mod.Context() i8Type := llvmCtx.Int8Type() + i16Type := llvmCtx.Int16Type() i32Type := llvmCtx.Int32Type() countType := llvmCtx.IntType(ctx.prog.PointerSize() * 8) recordType := llvmCtx.StructType([]llvm.Type{ - i32Type, - i32Type, - i32Type, - i32Type, + i16Type, + i16Type, + i16Type, + i16Type, + i16Type, + i16Type, i32Type, }, false) tablePtr := llvm.AddGlobal(mod, llvm.PointerType(recordType, 0), funcInfoTableSymbol) stringsPtr := llvm.AddGlobal(mod, llvm.PointerType(i8Type, 0), funcInfoStringsSymbol) - hashPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoHashSymbol) + stringOffsetsPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoStringOffsetsSymbol) + hashPtr := llvm.AddGlobal(mod, llvm.PointerType(i16Type, 0), funcInfoHashSymbol) count := llvm.AddGlobal(mod, countType, funcInfoCountSymbol) hashMask := llvm.AddGlobal(mod, countType, funcInfoHashMaskSymbol) if len(records) == 0 { tablePtr.SetInitializer(llvm.ConstPointerNull(tablePtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) + stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) count.SetInitializer(llvm.ConstInt(countType, 0, false)) hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) @@ -157,11 +164,13 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord values := make([]llvm.Value, 0, len(encoded.Records)) for _, rec := range encoded.Records { values = append(values, llvm.ConstNamedStruct(recordType, []llvm.Value{ - llvm.ConstInt(i32Type, uint64(rec.Symbol), false), - llvm.ConstInt(i32Type, uint64(rec.Name), false), - llvm.ConstInt(i32Type, uint64(rec.File), false), + llvm.ConstInt(i16Type, uint64(rec.SymbolPkg), false), + llvm.ConstInt(i16Type, uint64(rec.SymbolName), false), + llvm.ConstInt(i16Type, uint64(rec.NamePkg), false), + llvm.ConstInt(i16Type, uint64(rec.NameName), false), + llvm.ConstInt(i16Type, uint64(rec.FileRoot), false), + llvm.ConstInt(i16Type, uint64(rec.FileName), false), llvm.ConstInt(i32Type, uint64(rec.Line), false), - llvm.ConstInt(i32Type, uint64(rec.Column), false), })) } arrayType := llvm.ArrayType(recordType, len(values)) @@ -180,17 +189,17 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord stringData.SetUnnamedAddr(true) stringData.SetAlignment(1) - hashValues := make([]llvm.Value, 0, len(encoded.Hash)) - for _, idx := range encoded.Hash { - hashValues = append(hashValues, llvm.ConstInt(i32Type, uint64(idx), false)) + stringOffsetValues := make([]llvm.Value, 0, len(encoded.StringOffsets)) + for _, off := range encoded.StringOffsets { + stringOffsetValues = append(stringOffsetValues, llvm.ConstInt(i32Type, uint64(off), false)) } - hashArrayType := llvm.ArrayType(i32Type, len(hashValues)) - hashData := llvm.AddGlobal(mod, hashArrayType, funcInfoHashDataSymbol) - hashData.SetInitializer(llvm.ConstArray(i32Type, hashValues)) - hashData.SetLinkage(llvm.PrivateLinkage) - hashData.SetGlobalConstant(true) - hashData.SetUnnamedAddr(true) - hashData.SetAlignment(4) + stringOffsetsArrayType := llvm.ArrayType(i32Type, len(stringOffsetValues)) + stringOffsetsData := llvm.AddGlobal(mod, stringOffsetsArrayType, funcInfoStringOffsetsDataSymbol) + stringOffsetsData.SetInitializer(llvm.ConstArray(i32Type, stringOffsetValues)) + stringOffsetsData.SetLinkage(llvm.PrivateLinkage) + stringOffsetsData.SetGlobalConstant(true) + stringOffsetsData.SetUnnamedAddr(true) + stringOffsetsData.SetAlignment(4) tablePtr.SetInitializer(llvm.ConstInBoundsGEP(arrayType, data, []llvm.Value{ llvm.ConstInt(countType, 0, false), @@ -200,12 +209,32 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord llvm.ConstInt(countType, 0, false), llvm.ConstInt(countType, 0, false), })) - hashPtr.SetInitializer(llvm.ConstInBoundsGEP(hashArrayType, hashData, []llvm.Value{ + stringOffsetsPtr.SetInitializer(llvm.ConstInBoundsGEP(stringOffsetsArrayType, stringOffsetsData, []llvm.Value{ llvm.ConstInt(countType, 0, false), llvm.ConstInt(countType, 0, false), })) + if len(encoded.Hash) == 0 { + hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) + hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) + } else { + hashValues := make([]llvm.Value, 0, len(encoded.Hash)) + for _, idx := range encoded.Hash { + hashValues = append(hashValues, llvm.ConstInt(i16Type, uint64(idx), false)) + } + hashArrayType := llvm.ArrayType(i16Type, len(hashValues)) + hashData := llvm.AddGlobal(mod, hashArrayType, funcInfoHashDataSymbol) + hashData.SetInitializer(llvm.ConstArray(i16Type, hashValues)) + hashData.SetLinkage(llvm.PrivateLinkage) + hashData.SetGlobalConstant(true) + hashData.SetUnnamedAddr(true) + hashData.SetAlignment(2) + hashPtr.SetInitializer(llvm.ConstInBoundsGEP(hashArrayType, hashData, []llvm.Value{ + llvm.ConstInt(countType, 0, false), + llvm.ConstInt(countType, 0, false), + })) + hashMask.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Hash)-1), false)) + } count.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Records)), false)) - hashMask.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Hash)-1), false)) } func toFuncInfoRecords(records []funcInfoRecord) []buildfuncinfo.Record { diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index f7367ebf56..649a20a8bc 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -56,17 +56,18 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) for _, want := range []string{ "@__llgo_funcinfo_table = global ptr", "@__llgo_funcinfo_strings = global ptr", + "@__llgo_funcinfo_string_offsets = global ptr", "@__llgo_funcinfo_hash = global ptr", "@__llgo_funcinfo_count = global i64 1", "@__llgo_funcinfo_hash_mask = global i64 1", - `@"__llgo_funcinfo_table$data" = private unnamed_addr constant [1 x { i32, i32, i32, i32, i32 }]`, - `@"__llgo_funcinfo_strings$data" = private unnamed_addr constant [47 x i8]`, - `@"__llgo_funcinfo_hash$data" = private unnamed_addr constant [2 x i32]`, - `example.com/p.live\00`, - `example.com/p.Live\00`, + `@"__llgo_funcinfo_table$data" = private unnamed_addr constant [1 x { i16, i16, i16, i16, i16, i16, i32 }]`, + `@"__llgo_funcinfo_string_offsets$data" = private unnamed_addr constant`, + `@"__llgo_funcinfo_hash$data" = private unnamed_addr constant [2 x i16]`, + `example.com/p\00`, + `live\00`, + `Live\00`, `live.go\00`, "i32 17", - "i32 3", } { if !strings.Contains(ir, want) { t.Fatalf("funcinfo table IR missing %q:\n%s", want, ir) @@ -138,6 +139,7 @@ func TestFuncInfoTableEmptyDefinitions(t *testing.T) { for _, want := range []string{ "@__llgo_funcinfo_table = global ptr null", "@__llgo_funcinfo_strings = global ptr null", + "@__llgo_funcinfo_string_offsets = global ptr null", "@__llgo_funcinfo_hash = global ptr null", "@__llgo_funcinfo_count = global i64 0", "@__llgo_funcinfo_hash_mask = global i64 0", diff --git a/runtime/internal/lib/runtime/extern.go b/runtime/internal/lib/runtime/extern.go index d6835b794f..66f95de36f 100644 --- a/runtime/internal/lib/runtime/extern.go +++ b/runtime/internal/lib/runtime/extern.go @@ -54,6 +54,7 @@ func callers(skip int, pc []uintptr) int { } pc[n] = fr.PC recordFrameSymbol(fr.PC, fr.Offset, fr.Name) + rtdebug.BindCallerLocation(fr.PC, fr.Name) n++ return true }) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index bd131c9a25..8fac1ada4b 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -2,7 +2,10 @@ package runtime -import llrt "github.com/goplus/llgo/runtime/internal/runtime" +import ( + "github.com/goplus/llgo/runtime/internal/clite/tls" + llrt "github.com/goplus/llgo/runtime/internal/runtime" +) type StackRecord struct { Stack []uintptr @@ -84,10 +87,32 @@ func NumGoroutine() int { func SetCPUProfileRate(hz int) {} +const funcForPCCacheSize = 256 + +type funcForPCCacheEntry struct { + pc uintptr + fn *Func +} + +type funcForPCCache struct { + entries [funcForPCCacheSize]funcForPCCacheEntry +} + +var funcForPCCacheTLS = tls.Alloc[*funcForPCCache](nil) + func FuncForPC(pc uintptr) *Func { + if fn := cachedFuncForPC(pc); fn != nil { + return fn + } sym := frameSymbol(pc) + fn := newFuncForPC(pc, sym) + cacheFuncForPC(pc, fn) + return fn +} + +func newFuncForPC(pc uintptr, sym pcSymbol) *Func { if !sym.ok && sym.function == "" { - return &Func{entry: pc, name: unknownFunctionName(pc)} + return &Func{entry: pc, name: unknownFunctionName(pc), pc: pc} } name := sym.function if name == "" { @@ -97,5 +122,39 @@ func FuncForPC(pc uintptr) *Func { if entry == 0 { entry = pc } - return &Func{entry: entry, name: name} + return &Func{ + entry: entry, + name: name, + pc: pc, + file: sym.file, + line: sym.line, + } +} + +func cachedFuncForPC(pc uintptr) *Func { + cache := funcForPCCacheTLS.Get() + if cache == nil { + return nil + } + entry := &cache.entries[funcForPCCacheIndex(pc)] + if entry.pc == pc && entry.fn != nil { + return entry.fn + } + return nil +} + +func cacheFuncForPC(pc uintptr, fn *Func) { + cache := funcForPCCacheTLS.Get() + if cache == nil { + cache = new(funcForPCCache) + funcForPCCacheTLS.Set(cache) + } + cache.entries[funcForPCCacheIndex(pc)] = funcForPCCacheEntry{ + pc: pc, + fn: fn, + } +} + +func funcForPCCacheIndex(pc uintptr) uintptr { + return (pc >> 4) & (funcForPCCacheSize - 1) } diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index bd0a616018..919cd74ed3 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -135,11 +135,13 @@ func recordFrameSymbol(pc, offset uintptr, name string) { } type runtimeFuncInfoRecord struct { - symbol uint32 - name uint32 - file uint32 - line uint32 - column uint32 + symbolPkg uint16 + symbolName uint16 + namePkg uint16 + nameName uint16 + fileRoot uint16 + fileName uint16 + line uint32 } //go:linkname runtimeFuncInfoTable __llgo_funcinfo_table @@ -148,8 +150,11 @@ var runtimeFuncInfoTable *runtimeFuncInfoRecord //go:linkname runtimeFuncInfoStrings __llgo_funcinfo_strings var runtimeFuncInfoStrings *c.Char +//go:linkname runtimeFuncInfoStringOffsets __llgo_funcinfo_string_offsets +var runtimeFuncInfoStringOffsets *uint32 + //go:linkname runtimeFuncInfoHash __llgo_funcinfo_hash -var runtimeFuncInfoHash *uint32 +var runtimeFuncInfoHash *uint16 //go:linkname runtimeFuncInfoCount __llgo_funcinfo_count var runtimeFuncInfoCount uintptr @@ -180,10 +185,6 @@ func publicFunctionName(name string) string { return name } -func cStringEqual(cstr *c.Char, s string) bool { - return cStringCompare(cstr, s) == 0 -} - func cStringCompare(cstr *c.Char, s string) int { if cstr == nil { if s == "" { @@ -212,10 +213,37 @@ func cStringCompare(cstr *c.Char, s string) int { } } -func funcInfoCString(off uint32) *c.Char { - if runtimeFuncInfoStrings == nil { +func cStringLen(cstr *c.Char) int { + if cstr == nil { + return 0 + } + ptr := unsafe.Pointer(cstr) + for i := 0; ; i++ { + if *(*byte)(unsafe.Add(ptr, i)) == 0 { + return i + } + } +} + +func cStringAppend(dst []byte, cstr *c.Char) []byte { + if cstr == nil { + return dst + } + ptr := unsafe.Pointer(cstr) + for i := 0; ; i++ { + c := *(*byte)(unsafe.Add(ptr, i)) + if c == 0 { + return dst + } + dst = append(dst, c) + } +} + +func funcInfoCString(id uint16) *c.Char { + if runtimeFuncInfoStrings == nil || runtimeFuncInfoStringOffsets == nil { return nil } + off := *(*uint32)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoStringOffsets), uintptr(id)*unsafe.Sizeof(*runtimeFuncInfoStringOffsets))) return (*c.Char)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoStrings), uintptr(off))) } @@ -237,6 +265,58 @@ func funcInfoHashString(s string) uintptr { return uintptr(h) } +func funcInfoSymbolEqual(rec *runtimeFuncInfoRecord, symbol string) bool { + pkg := funcInfoCString(rec.symbolPkg) + name := funcInfoCString(rec.symbolName) + pkgLen := cStringLen(pkg) + nameLen := cStringLen(name) + if pkgLen == 0 { + return cStringCompare(name, symbol) == 0 + } + if len(symbol) != pkgLen+1+nameLen { + return false + } + if cStringCompare(pkg, symbol[:pkgLen]) != 0 || symbol[pkgLen] != '.' { + return false + } + return cStringCompare(name, symbol[pkgLen+1:]) == 0 +} + +func funcInfoJoinName(pkgID, nameID uint16) string { + pkg := funcInfoCString(pkgID) + name := funcInfoCString(nameID) + pkgLen := cStringLen(pkg) + nameLen := cStringLen(name) + if pkgLen == 0 { + return safeGoString(name, "") + } + if nameLen == 0 { + return safeGoString(pkg, "") + } + buf := make([]byte, 0, pkgLen+1+nameLen) + buf = cStringAppend(buf, pkg) + buf = append(buf, '.') + buf = cStringAppend(buf, name) + return string(buf) +} + +func funcInfoJoinFile(rootID, nameID uint16) string { + root := funcInfoCString(rootID) + name := funcInfoCString(nameID) + rootLen := cStringLen(root) + nameLen := cStringLen(name) + if rootLen == 0 { + return safeGoString(name, "") + } + if nameLen == 0 { + return safeGoString(root, "") + } + buf := make([]byte, 0, rootLen+nameLen) + buf = cStringAppend(buf, root) + buf = cStringAppend(buf, name) + return string(buf) +} + func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { if symbol == "" || runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 { return nil @@ -244,13 +324,13 @@ func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { if runtimeFuncInfoHash != nil && runtimeFuncInfoHashMask != 0 { slot := funcInfoHashString(symbol) & runtimeFuncInfoHashMask for probes := uintptr(0); probes <= runtimeFuncInfoHashMask; probes++ { - idx := *(*uint32)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoHash), slot*unsafe.Sizeof(*runtimeFuncInfoHash))) + idx := *(*uint16)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoHash), slot*unsafe.Sizeof(*runtimeFuncInfoHash))) if idx == 0 { return nil } if uintptr(idx) <= runtimeFuncInfoCount { rec := funcInfoAt(uintptr(idx) - 1) - if cStringEqual(funcInfoCString(rec.symbol), symbol) { + if funcInfoSymbolEqual(rec, symbol) { return rec } } @@ -260,7 +340,7 @@ func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { } for i := uintptr(0); i < runtimeFuncInfoCount; i++ { rec := funcInfoAt(i) - if cStringEqual(funcInfoCString(rec.symbol), symbol) { + if funcInfoSymbolEqual(rec, symbol) { return rec } } @@ -278,10 +358,10 @@ func applyFuncInfo(sym *pcSymbol, rawFunction string) { if rec == nil { return } - if name := safeGoString(funcInfoCString(rec.name), ""); name != "" { + if name := funcInfoJoinName(rec.namePkg, rec.nameName); name != "" { sym.function = publicFunctionName(name) } - if file := safeGoString(funcInfoCString(rec.file), ""); file != "" { + if file := funcInfoJoinFile(rec.fileRoot, rec.fileName); file != "" { if sym.file == "" { sym.file = file } @@ -392,7 +472,13 @@ func (ci *Frames) Next() (frame Frame, more bool) { } var f *Func if sym.entry != 0 || fn != "" { - f = &Func{entry: sym.entry, name: fn} + f = &Func{ + entry: sym.entry, + name: fn, + pc: pc, + file: sym.file, + line: sym.line, + } } ci.frames = append(ci.frames, Frame{ PC: pc, @@ -438,6 +524,9 @@ func CallersFrames(callers []uintptr) *Frames { type Func struct { entry uintptr name string + pc uintptr + file string + line int } func (f *Func) Name() string { @@ -455,6 +544,9 @@ func (f *Func) Entry() uintptr { } func (f *Func) FileLine(pc uintptr) (file string, line int) { + if f != nil && f.pc == pc && (f.file != "" || f.line != 0) { + return f.file, f.line + } sym := frameSymbol(pc) return sym.file, sym.line } diff --git a/runtime/internal/runtime/caller.go b/runtime/internal/runtime/caller.go index ba2a212fed..e4cbd3cf03 100644 --- a/runtime/internal/runtime/caller.go +++ b/runtime/internal/runtime/caller.go @@ -19,6 +19,7 @@ package runtime import ( "unsafe" + clitedebug "github.com/goplus/llgo/runtime/internal/clite/debug" "github.com/goplus/llgo/runtime/internal/clite/tls" ) @@ -31,35 +32,28 @@ type CallerFrame struct { StartLine int } +const callerLocationLimit = 4096 + const ( callerPCMask = uintptr(3) callerPCValue = uintptr(1) callersPCValue = uintptr(3) - callerPCRingSize = 1024 + callerPCHashInit = 64 ) -type callerPCStore struct { - next uintptr - frames [callerPCRingSize]CallerFrame +type callerLocationStore struct { + frames []CallerFrame + stack []CallerFrame + synthetic []CallerFrame + syntheticHash []uintptr } -var ( - callerFrameTLS = tls.Alloc[[]CallerFrame](nil) - callerPCStoreTLS = tls.Alloc[*callerPCStore](nil) - callerLookupTLS = tls.Alloc[bool](nil) - panicCallerFrameTLS = tls.Alloc[[]CallerFrame](nil) -) +var callerLocationTLS = tls.Alloc[*callerLocationStore](nil) -var ( - runtimeCallersFrame = CallerFrame{Function: "runtime.Callers"} - runtimeMainFrame = CallerFrame{Function: "runtime.main"} - runtimeGoexitFrame = CallerFrame{Function: "runtime.goexit"} -) - -func PushCallerFrame(entry uintptr, name, file string, startLine int) int { - frames := callerFrameTLS.Get() - mark := len(frames) - frames = append(frames, CallerFrame{ +func PushCallerLocationFrame(entry uintptr, name, file string, startLine int) int { + store := callerLocationStoreForThread() + mark := len(store.stack) + store.stack = append(store.stack, CallerFrame{ PC: entry, Entry: entry, Function: name, @@ -67,107 +61,114 @@ func PushCallerFrame(entry uintptr, name, file string, startLine int) int { Line: startLine, StartLine: startLine, }) - callerFrameTLS.Set(frames) return mark } -func SetCallerLine(line int) { - frames := callerFrameTLS.Get() - if line <= 0 || len(frames) == 0 { +func PopCallerLocationFrame(mark int) { + store := callerLocationTLS.Get() + if store == nil { return } - frames[len(frames)-1].Line = line - callerFrameTLS.Set(frames) -} - -func SetCallerLookupLine(line int) { - SetCallerLine(line) - callerLookupTLS.Set(true) -} - -func PopCallerFrame(mark int) { - frames := callerFrameTLS.Get() - oldLen := len(frames) + oldLen := len(store.stack) if mark < 0 || mark > oldLen { return } var zero CallerFrame for i := mark; i < oldLen; i++ { - frames[i] = zero + store.stack[i] = zero } - callerFrameTLS.Set(frames[:mark]) + store.stack = store.stack[:mark] +} - panicFrames := panicCallerFrameTLS.Get() - if len(panicFrames) > 0 && oldLen >= len(panicFrames) && mark <= len(panicFrames) { - for i := range panicFrames { - panicFrames[i] = zero - } - panicCallerFrameTLS.Clear() +func RecordCallerLocation(entry uintptr, name, file string, line int) { + if entry == 0 || line <= 0 { + return } + updateCurrentFrame(entry, name, file, line) + recordPCLocation(0, entry, name, file, line) } -func SavePanicCallerFrames() { - frames := callerFrameTLS.Get() - if len(frames) == 0 { - panicCallerFrameTLS.Clear() +func RecordPanicLocation(entry uintptr, name, file string, line int) { + if entry == 0 || line <= 0 { + return + } + updateCurrentFrame(entry, name, file, line) + recordPCLocation(0, entry, name, file, line) +} + +func updateCurrentFrame(entry uintptr, name, file string, line int) { + store := callerLocationTLS.Get() + if store == nil { return } - panicFrames := panicCallerFrameTLS.Get() - if cap(panicFrames) < len(frames) { - panicFrames = make([]CallerFrame, len(frames)) - } else { - panicFrames = panicFrames[:len(frames)] + for i := len(store.stack) - 1; i >= 0; i-- { + frame := &store.stack[i] + if frame.Entry == entry { + frame.Function = name + frame.File = file + frame.Line = line + return + } } - copy(panicFrames, frames) - panicCallerFrameTLS.Set(panicFrames) } -func Caller(skip int) (CallerFrame, bool) { - if !takeCallerLookup() { - return CallerFrame{}, false +func recordPCLocation(pc, entry uintptr, name, file string, line int) { + store := callerLocationStoreForThread() + for i := range store.frames { + frame := &store.frames[i] + if (pc != 0 && frame.PC == pc) || (pc == 0 && frame.PC == 0 && frame.Entry == entry) { + frame.PC = pc + frame.Entry = entry + frame.Function = name + frame.File = file + frame.Line = line + return + } + } + if len(store.frames) >= callerLocationLimit { + copy(store.frames, store.frames[1:]) + store.frames[len(store.frames)-1] = CallerFrame{} + store.frames = store.frames[:len(store.frames)-1] } + store.frames = append(store.frames, CallerFrame{ + PC: pc, + Entry: entry, + Function: name, + File: file, + Line: line, + }) +} + +func Caller(skip int) (CallerFrame, bool) { if skip < 0 { return CallerFrame{}, false } - frames := callerFrameTLS.Get() - panicFrames := panicCallerFrameTLS.Get() - if len(frames) == 0 { - if skip < len(panicFrames) { - return captureFrame(panicFrames[len(panicFrames)-1-skip], callerPCValue), true - } + store := callerLocationTLS.Get() + if store == nil || len(store.stack) == 0 { return CallerFrame{}, false } - if skip < len(frames) { - return captureFrame(frames[len(frames)-1-skip], callerPCValue), true + if skip < len(store.stack) { + return store.captureFrame(store.stack[len(store.stack)-1-skip], callerPCValue), true } - if len(panicFrames) > len(frames) { - idx := len(panicFrames) - 1 - skip - if idx >= 0 { - return captureFrame(panicFrames[idx], callerPCValue), true - } - } - switch skip - len(frames) { + switch skip - len(store.stack) { case 0: - return captureFrame(runtimeMainFrame, callerPCValue), true + return store.captureFrame(runtimeMainFrame, callerPCValue), true case 1: - return captureFrame(runtimeGoexitFrame, callerPCValue), true + return store.captureFrame(runtimeGoexitFrame, callerPCValue), true default: return CallerFrame{}, false } } func Callers(skip int, pcs []uintptr) int { - if !takeCallerLookup() { + if len(pcs) == 0 { return 0 } if skip < 0 { skip = 0 } - frames := callerFrameTLS.Get() - if len(frames) == 0 { - frames = panicCallerFrameTLS.Get() - } - if len(frames) == 0 { + store := callerLocationTLS.Get() + if store == nil || len(store.stack) == 0 { return 0 } n := 0 @@ -179,15 +180,15 @@ func Callers(skip int, pcs []uintptr) int { if n >= len(pcs) { return false } - pcs[n] = captureFrame(frame, callersPCValue).PC + pcs[n] = store.captureFrame(frame, callersPCValue).PC n++ return true } if !add(runtimeCallersFrame) { return n } - for i := len(frames) - 1; i >= 0; i-- { - if !add(frames[i]) { + for i := len(store.stack) - 1; i >= 0; i-- { + if !add(store.stack[i]) { return n } } @@ -196,59 +197,234 @@ func Callers(skip int, pcs []uintptr) int { return n } -func takeCallerLookup() bool { - if !callerLookupTLS.Get() { - return false +func SavePanicCallerFrames() { +} + +func BindCallerLocation(pc uintptr, rawName string) { + store := callerLocationTLS.Get() + if store == nil || pc == 0 { + return + } + if frame, ok := callerLocationByName(store, rawName); ok { + bindCallerLocationPC(pc, frame) + return + } +} + +var ( + runtimeCallersFrame = CallerFrame{Function: "runtime.Callers"} + runtimeMainFrame = CallerFrame{Function: "runtime.main"} + runtimeGoexitFrame = CallerFrame{Function: "runtime.goexit"} +) + +func callerLocationByName(store *callerLocationStore, rawName string) (CallerFrame, bool) { + if rawName == "" { + return CallerFrame{}, false + } + name := normalizeRuntimeFuncName(rawName) + for i := len(store.frames) - 1; i >= 0; i-- { + frame := store.frames[i] + if frame.PC == 0 && frame.Function == name && frame.Line != 0 { + return frame, true + } + } + return CallerFrame{}, false +} + +func bindCallerLocationPC(pc uintptr, frame CallerFrame) { + recordPCLocation(pc, frame.Entry, frame.Function, frame.File, frame.Line) + if pc > 0 { + recordPCLocation(pc-1, frame.Entry, frame.Function, frame.File, frame.Line) } - callerLookupTLS.Set(false) - return true } func FrameForPC(pc uintptr) (CallerFrame, bool) { - if pc&callerPCMask == 0 { + if pc&callerPCMask != 0 { + if frame, ok := syntheticFrameForPC(pc); ok { + return frame, true + } + } + store := callerLocationTLS.Get() + if store == nil || pc == 0 { + return CallerFrame{}, false + } + for i := len(store.frames) - 1; i >= 0; i-- { + frame := store.frames[i] + if frame.PC == pc { + return frame, true + } + } + entry := entryForPC(pc) + if entry == 0 { return CallerFrame{}, false } - store := callerPCStoreTLS.Get() + var best CallerFrame + for _, frame := range store.frames { + if frame.PC == 0 || frame.PC > pc || frame.Entry != entry { + continue + } + if best.PC == 0 || frame.PC > best.PC { + best = frame + } + } + if best.PC != 0 { + best.PC = pc + return best, true + } + for i := len(store.frames) - 1; i >= 0; i-- { + frame := store.frames[i] + if frame.PC == 0 && frame.Entry == entry { + frame.PC = pc + return frame, true + } + } + return CallerFrame{}, false +} + +func syntheticFrameForPC(pc uintptr) (CallerFrame, bool) { + store := callerLocationTLS.Get() if store == nil { return CallerFrame{}, false } - addr := pc &^ callerPCMask - if !store.contains(addr) { + seq := pc >> 2 + if seq == 0 || seq > uintptr(len(store.synthetic)) { + return CallerFrame{}, false + } + frame := store.synthetic[seq-1] + if frame.PC>>2 != seq { return CallerFrame{}, false } - frame := *(*CallerFrame)(unsafe.Pointer(addr)) + frame.PC = pc + if frame.Entry == 0 { + frame.Entry = pc + } return frame, true } -func callerPCStoreForThread() *callerPCStore { - store := callerPCStoreTLS.Get() +func callerLocationStoreForThread() *callerLocationStore { + store := callerLocationTLS.Get() if store == nil { - store = new(callerPCStore) - callerPCStoreTLS.Set(store) + store = new(callerLocationStore) + callerLocationTLS.Set(store) } return store } -func captureFrame(frame CallerFrame, pcValue uintptr) CallerFrame { - store := callerPCStoreForThread() - idx := store.next & (callerPCRingSize - 1) - store.next++ - store.frames[idx] = frame - rec := &store.frames[idx] - pc := uintptr(unsafe.Pointer(rec)) | pcValue - rec.PC = pc +func (s *callerLocationStore) captureFrame(frame CallerFrame, pcValue uintptr) CallerFrame { + idx := s.internSyntheticFrame(frame) + rec := s.synthetic[idx] + seq := uintptr(idx + 1) + rec.PC = (seq << 2) | pcValue if rec.Entry == 0 { - rec.Entry = pc + rec.Entry = rec.PC + } + return rec +} + +func (s *callerLocationStore) internSyntheticFrame(frame CallerFrame) int { + if len(s.syntheticHash) == 0 { + s.syntheticHash = make([]uintptr, callerPCHashInit) + } + if len(s.synthetic)*2 >= len(s.syntheticHash) { + s.rehashSyntheticFrames(len(s.syntheticHash) * 2) + } + slot := s.syntheticSlot(frame) + for { + idx := s.syntheticHash[slot] + if idx == 0 { + frame.PC = (uintptr(len(s.synthetic)+1) << 2) | callerPCValue + s.synthetic = append(s.synthetic, frame) + s.syntheticHash[slot] = uintptr(len(s.synthetic)) + return len(s.synthetic) - 1 + } + existing := s.synthetic[idx-1] + if sameSyntheticFrame(existing, frame) { + return int(idx - 1) + } + slot = (slot + 1) & (uintptr(len(s.syntheticHash)) - 1) + } +} + +func (s *callerLocationStore) rehashSyntheticFrames(size int) { + old := s.syntheticHash + s.syntheticHash = make([]uintptr, size) + for _, idx := range old { + if idx == 0 { + continue + } + frame := s.synthetic[idx-1] + slot := s.syntheticSlot(frame) + for s.syntheticHash[slot] != 0 { + slot = (slot + 1) & (uintptr(len(s.syntheticHash)) - 1) + } + s.syntheticHash[slot] = idx + } +} + +func (s *callerLocationStore) syntheticSlot(frame CallerFrame) uintptr { + h := frame.Entry ^ (uintptr(frame.Line) << 12) ^ (uintptr(frame.StartLine) << 24) + h ^= uintptr(len(frame.Function)) << 4 + h ^= uintptr(len(frame.File)) << 8 + return h & (uintptr(len(s.syntheticHash)) - 1) +} + +func sameSyntheticFrame(a, b CallerFrame) bool { + return a.Entry == b.Entry && + a.Function == b.Function && + a.File == b.File && + a.Line == b.Line && + a.StartLine == b.StartLine +} + +func entryForPC(pc uintptr) uintptr { + var info clitedebug.Info + if clitedebug.Addrinfo(unsafe.Pointer(pc), &info) == 0 { + return 0 + } + return uintptr(info.Saddr) +} + +func normalizeRuntimeFuncName(name string) string { + const commandLineArguments = "command-line-arguments." + if hasPrefix(name, commandLineArguments) { + name = "main." + name[len(commandLineArguments):] } - return *rec + if len(name) > 0 && name[0] == '_' { + name = name[1:] + } + return normalizeRuntimeAnonFuncName(name) } -func (s *callerPCStore) contains(addr uintptr) bool { - start := uintptr(unsafe.Pointer(&s.frames[0])) - size := unsafe.Sizeof(s.frames) - end := start + size - if addr < start || addr >= end { +func normalizeRuntimeAnonFuncName(name string) string { + dollar := lastIndexByte(name, '$') + if dollar < 0 || dollar == len(name)-1 { + return name + } + for i := dollar + 1; i < len(name); i++ { + if name[i] < '0' || name[i] > '9' { + return name + } + } + return name[:dollar] + ".func" + name[dollar+1:] +} + +func hasPrefix(s, prefix string) bool { + if len(s) < len(prefix) { return false } - return (addr-start)%unsafe.Sizeof(s.frames[0]) == 0 + for i := 0; i < len(prefix); i++ { + if s[i] != prefix[i] { + return false + } + } + return true +} + +func lastIndexByte(s string, c byte) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == c { + return i + } + } + return -1 } From 29164c004be6f3f35a2f3c1e3583f7c96f128418 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 13:47:42 +0800 Subject: [PATCH 05/23] test: cover indirect runtime caller paths --- cl/caller_frame_test.go | 10 +++++- test/go/runtime_statement_line_test.go | 49 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 4239a062a9..b53fe93d97 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -134,10 +134,18 @@ func TestRuntimeCallerPackageDetection(t *testing.T) { import "runtime" import "runtime/debug" +type callerIface interface { Call() } +type callerImpl struct{} + func direct() { runtime.Caller(0) } func indirect() { direct() } func dynamic(f func()) { f() } func dynamicCaller() { dynamic(direct) } +func (callerImpl) Call() { direct() } +func interfaceDispatch(c callerIface) { c.Call() } +func interfaceCaller(c callerIface) { interfaceDispatch(c) } +func closureLayer(next func()) func() { return func() { next() } } +func closureCaller() { closureLayer(closureLayer(direct))() } func stack() { _ = debug.Stack() } func anonOnly() { func() { runtime.FuncForPC(0) }() } func plain() {} @@ -161,7 +169,7 @@ func plain() {} t.Fatal("plain function should not report runtime caller usage") } runtimeCallerFuncs := runtimeCallerFuncSet(ssapkg) - for _, name := range []string{"dynamic", "dynamicCaller"} { + for _, name := range []string{"dynamic", "dynamicCaller", "interfaceDispatch", "interfaceCaller", "closureLayer", "closureCaller"} { if !runtimeCallerFuncs[ssapkg.Func(name)] { t.Fatalf("%s should be tracked because dynamic calls may reach runtime stack APIs", name) } diff --git a/test/go/runtime_statement_line_test.go b/test/go/runtime_statement_line_test.go index d534b36dc8..81430b49f0 100644 --- a/test/go/runtime_statement_line_test.go +++ b/test/go/runtime_statement_line_test.go @@ -45,6 +45,8 @@ func (w Wrapper) Get(i int) int { func main() { checkCallerStatement() checkCallersFramesStatement() + checkInterfaceIndirectCaller() + checkClosureIndirectCaller() checkAdjacentRuntimeStack() checkRecoveredDebugStackBounds() } @@ -89,6 +91,51 @@ func checkCallersFramesStatement() { panic("missing callers frame") } +type indirectCaller interface { + call() +} + +type indirectCallerImpl struct{} + +//go:noinline +func checkInterfaceIndirectCaller() { + var c indirectCaller = indirectCallerImpl{} + c.call() // INTERFACE_CALL_MARK +} + +//go:noinline +func (indirectCallerImpl) call() { + interfaceMiddle() +} + +//go:noinline +func interfaceMiddle() { + checkCallerLine("interface", 2, INTERFACE_CALL_LINE) +} + +//go:noinline +func checkClosureIndirectCaller() { + f := closureLayer(closureLayer(func() { + checkCallerLine("closure", 3, CLOSURE_CALL_LINE) + })) + f() // CLOSURE_CALL_MARK +} + +//go:noinline +func closureLayer(next func()) func() { + return func() { + next() + } +} + +//go:noinline +func checkCallerLine(kind string, skip, want int) { + _, file, line, ok := runtime.Caller(skip) + if !ok || !strings.HasSuffix(file, "main.go") || line != want { + panic("bad " + kind + " indirect caller line: " + file + ":" + strconv.Itoa(line)) + } +} + //go:noinline func checkAdjacentRuntimeStack() { var buf1, buf2 [4096]byte @@ -142,6 +189,8 @@ func TestRuntimeStatementLineInfo(t *testing.T) { source := runtimeStatementLineProbe source = strings.ReplaceAll(source, "CALLER_STMT_LINE", strconv.Itoa(markerLine(source, "CALLER_STMT_MARK"))) source = strings.ReplaceAll(source, "CALLERS_STMT_LINE", strconv.Itoa(markerLine(source, "CALLERS_STMT_MARK"))) + source = strings.ReplaceAll(source, "INTERFACE_CALL_LINE", strconv.Itoa(markerLine(source, "INTERFACE_CALL_MARK"))) + source = strings.ReplaceAll(source, "CLOSURE_CALL_LINE", strconv.Itoa(markerLine(source, "CLOSURE_CALL_MARK"))) source = strings.ReplaceAll(source, "STACK_ONE_LINE", strconv.Itoa(markerLine(source, "STACK_ONE_MARK"))) source = strings.ReplaceAll(source, "STACK_TWO_LINE", strconv.Itoa(markerLine(source, "STACK_TWO_MARK"))) source = strings.ReplaceAll(source, "BOUNDS_LINE", strconv.Itoa(markerLine(source, "BOUNDS_MARK"))) From 300e8bb0509713b8e63535251ef6fbc3cd6cad27 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 15:42:16 +0800 Subject: [PATCH 06/23] test: yield during makefunc GC stress --- test/go/reflect_makefunc_goroutine_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/go/reflect_makefunc_goroutine_test.go b/test/go/reflect_makefunc_goroutine_test.go index 0f4e4da273..e37aece153 100644 --- a/test/go/reflect_makefunc_goroutine_test.go +++ b/test/go/reflect_makefunc_goroutine_test.go @@ -20,6 +20,7 @@ func TestReflectMakeFuncGoroutineStartup(t *testing.T) { return default: runtime.GC() + runtime.Gosched() } } }() From 3f905e9b13e8a8d78b166282d227abf84e1e59b9 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 19:18:19 +0800 Subject: [PATCH 07/23] cl: narrow runtime caller tracking --- cl/caller_frame_test.go | 13 ++ cl/instr.go | 356 +++++++++++++++++++++++++++++++++++----- 2 files changed, 330 insertions(+), 39 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index b53fe93d97..2a4c6eda4b 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -136,6 +136,8 @@ import "runtime/debug" type callerIface interface { Call() } type callerImpl struct{} +type workerIface interface { Work() } +type workerImpl struct{} func direct() { runtime.Caller(0) } func indirect() { direct() } @@ -148,6 +150,12 @@ func closureLayer(next func()) func() { return func() { next() } } func closureCaller() { closureLayer(closureLayer(direct))() } func stack() { _ = debug.Stack() } func anonOnly() { func() { runtime.FuncForPC(0) }() } +func leaf() {} +func callFunc(f func()) { f() } +func callFuncHot() { callFunc(leaf) } +func (workerImpl) Work() {} +func callWorker(w workerIface) { w.Work() } +func workerHot() { var w workerIface = workerImpl{}; callWorker(w) } func plain() {} `) if !packageUsesRuntimeCaller(ssapkg) { @@ -174,6 +182,11 @@ func plain() {} t.Fatalf("%s should be tracked because dynamic calls may reach runtime stack APIs", name) } } + for _, name := range []string{"leaf", "callFunc", "callFuncHot", "callWorker", "workerHot"} { + if runtimeCallerFuncs[ssapkg.Func(name)] { + t.Fatalf("%s should not be tracked when resolved dynamic targets do not reach runtime stack APIs", name) + } + } if runtimeCallerFuncs[ssapkg.Func("plain")] { t.Fatal("plain function should not be tracked") } diff --git a/cl/instr.go b/cl/instr.go index 357107167b..2a87f99334 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -882,28 +882,41 @@ func packageUsesRuntimeCaller(pkg *ssa.Package) bool { } func fnUsesRuntimeCaller(fn *ssa.Function) bool { - return runtimeCallerFuncSetFor(fn, make(map[*ssa.Function]bool), make(map[*ssa.Function]bool), false) + if fn == nil { + return false + } + if fn.Pkg == nil { + return fnHasDirectRuntimeCaller(fn) + } + return runtimeCallerFuncSet(fn.Pkg)[fn] } func runtimeCallerFuncSet(pkg *ssa.Package) map[*ssa.Function]bool { if pkg == nil { return nil } - dynamicCallsMayReachRuntimeCaller := packageHasDirectRuntimeCaller(pkg) - if !dynamicCallsMayReachRuntimeCaller { - return nil + funcs, trackable := collectRuntimeCallerFunctions(pkg) + analysis := &runtimeCallerAnalysis{ + pkg: pkg, + funcs: funcs, + trackable: trackable, + callsites: collectRuntimeCallerCallsites(funcs), + memo: make(map[*ssa.Function]bool), + visiting: make(map[*ssa.Function]bool), } - memo := make(map[*ssa.Function]bool) - visiting := make(map[*ssa.Function]bool) - for _, member := range pkg.Members { - if fn, ok := member.(*ssa.Function); ok { - runtimeCallerFuncSetFor(fn, memo, visiting, dynamicCallsMayReachRuntimeCaller) - } + if !analysis.packageHasRuntimeCaller() { + return nil } out := make(map[*ssa.Function]bool) - for fn, ok := range memo { - if ok { - out[fn] = true + for { + ntrack := len(trackable) + for fn := range trackable { + if analysis.fnMayReachRuntimeCaller(fn) { + out[fn] = true + } + } + if len(trackable) == ntrack { + break } } if len(out) == 0 { @@ -912,9 +925,105 @@ func runtimeCallerFuncSet(pkg *ssa.Package) map[*ssa.Function]bool { return out } -func packageHasDirectRuntimeCaller(pkg *ssa.Package) bool { +type runtimeCallerAnalysis struct { + pkg *ssa.Package + funcs map[*ssa.Function]bool + trackable map[*ssa.Function]bool + callsites map[*ssa.Function][]*ssa.CallCommon + memo map[*ssa.Function]bool + visiting map[*ssa.Function]bool +} + +func collectRuntimeCallerFunctions(pkg *ssa.Package) (funcs, trackable map[*ssa.Function]bool) { + funcs = make(map[*ssa.Function]bool) + trackable = make(map[*ssa.Function]bool) + var add func(*ssa.Function, bool) bool + add = func(fn *ssa.Function, track bool) bool { + if fn == nil || !functionBelongsToPackage(pkg, fn) { + return false + } + if track { + trackable[fn] = true + } + if funcs[fn] { + return false + } + funcs[fn] = true + for _, anon := range fn.AnonFuncs { + add(anon, false) + } + return true + } for _, member := range pkg.Members { - if fn, ok := member.(*ssa.Function); ok && fnHasDirectRuntimeCaller(fn) { + if fn, ok := member.(*ssa.Function); ok { + add(fn, true) + } + } + if pkg.Prog != nil && pkg.Pkg != nil { + for _, typ := range pkg.Prog.RuntimeTypes() { + if !typeBelongsToPackage(typ, pkg.Pkg) { + continue + } + methods := pkg.Prog.MethodSets.MethodSet(typ) + for i := 0; i < methods.Len(); i++ { + add(pkg.Prog.MethodValue(methods.At(i)), false) + } + } + } + for changed := true; changed; { + changed = false + for fn := range funcs { + forEachCall(fn, func(call *ssa.CallCommon) { + if add(call.StaticCallee(), trackable[fn]) { + changed = true + } + }) + } + } + return funcs, trackable +} + +func collectRuntimeCallerCallsites(funcs map[*ssa.Function]bool) map[*ssa.Function][]*ssa.CallCommon { + callsites := make(map[*ssa.Function][]*ssa.CallCommon) + for fn := range funcs { + forEachCall(fn, func(call *ssa.CallCommon) { + callee := call.StaticCallee() + if funcs[callee] { + callsites[callee] = append(callsites[callee], call) + } + }) + } + return callsites +} + +func functionBelongsToPackage(pkg *ssa.Package, fn *ssa.Function) bool { + if pkg == nil || fn == nil { + return false + } + if fn.Pkg == pkg { + return true + } + return fn.Pkg == nil && fn.Parent() != nil && functionBelongsToPackage(pkg, fn.Parent()) +} + +func typeBelongsToPackage(typ types.Type, pkg *types.Package) bool { + if pkg == nil { + return false + } + for { + if ptr, ok := types.Unalias(typ).(*types.Pointer); ok { + typ = ptr.Elem() + continue + } + break + } + named, ok := types.Unalias(typ).(*types.Named) + return ok && named.Obj() != nil && named.Obj().Pkg() == pkg +} + +func (a *runtimeCallerAnalysis) packageHasRuntimeCaller() bool { + for fn := range a.funcs { + if fnHasDirectRuntimeCaller(fn) { return true } } @@ -944,46 +1053,215 @@ func fnHasDirectRuntimeCaller(fn *ssa.Function) bool { return false } -func runtimeCallerFuncSetFor(fn *ssa.Function, memo, visiting map[*ssa.Function]bool, dynamicCallsMayReachRuntimeCaller bool) bool { +func (a *runtimeCallerAnalysis) fnMayReachRuntimeCaller(fn *ssa.Function) bool { if fn == nil { return false } - if ok, done := memo[fn]; done { + if isRuntimeCallerFunc(fn) { + return true + } + if !a.funcs[fn] { + return false + } + if ok, done := a.memo[fn]; done { return ok } - if visiting[fn] { + if a.visiting[fn] { return false } - visiting[fn] = true - defer delete(visiting, fn) - for _, block := range fn.Blocks { - for _, instr := range block.Instrs { - call, ok := instr.(ssa.CallInstruction) - if !ok { - continue - } - callee := call.Common().StaticCallee() - if callee == nil && dynamicCallsMayReachRuntimeCaller { - memo[fn] = true - return true - } - if isRuntimeCallerFunc(callee) || - (callee != nil && callee.Pkg == fn.Pkg && runtimeCallerFuncSetFor(callee, memo, visiting, dynamicCallsMayReachRuntimeCaller)) { - memo[fn] = true - return true + a.visiting[fn] = true + defer delete(a.visiting, fn) + reaches := false + forEachCall(fn, func(call *ssa.CallCommon) { + if reaches { + return + } + callee := call.StaticCallee() + switch { + case isRuntimeCallerFunc(callee): + reaches = true + case callee != nil: + reaches = a.fnMayReachRuntimeCaller(callee) + case call.Method != nil: + reaches = a.interfaceInvokeMayReachRuntimeCaller(fn, call) + default: + reaches = a.functionValueCallMayReachRuntimeCaller(fn, call.Value) + } + }) + if !reaches { + for _, anon := range fn.AnonFuncs { + if a.fnMayReachRuntimeCaller(anon) { + if a.trackable[fn] { + a.trackable[anon] = true + } + reaches = true + break } } } - for _, anon := range fn.AnonFuncs { - if runtimeCallerFuncSetFor(anon, memo, visiting, dynamicCallsMayReachRuntimeCaller) { - memo[fn] = true + a.memo[fn] = reaches + return reaches +} + +func (a *runtimeCallerAnalysis) functionValueCallMayReachRuntimeCaller(fn *ssa.Function, value ssa.Value) bool { + targets, ok := a.functionValueTargets(fn, value) + if !ok { + return true + } + for target := range targets { + if a.fnMayReachRuntimeCaller(target) { + return true + } + } + return false +} + +func (a *runtimeCallerAnalysis) functionValueTargets(fn *ssa.Function, value ssa.Value) (map[*ssa.Function]bool, bool) { + if targets, ok := staticFunctionTargets(value); ok { + return targets, true + } + param, ok := value.(*ssa.Parameter) + if !ok || param.Parent() != fn { + return nil, false + } + idx, ok := parameterIndex(fn, param) + if !ok { + return nil, false + } + return a.functionParamTargets(fn, idx) +} + +func (a *runtimeCallerAnalysis) functionParamTargets(fn *ssa.Function, idx int) (map[*ssa.Function]bool, bool) { + callsites := a.callsites[fn] + if len(callsites) == 0 { + return nil, false + } + targets := make(map[*ssa.Function]bool) + for _, call := range callsites { + args := call.Args + if idx >= len(args) { + return nil, false + } + argTargets, ok := staticFunctionTargets(args[idx]) + if !ok { + return nil, false + } + for target := range argTargets { + targets[target] = true + } + } + return targets, true +} + +func staticFunctionTargets(value ssa.Value) (map[*ssa.Function]bool, bool) { + switch v := value.(type) { + case *ssa.Function: + return map[*ssa.Function]bool{v: true}, true + case *ssa.MakeClosure: + if fn, ok := v.Fn.(*ssa.Function); ok { + return map[*ssa.Function]bool{fn: true}, true + } + } + return nil, false +} + +func (a *runtimeCallerAnalysis) interfaceInvokeMayReachRuntimeCaller(fn *ssa.Function, call *ssa.CallCommon) bool { + targets, ok := a.interfaceMethodTargets(fn, call.Value, call.Method) + if !ok { + return true + } + for target := range targets { + if a.fnMayReachRuntimeCaller(target) { return true } } - memo[fn] = false return false } +func (a *runtimeCallerAnalysis) interfaceMethodTargets(fn *ssa.Function, value ssa.Value, method *types.Func) (map[*ssa.Function]bool, bool) { + if targets, ok := a.staticInterfaceMethodTargets(value, method); ok { + return targets, true + } + param, ok := value.(*ssa.Parameter) + if !ok || param.Parent() != fn { + return nil, false + } + idx, ok := parameterIndex(fn, param) + if !ok { + return nil, false + } + callsites := a.callsites[fn] + if len(callsites) == 0 { + return nil, false + } + targets := make(map[*ssa.Function]bool) + for _, call := range callsites { + args := call.Args + if idx >= len(args) { + return nil, false + } + argTargets, ok := a.staticInterfaceMethodTargets(args[idx], method) + if !ok { + return nil, false + } + for target := range argTargets { + targets[target] = true + } + } + return targets, true +} + +func (a *runtimeCallerAnalysis) staticInterfaceMethodTargets(value ssa.Value, method *types.Func) (map[*ssa.Function]bool, bool) { + switch v := value.(type) { + case *ssa.MakeInterface: + return a.methodTargetsForType(v.X.Type(), method) + case *ssa.ChangeInterface: + return a.staticInterfaceMethodTargets(v.X, method) + } + return nil, false +} + +func (a *runtimeCallerAnalysis) methodTargetsForType(typ types.Type, method *types.Func) (map[*ssa.Function]bool, bool) { + if a.pkg == nil || a.pkg.Prog == nil || method == nil { + return nil, false + } + methods := a.pkg.Prog.MethodSets.MethodSet(typ) + for i := 0; i < methods.Len(); i++ { + sel := methods.At(i) + if sel.Obj().Name() != method.Name() { + continue + } + fn := a.pkg.Prog.MethodValue(sel) + if fn == nil { + return nil, false + } + return map[*ssa.Function]bool{fn: true}, true + } + return nil, false +} + +func parameterIndex(fn *ssa.Function, param *ssa.Parameter) (int, bool) { + for i, candidate := range fn.Params { + if candidate == param { + return i, true + } + } + return 0, false +} + +func forEachCall(fn *ssa.Function, do func(*ssa.CallCommon)) { + if fn == nil { + return + } + for _, block := range fn.Blocks { + for _, instr := range block.Instrs { + if call, ok := instr.(ssa.CallInstruction); ok { + do(call.Common()) + } + } + } +} + func isRuntimeCallerFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false From c9286beb4ef5d87bac5c75bb9d8a1c73de9c6d0e Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 21:44:48 +0800 Subject: [PATCH 08/23] runtime: add compact pc-line funcinfo table --- cl/caller_frame_test.go | 95 ++++++ cl/compile.go | 16 +- cl/instr.go | 108 +++++++ internal/build/build.go | 5 +- internal/build/funcinfo/funcinfo.go | 68 ++++- internal/build/funcinfo/funcinfo_test.go | 30 ++ internal/build/funcinfo_table.go | 176 ++++++++++- internal/build/funcinfo_table_test.go | 60 ++++ internal/build/main_module.go | 3 +- runtime/internal/clite/debug/_wrap/debug.c | 7 + runtime/internal/clite/debug/debug.go | 3 + runtime/internal/lib/runtime/symtab.go | 339 ++++++++++++++++++++- ssa/funcinfo.go | 30 ++ 13 files changed, 919 insertions(+), 21 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 2a4c6eda4b..d3b53cf582 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -338,6 +338,101 @@ func f() { } } +func TestCompileRuntimeCallerPCLineMetadata(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func top() { + runtime.Caller(0) + leaf() +} + +func leaf() {} +`) + prog := newLLSSAProg(t) + prog.Target().GOOS = "linux" + prog.Target().GOARCH = "amd64" + prog.EnableFuncInfoMetadata(true) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + for _, want := range []string{ + `!llgo.pcline = !{!`, + `!"example.com/foo.top"`, + `!"caller_frame_compile.go"`, + "__llgo_pcsite_", + `.pushsection llgo_pcline`, + `.quad __llgo_pcsite_`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("missing pcline metadata %s:\n%s", want, ir) + } + } + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, "!llgo.pcline") || strings.Contains(line, `!"example.com/foo.top"`) { + if strings.Contains(line, `ptr @`) { + t.Fatalf("pcline metadata should use symbol strings, not function pointers:\n%s", line) + } + } + } +} + +func TestCompileRuntimeCallerPCLineEscapesDollarInInlineAsm(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func top() { + func() { + runtime.Caller(0) + }() +} +`) + prog := newLLSSAProg(t) + prog.Target().GOOS = "linux" + prog.Target().GOARCH = "amd64" + prog.EnableFuncInfoMetadata(true) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + if !strings.Contains(ir, `!"example.com/foo.top$1"`) { + t.Fatalf("metadata should keep the original Go symbol name:\n%s", ir) + } + if !strings.Contains(ir, `example.com/foo.top$$1`) { + t.Fatalf("inline asm should escape $ in the associated symbol:\n%s", ir) + } + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, `.pushsection llgo_pcline`) && strings.Contains(line, `example.com/foo.top$1`) && !strings.Contains(line, `example.com/foo.top$$1`) { + t.Fatalf("inline asm has an unescaped $ operand:\n%s", line) + } + } +} + +func TestCompileRuntimeCallerPCLineMetadataSkippedOnDarwin(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func top() { + runtime.Caller(0) +} +`) + prog := newLLSSAProg(t) + prog.Target().GOOS = "darwin" + prog.Target().GOARCH = "arm64" + prog.EnableFuncInfoMetadata(true) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + if strings.Contains(ir, `!llgo.pcline`) || strings.Contains(ir, "__llgo_pcsite_") { + t.Fatalf("darwin should not emit inline asm pc-site labels:\n%s", ir) + } +} + func TestCompileRuntimeCallerFrameUsesGoNameForLinkname(t *testing.T) { ssapkg, files := buildCallerFrameSSAPackage(t, "command-line-arguments", `package main import "runtime" diff --git a/cl/compile.go b/cl/compile.go index 20531efc4d..443a0cc9cb 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -177,6 +177,7 @@ type context struct { anonDefers map[*ssa.Function]bool paramDIVars map[*types.Var]llssa.DIVar runtimeCallerFuncs map[*ssa.Function]bool + pcLineSeq uint64 patches Patches blkInfos []blocks.Info @@ -549,10 +550,11 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun } noInlineDirective := hasNoInlineDirective(f) runtimeStackNoInline := needsRuntimeStackNoInline(pkgTypes, f) - if disableInline || noInlineDirective || runtimeStackNoInline { + pcLineNoInline := p.needsPCLineNoInline(f) + if disableInline || noInlineDirective || runtimeStackNoInline || pcLineNoInline { fn.Inline(llssa.NoInline) } - if noInlineDirective || runtimeStackNoInline { + if noInlineDirective || runtimeStackNoInline || pcLineNoInline { fn.DisableTailCalls() } p.funcs[f] = fn @@ -680,6 +682,16 @@ func needsRuntimeStackNoInline(pkg *types.Package, f *ssa.Function) bool { return false } +func (p *context) needsPCLineNoInline(f *ssa.Function) bool { + if p == nil || f == nil || !p.prog.FuncInfoMetadataEnabled() || !p.trackCallerFrames || !p.runtimeCallerFuncs[f] { + return false + } + if !canEmitPCLineLabelsForTarget(p.prog.Target()) { + return false + } + return p.pkg != nil && canTrackCallerFramesForPackage(p.pkg.Path()) +} + func (p *context) getFuncBodyPos(f *ssa.Function) token.Position { if f.Object() != nil { if fn, ok := f.Object().(*types.Func); ok && fn.Scope() != nil { diff --git a/cl/instr.go b/cl/instr.go index 2a87f99334..be19636a60 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -1366,6 +1366,113 @@ func (p *context) recordCallerLocationForCall(b llssa.Builder, call *ssa.CallCom p.recordPanicLocation(b, call.Pos()) } +func (p *context) emitPCLineLabel(b llssa.Builder, pos token.Pos) { + if p == nil || p.pkg == nil || p.fn == nil || !p.prog.FuncInfoMetadataEnabled() || !p.shouldTrackCallerFrames() { + return + } + target := p.prog.Target() + if !canEmitPCLineLabelsForTarget(target) { + return + } + position := p.fset.Position(pos) + if position.Line <= 0 || position.Filename == "" { + return + } + p.pcLineSeq++ + id := pcLineID(p.fn.Name(), p.pcLineSeq) + label := pcLineLabelName(id) + ptrDirective := ".quad" + align := "3" + if p.prog.PointerSize() == 4 { + ptrDirective = ".long" + align = "2" + } + b.InlineAsm( + label + ":\n" + + ".pushsection llgo_pcline,\"ao\",@progbits," + asmQuoteSymbol(p.fn.Name()) + "\n" + + ".p2align " + align + "\n" + + ptrDirective + " " + label + "\n" + + ".quad " + uint64Hex(id) + "\n" + + ".popsection", + ) + p.pkg.EmitPCLineInfo(id, p.fn.Name(), position.Filename, position.Line, position.Column) +} + +func canEmitPCLineLabelsForTarget(target *llssa.Target) bool { + if target == nil { + return false + } + if target.Target != "" || target.GOARCH == "wasm" { + return false + } + // This path uses ELF SHF_LINK_ORDER section syntax. Darwin needs a Mach-O + // live_support section path, and other object formats need separate support. + return target.GOOS == "linux" +} + +func pcLineID(symbol string, seq uint64) uint64 { + const ( + offset = uint64(14695981039346656037) + prime = uint64(1099511628211) + ) + h := offset + for i := 0; i < len(symbol); i++ { + h ^= uint64(symbol[i]) + h *= prime + } + for i := 0; i < 8; i++ { + h ^= byteOfUint64(seq, uint(i*8)) + h *= prime + } + if h == 0 { + return 1 + } + return h +} + +func byteOfUint64(v uint64, shift uint) uint64 { + return (v >> shift) & 0xff +} + +func pcLineLabelName(id uint64) string { + const hexdigits = "0123456789abcdef" + var buf [16]byte + for i := len(buf) - 1; i >= 0; i-- { + buf[i] = hexdigits[id&0xf] + id >>= 4 + } + return "__llgo_pcsite_" + string(buf[:]) +} + +func uint64Hex(v uint64) string { + const hexdigits = "0123456789abcdef" + var buf [18]byte + buf[0] = '0' + buf[1] = 'x' + for i := len(buf) - 1; i >= 2; i-- { + buf[i] = hexdigits[v&0xf] + v >>= 4 + } + return string(buf[:]) +} + +func asmQuoteSymbol(symbol string) string { + var b strings.Builder + b.Grow(len(symbol) + 2) + b.WriteByte('"') + for i := 0; i < len(symbol); i++ { + switch symbol[i] { + case '\\', '"': + b.WriteByte('\\') + case '$': + b.WriteByte('$') + } + b.WriteByte(symbol[i]) + } + b.WriteByte('"') + return b.String() +} + func (p *context) popCallerLocationFrame(b llssa.Builder) { if p.callerFrameMark.IsNil() { return @@ -1634,6 +1741,7 @@ func collectMethodNilDerefChecks(fn *ssa.Function) map[*ssa.UnOp]none { func (p *context) callEx(b llssa.Builder, act llssa.DoAction, call *ssa.CallCommon, ds *explicitDeferStack) (ret llssa.Expr) { p.recordCallerLocationForCall(b, call) + p.emitPCLineLabel(b, call.Pos()) cv := call.Value if mthd := call.Method; mthd != nil { reflectCheck := p.reflectTypeMethodCheck(call, mthd) diff --git a/internal/build/build.go b/internal/build/build.go index 93041b7406..ea58216d85 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1044,6 +1044,8 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa // Generate main module file (needed for global variables even in library modes) // This is compiled directly to .o and added to linkInputs (not cached) // Use a stable synthetic name to avoid confusing it with the real main package in traces/logs. + funcInfo := prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil) + pcLineInfo := collectPCLineInfo(linkedOrder) entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{ rtInit: needRuntime, pyInit: needPyInit, @@ -1051,7 +1053,8 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa methodByIndex: methodByIndex, methodByName: methodByName, abiSymbols: linkedModuleGlobals(linkedOrder), - funcInfo: prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil), + funcInfo: funcInfo, + pcLineInfo: pcLineInfo, }) entryObjFile, err := exportObject(ctx, "entry_main", entryPkg.ExportFile, entryPkg.LPkg) if err != nil { diff --git a/internal/build/funcinfo/funcinfo.go b/internal/build/funcinfo/funcinfo.go index c6043484c6..10dc6aab6c 100644 --- a/internal/build/funcinfo/funcinfo.go +++ b/internal/build/funcinfo/funcinfo.go @@ -31,6 +31,14 @@ type Record struct { Column uint32 } +type PCLineRecord struct { + ID uint64 + Symbol string + File string + Line uint32 + Column uint32 +} + type EncodedRecord struct { SymbolPkg uint16 SymbolName uint16 @@ -41,18 +49,43 @@ type EncodedRecord struct { Line uint32 } +type EncodedPCLineRecord struct { + ID uint64 + Func uint32 + File uint32 + Line uint32 +} + type Table struct { Records []EncodedRecord + PCLines []EncodedPCLineRecord StringOffsets []uint32 Strings []byte Hash []uint16 } func Encode(records []Record) (Table, error) { - if len(records) == 0 { + return EncodeWithPCLines(records, nil) +} + +func EncodeWithPCLines(records []Record, pcLines []PCLineRecord) (Table, error) { + funcIndex := make(map[string]uint32, len(records)) + for i, rec := range records { + if rec.Symbol != "" { + funcIndex[rec.Symbol] = uint32(i + 1) + } + } + filteredPCLines := make([]PCLineRecord, 0, len(pcLines)) + for _, rec := range pcLines { + if rec.ID == 0 || funcIndex[rec.Symbol] == 0 { + continue + } + filteredPCLines = append(filteredPCLines, rec) + } + if len(records) == 0 && len(filteredPCLines) == 0 { return Table{}, nil } - ids, offsets, strings, err := buildStringTable(collectStrings(records)) + ids, offsets, strings, err := buildStringTable(collectStrings(records, filteredPCLines)) if err != nil { return Table{}, err } @@ -75,6 +108,20 @@ func Encode(records []Record) (Table, error) { Line: rec.Line, }) } + out.PCLines = make([]EncodedPCLineRecord, 0, len(filteredPCLines)) + for _, rec := range filteredPCLines { + idx := funcIndex[rec.Symbol] + fileRoot, fileName := splitFileName(rec.File) + out.PCLines = append(out.PCLines, EncodedPCLineRecord{ + ID: rec.ID, + Func: idx, + File: packStringIDs(ids[fileRoot], ids[fileName]), + Line: rec.Line, + }) + } + sort.Slice(out.PCLines, func(i, j int) bool { + return out.PCLines[i].ID < out.PCLines[j].ID + }) out.Hash, err = buildHash(records) if err != nil { return Table{}, err @@ -82,13 +129,18 @@ func Encode(records []Record) (Table, error) { return out, nil } -func collectStrings(records []Record) []string { +func collectStrings(records []Record, pcLines []PCLineRecord) []string { seen := make(map[string]bool) for _, rec := range records { for _, s := range splitRecordStrings(rec) { seen[s] = true } } + for _, rec := range pcLines { + fileRoot, fileName := splitFileName(rec.File) + seen[fileRoot] = true + seen[fileName] = true + } delete(seen, "") out := make([]string, 0, len(seen)) for s := range seen { @@ -103,6 +155,10 @@ func collectStrings(records []Record) []string { return out } +func packStringIDs(hi, lo uint16) uint32 { + return uint32(hi)<<16 | uint32(lo) +} + func splitRecordStrings(rec Record) []string { symPkg, symName := splitQualifiedName(rec.Symbol) namePkg, nameName := splitQualifiedName(rec.Name) @@ -246,6 +302,10 @@ func (t Table) File(rec EncodedRecord) string { return t.String(rec.FileRoot) + t.String(rec.FileName) } +func (t Table) PCLineFile(rec EncodedPCLineRecord) string { + return t.String(uint16(rec.File>>16)) + t.String(uint16(rec.File)) +} + func (t Table) LookupSymbol(symbol string) (int, bool) { if len(t.Hash) == 0 { return 0, false @@ -267,7 +327,7 @@ func (t Table) LookupSymbol(symbol string) (int, bool) { } func (t Table) SizeBytes() int { - return len(t.Records)*16 + len(t.StringOffsets)*4 + len(t.Strings) + len(t.Hash)*2 + return len(t.Records)*16 + len(t.PCLines)*24 + len(t.StringOffsets)*4 + len(t.Strings) + len(t.Hash)*2 } func joinQualified(pkg, local string) string { diff --git a/internal/build/funcinfo/funcinfo_test.go b/internal/build/funcinfo/funcinfo_test.go index 238543d3e9..78a59fad2a 100644 --- a/internal/build/funcinfo/funcinfo_test.go +++ b/internal/build/funcinfo/funcinfo_test.go @@ -49,6 +49,36 @@ func TestEncodePoolsStringsAndBuildsHash(t *testing.T) { } } +func TestEncodeWithPCLines(t *testing.T) { + table, err := EncodeWithPCLines( + []Record{ + {Symbol: "example.com/p.f", Name: "example.com/p.F", File: "/src/p/f.go", Line: 10, Column: 1}, + {Symbol: "example.com/p.g", Name: "example.com/p.G", File: "/src/p/g.go", Line: 20, Column: 1}, + }, + []PCLineRecord{ + {ID: 3, Symbol: "missing", File: "missing.go", Line: 30}, + {ID: 2, Symbol: "example.com/p.g", File: "/src/p/call_g.go", Line: 22}, + {ID: 1, Symbol: "example.com/p.f", File: "/src/p/call_f.go", Line: 12}, + {ID: 0, Symbol: "example.com/p.f", File: "zero.go", Line: 1}, + }, + ) + if err != nil { + t.Fatal(err) + } + if len(table.PCLines) != 2 { + t.Fatalf("encoded pclines = %d, want 2", len(table.PCLines)) + } + if got := table.PCLines[0]; got.ID != 1 || got.Func != 1 || got.Line != 12 { + t.Fatalf("first pcline = %+v, want id 1 func 1 line 12", got) + } + if got := table.PCLines[1]; got.ID != 2 || got.Func != 2 || got.Line != 22 { + t.Fatalf("second pcline = %+v, want id 2 func 2 line 22", got) + } + if got := table.PCLineFile(table.PCLines[0]); got != "/src/p/call_f.go" { + t.Fatalf("pcline file = %q, want /src/p/call_f.go", got) + } +} + func TestEncodeRoundTripsSingleRecord(t *testing.T) { table, err := Encode([]Record{{Symbol: "s", Name: "n", File: "f", Line: 1, Column: 2}}) if err != nil { diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 37402229dd..83b22d8bd1 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -32,7 +32,14 @@ const ( funcInfoStringOffsetsSymbol = "__llgo_funcinfo_string_offsets" funcInfoHashSymbol = "__llgo_funcinfo_hash" funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" + pcLineTableSymbol = "__llgo_pcline_table" + pcLineCountSymbol = "__llgo_pcline_count" + pcSiteStartPtrSymbol = "__llgo_pcsite_start" + pcSiteEndPtrSymbol = "__llgo_pcsite_end" + pcSiteStartSymbol = "__start_llgo_pcline" + pcSiteEndSymbol = "__stop_llgo_pcline" funcInfoDataSymbol = "__llgo_funcinfo_table$data" + pcLineDataSymbol = "__llgo_pcline_table$data" funcInfoStringsDataSymbol = "__llgo_funcinfo_strings$data" funcInfoStringOffsetsDataSymbol = "__llgo_funcinfo_string_offsets$data" funcInfoHashDataSymbol = "__llgo_funcinfo_hash$data" @@ -46,6 +53,14 @@ type funcInfoRecord struct { column uint32 } +type pcLineRecord struct { + id uint64 + symbol string + file string + line uint32 + column uint32 +} + func collectFuncInfo(pkgs []Package) []funcInfoRecord { seen := make(map[string]funcInfoRecord) for _, pkg := range pkgs { @@ -74,6 +89,36 @@ func collectFuncInfo(pkgs []Package) []funcInfoRecord { return out } +func collectPCLineInfo(pkgs []Package) []pcLineRecord { + var out []pcLineRecord + seen := make(map[uint64]none) + for _, pkg := range pkgs { + if pkg == nil || pkg.LPkg == nil { + continue + } + for _, rec := range readPCLineInfo(pkg.LPkg.Module()) { + if rec.id == 0 || rec.symbol == "" { + continue + } + if _, ok := seen[rec.id]; ok { + continue + } + seen[rec.id] = none{} + out = append(out, rec) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].symbol != out[j].symbol { + return out[i].symbol < out[j].symbol + } + if out[i].line != out[j].line { + return out[i].line < out[j].line + } + return out[i].id < out[j].id + }) + return out +} + func prepareFuncInfoTableRecords(records []funcInfoRecord, liveSymbols map[string]none) []funcInfoRecord { if len(records) == 0 { return nil @@ -123,12 +168,38 @@ func readFuncInfo(mod llvm.Module) []funcInfoRecord { return out } -func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord) { +func readPCLineInfo(mod llvm.Module) []pcLineRecord { + rows := mod.NamedMetadataOperands(llssa.PCLineMetadataName) + if len(rows) == 0 { + return nil + } + out := make([]pcLineRecord, 0, len(rows)) + for _, row := range rows { + fields := row.MDNodeOperands() + if len(fields) != 6 || fields[0].ZExtValue() != 1 { + continue + } + if !fields[2].IsAMDString() || !fields[3].IsAMDString() { + continue + } + out = append(out, pcLineRecord{ + id: fields[1].ZExtValue(), + symbol: fields[2].MDString(), + file: fields[3].MDString(), + line: uint32(fields[4].ZExtValue()), + column: uint32(fields[5].ZExtValue()), + }) + } + return out +} + +func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord, pcLines []pcLineRecord) { mod := pkg.Module() llvmCtx := mod.Context() i8Type := llvmCtx.Int8Type() i16Type := llvmCtx.Int16Type() i32Type := llvmCtx.Int32Type() + i64Type := llvmCtx.Int64Type() countType := llvmCtx.IntType(ctx.prog.PointerSize() * 8) recordType := llvmCtx.StructType([]llvm.Type{ i16Type, @@ -139,27 +210,58 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord i16Type, i32Type, }, false) + pcLineRecordType := llvmCtx.StructType([]llvm.Type{ + i64Type, + i32Type, + i32Type, + i32Type, + }, false) + pcSiteRecordType := llvmCtx.StructType([]llvm.Type{ + llvm.PointerType(i8Type, 0), + i64Type, + }, false) tablePtr := llvm.AddGlobal(mod, llvm.PointerType(recordType, 0), funcInfoTableSymbol) + pcLinePtr := llvm.AddGlobal(mod, llvm.PointerType(pcLineRecordType, 0), pcLineTableSymbol) + pcSiteStartPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteStartPtrSymbol) + pcSiteEndPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteEndPtrSymbol) stringsPtr := llvm.AddGlobal(mod, llvm.PointerType(i8Type, 0), funcInfoStringsSymbol) stringOffsetsPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoStringOffsetsSymbol) hashPtr := llvm.AddGlobal(mod, llvm.PointerType(i16Type, 0), funcInfoHashSymbol) count := llvm.AddGlobal(mod, countType, funcInfoCountSymbol) + pcLineCount := llvm.AddGlobal(mod, countType, pcLineCountSymbol) hashMask := llvm.AddGlobal(mod, countType, funcInfoHashMaskSymbol) - if len(records) == 0 { + if len(records) == 0 && len(pcLines) == 0 { tablePtr.SetInitializer(llvm.ConstPointerNull(tablePtr.GlobalValueType())) + pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) + pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) + pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) count.SetInitializer(llvm.ConstInt(countType, 0, false)) + pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) return } - encoded, err := buildfuncinfo.Encode(toFuncInfoRecords(records)) + encoded, err := buildfuncinfo.EncodeWithPCLines(toFuncInfoRecords(records), toPCLineRecords(pcLines)) if err != nil { panic(err) } + if len(encoded.Records) == 0 && len(encoded.PCLines) == 0 { + tablePtr.SetInitializer(llvm.ConstPointerNull(tablePtr.GlobalValueType())) + pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) + pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) + pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) + stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) + hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) + count.SetInitializer(llvm.ConstInt(countType, 0, false)) + pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) + hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) + return + } values := make([]llvm.Value, 0, len(encoded.Records)) for _, rec := range encoded.Records { @@ -181,6 +283,45 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord data.SetUnnamedAddr(true) data.SetAlignment(4) + pcLineValues := make([]llvm.Value, 0, len(encoded.PCLines)) + for _, rec := range encoded.PCLines { + pcLineValues = append(pcLineValues, llvm.ConstNamedStruct(pcLineRecordType, []llvm.Value{ + llvm.ConstInt(i64Type, rec.ID, false), + llvm.ConstInt(i32Type, uint64(rec.Func), false), + llvm.ConstInt(i32Type, uint64(rec.File), false), + llvm.ConstInt(i32Type, uint64(rec.Line), false), + })) + } + if len(pcLineValues) == 0 { + pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) + pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) + pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) + pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + } else { + pcLineArrayType := llvm.ArrayType(pcLineRecordType, len(pcLineValues)) + pcLineData := llvm.AddGlobal(mod, pcLineArrayType, pcLineDataSymbol) + pcLineData.SetInitializer(llvm.ConstArray(pcLineRecordType, pcLineValues)) + pcLineData.SetLinkage(llvm.PrivateLinkage) + pcLineData.SetGlobalConstant(true) + pcLineData.SetUnnamedAddr(true) + pcLineData.SetAlignment(8) + pcLinePtr.SetInitializer(llvm.ConstInBoundsGEP(pcLineArrayType, pcLineData, []llvm.Value{ + llvm.ConstInt(countType, 0, false), + llvm.ConstInt(countType, 0, false), + })) + pcLineCount.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.PCLines)), false)) + if ctx.buildConf.Goos == "linux" && ctx.buildConf.Target == "" { + emitPCSiteSentinel(mod, ctx.prog.PointerSize()) + pcSiteStart := llvm.AddGlobal(mod, pcSiteRecordType, pcSiteStartSymbol) + pcSiteEnd := llvm.AddGlobal(mod, pcSiteRecordType, pcSiteEndSymbol) + pcSiteStartPtr.SetInitializer(pcSiteStart) + pcSiteEndPtr.SetInitializer(pcSiteEnd) + } else { + pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) + pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + } + } + stringArrayType := llvm.ArrayType(i8Type, len(encoded.Strings)) stringData := llvm.AddGlobal(mod, stringArrayType, funcInfoStringsDataSymbol) stringData.SetInitializer(llvmCtx.ConstString(string(encoded.Strings), false)) @@ -237,6 +378,21 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord count.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Records)), false)) } +func emitPCSiteSentinel(mod llvm.Module, pointerSize int) { + ptrDirective := ".quad" + align := "3" + if pointerSize == 4 { + ptrDirective = ".long" + align = "2" + } + mod.SetInlineAsm( + ".section llgo_pcline,\"aR\",@progbits\n" + + ".p2align " + align + "\n" + + ptrDirective + " 0\n" + + ".quad 0\n", + ) +} + func toFuncInfoRecords(records []funcInfoRecord) []buildfuncinfo.Record { out := make([]buildfuncinfo.Record, len(records)) for i, rec := range records { @@ -250,3 +406,17 @@ func toFuncInfoRecords(records []funcInfoRecord) []buildfuncinfo.Record { } return out } + +func toPCLineRecords(records []pcLineRecord) []buildfuncinfo.PCLineRecord { + out := make([]buildfuncinfo.PCLineRecord, len(records)) + for i, rec := range records { + out[i] = buildfuncinfo.PCLineRecord{ + ID: rec.id, + Symbol: rec.symbol, + File: rec.file, + Line: rec.line, + Column: rec.column, + } + } + return out +} diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index 649a20a8bc..6371a0ec4e 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -55,10 +55,14 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) ir := entry.LPkg.String() for _, want := range []string{ "@__llgo_funcinfo_table = global ptr", + "@__llgo_pcline_table = global ptr null", + "@__llgo_pcsite_start = global ptr null", + "@__llgo_pcsite_end = global ptr null", "@__llgo_funcinfo_strings = global ptr", "@__llgo_funcinfo_string_offsets = global ptr", "@__llgo_funcinfo_hash = global ptr", "@__llgo_funcinfo_count = global i64 1", + "@__llgo_pcline_count = global i64 0", "@__llgo_funcinfo_hash_mask = global i64 1", `@"__llgo_funcinfo_table$data" = private unnamed_addr constant [1 x { i16, i16, i16, i16, i16, i16, i32 }]`, `@"__llgo_funcinfo_string_offsets$data" = private unnamed_addr constant`, @@ -78,6 +82,58 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) } } +func TestFuncInfoTableMaterializesPCLineMetadata(t *testing.T) { + prog := llssa.NewProgram(nil) + src := prog.NewPackage("example.com/p", "example.com/p") + src.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "live.go", 17, 3) + src.EmitPCLineInfo(0x1234, "example.com/p.live", "call.go", 23, 5) + src.EmitPCLineInfo(0x5678, "example.com/p.missing", "missing.go", 99, 1) + + records := collectFuncInfo([]Package{{LPkg: src}}) + pcLines := collectPCLineInfo([]Package{{LPkg: src}}) + if len(records) != 1 { + t.Fatalf("collectFuncInfo returned %d records, want 1", len(records)) + } + if len(pcLines) != 2 { + t.Fatalf("collectPCLineInfo returned %d records, want 2", len(pcLines)) + } + + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records, pcLineInfo: pcLines}) + ir := entry.LPkg.String() + for _, want := range []string{ + "@__llgo_pcline_table = global ptr", + "@__llgo_pcsite_start = global ptr @__start_llgo_pcline", + "@__llgo_pcsite_end = global ptr @__stop_llgo_pcline", + "@__llgo_pcline_count = global i64 1", + "module asm \".section llgo_pcline", + `@"__llgo_pcline_table$data" = private unnamed_addr constant [1 x { i64, i32, i32, i32 }]`, + "i64 4660", + "i32 23", + `call.go\00`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("pcline table IR missing %q:\n%s", want, ir) + } + } + if strings.Contains(ir, "missing.go") || strings.Contains(ir, "i64 22136") { + t.Fatalf("pcline table should drop records without matching function metadata:\n%s", ir) + } + if strings.Contains(ir, `ptr @"example.com/p.live"`) { + t.Fatalf("pcline table must not reference function pointers:\n%s", ir) + } +} + func TestPrepareFuncInfoTableRecordsFiltersLiveSymbols(t *testing.T) { records := []funcInfoRecord{ {symbol: "dead", name: "dead"}, @@ -138,10 +194,14 @@ func TestFuncInfoTableEmptyDefinitions(t *testing.T) { ir := entry.LPkg.String() for _, want := range []string{ "@__llgo_funcinfo_table = global ptr null", + "@__llgo_pcline_table = global ptr null", + "@__llgo_pcsite_start = global ptr null", + "@__llgo_pcsite_end = global ptr null", "@__llgo_funcinfo_strings = global ptr null", "@__llgo_funcinfo_string_offsets = global ptr null", "@__llgo_funcinfo_hash = global ptr null", "@__llgo_funcinfo_count = global i64 0", + "@__llgo_pcline_count = global i64 0", "@__llgo_funcinfo_hash_mask = global i64 0", } { if !strings.Contains(ir, want) { diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 9f68a976ac..83289dd23f 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -44,6 +44,7 @@ type genConfig struct { methodByName map[string]none abiSymbols map[string]none funcInfo []funcInfoRecord + pcLineInfo []pcLineRecord } // genMainModule generates the main entry module for an llgo program. @@ -61,7 +62,7 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g argvValueType := prog.Pointer(prog.CStr()) argvVar := mainPkg.NewVarEx("__llgo_argv", prog.Pointer(argvValueType)) argvVar.InitNil() - emitFuncInfoTable(ctx, mainPkg, cfg.funcInfo) + emitFuncInfoTable(ctx, mainPkg, cfg.funcInfo, cfg.pcLineInfo) exportFile := pkg.ExportFile if exportFile == "" { diff --git a/runtime/internal/clite/debug/_wrap/debug.c b/runtime/internal/clite/debug/_wrap/debug.c index cf050c8848..a03fb3ca1c 100644 --- a/runtime/internal/clite/debug/_wrap/debug.c +++ b/runtime/internal/clite/debug/_wrap/debug.c @@ -21,6 +21,13 @@ int llgo_addrinfo(void *addr, Dl_info *info) { return ret; } +void *llgo_symbol(char *name) { + int saved_errno = errno; + void *ret = dlsym(RTLD_DEFAULT, name); + errno = saved_errno; + return ret; +} + void llgo_stacktrace(int skip, void *ctx, int (*fn)(void *ctx, void *pc, void *offset, void *sp, char *name)) { int saved_errno = errno; unw_cursor_t cursor; diff --git a/runtime/internal/clite/debug/debug.go b/runtime/internal/clite/debug/debug.go index d35899cd99..d58a5f1941 100644 --- a/runtime/internal/clite/debug/debug.go +++ b/runtime/internal/clite/debug/debug.go @@ -25,6 +25,9 @@ func Address() unsafe.Pointer //go:linkname Addrinfo C.llgo_addrinfo func Addrinfo(addr unsafe.Pointer, info *Info) c.Int +//go:linkname Symbol C.llgo_symbol +func Symbol(name *c.Char) unsafe.Pointer + //go:linkname stacktrace C.llgo_stacktrace func stacktrace(skip c.Int, ctx unsafe.Pointer, fn func(ctx, pc, offset, sp unsafe.Pointer, name *c.Char) c.Int) diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 919cd74ed3..543d395cd7 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -127,7 +127,7 @@ const frameSymbolCacheSize = 128 var frameSymbolCache [frameSymbolCacheSize]frameSymbolCacheEntry func recordFrameSymbol(pc, offset uintptr, name string) { - if pc == 0 || name == "" { + if pc == 0 || name == "" || isPCSiteSymbol(name) { return } i := (pc >> 4) & (frameSymbolCacheSize - 1) @@ -162,6 +162,42 @@ var runtimeFuncInfoCount uintptr //go:linkname runtimeFuncInfoHashMask __llgo_funcinfo_hash_mask var runtimeFuncInfoHashMask uintptr +type runtimePCLineRecord struct { + id uint64 + funcIndex uint32 + file uint32 + line uint32 +} + +//go:linkname runtimePCLineTable __llgo_pcline_table +var runtimePCLineTable *runtimePCLineRecord + +//go:linkname runtimePCLineCount __llgo_pcline_count +var runtimePCLineCount uintptr + +type runtimePCSiteRecord struct { + pc uintptr + id uint64 +} + +//go:linkname runtimePCSiteStart __llgo_pcsite_start +var runtimePCSiteStart *runtimePCSiteRecord + +//go:linkname runtimePCSiteEnd __llgo_pcsite_end +var runtimePCSiteEnd *runtimePCSiteRecord + +type runtimePCLineFrame struct { + pc uintptr + entry uintptr + function string + file string + line int + startLine int +} + +var runtimePCLineInit bool +var runtimePCLineFrames []runtimePCLineFrame + func hasStringPrefix(s, prefix string) bool { if len(s) < len(prefix) { return false @@ -174,6 +210,15 @@ func hasStringPrefix(s, prefix string) bool { return true } +func isPCSiteSymbol(name string) bool { + for i := 0; i < len(name) && name[i] == '_'; i++ { + if hasStringPrefix(name[i:], "__llgo_pcsite_") { + return true + } + } + return false +} + func publicFunctionName(name string) string { const commandLineArguments = "command-line-arguments." if hasStringPrefix(name, commandLineArguments) { @@ -252,6 +297,11 @@ func funcInfoAt(i uintptr) *runtimeFuncInfoRecord { return (*runtimeFuncInfoRecord)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoTable), i*size)) } +func pcLineAt(i uintptr) *runtimePCLineRecord { + size := unsafe.Sizeof(*runtimePCLineTable) + return (*runtimePCLineRecord)(unsafe.Add(unsafe.Pointer(runtimePCLineTable), i*size)) +} + func funcInfoHashString(s string) uintptr { const ( offset = uint32(2166136261) @@ -317,10 +367,17 @@ func funcInfoJoinFile(rootID, nameID uint16) string { return string(buf) } +func funcInfoPackedFile(file uint32) string { + return funcInfoJoinFile(uint16(file>>16), uint16(file)) +} + func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { if symbol == "" || runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 { return nil } + if runtimeFuncInfoStrings == nil || runtimeFuncInfoStringOffsets == nil || runtimeFuncInfoCount > 1<<20 || runtimeFuncInfoHashMask > 1<<22 { + return nil + } if runtimeFuncInfoHash != nil && runtimeFuncInfoHashMask != 0 { slot := funcInfoHashString(symbol) & runtimeFuncInfoHashMask for probes := uintptr(0); probes <= runtimeFuncInfoHashMask; probes++ { @@ -382,6 +439,9 @@ func cachedFrameSymbol(pc uintptr) pcSymbol { return pcSymbol{pc: pc} } rawFn := entry.name + if isPCSiteSymbol(rawFn) { + return pcSymbol{pc: pc} + } fn := publicFunctionName(rawFn) sym := pcSymbol{ pc: pc, @@ -399,6 +459,9 @@ func addrInfoSymbol(pc uintptr) pcSymbol { return cachedFrameSymbol(pc) } rawFn := safeGoString(info.Sname, "") + if isPCSiteSymbol(rawFn) { + return pcSymbol{pc: pc} + } if rawFn == "" { if sym := cachedFrameSymbol(pc); sym.ok { return sym @@ -415,28 +478,284 @@ func addrInfoSymbol(pc uintptr) pcSymbol { return sym } -func frameSymbol(pc uintptr) pcSymbol { - if frame, ok := rtdebug.FrameForPC(pc); ok { - return pcSymbol{ +func initRuntimePCLineFrames() { + if runtimePCLineInit { + return + } + runtimePCLineInit = true + if runtimePCLineTable == nil || + runtimePCLineCount == 0 || + runtimePCSiteStart == nil || + runtimePCSiteEnd == nil || + runtimeFuncInfoTable == nil || + runtimeFuncInfoCount == 0 || + runtimeFuncInfoStrings == nil || + runtimeFuncInfoStringOffsets == nil { + return + } + if runtimePCLineCount > 1<<20 || runtimePCLineCount > runtimeFuncInfoCount*1024 { + return + } + start := uintptr(unsafe.Pointer(runtimePCSiteStart)) + end := uintptr(unsafe.Pointer(runtimePCSiteEnd)) + size := unsafe.Sizeof(*runtimePCSiteStart) + if end <= start || size == 0 || (end-start)%size != 0 { + return + } + nsite := (end - start) / size + if nsite > runtimePCLineCount*1024 || nsite > 1<<22 { + return + } + frames := make([]runtimePCLineFrame, 0, nsite) + for i := uintptr(0); i < nsite; i++ { + site := (*runtimePCSiteRecord)(unsafe.Pointer(start + i*size)) + if site == nil || site.id == 0 || site.pc == 0 { + continue + } + rec := pcLineInfoForID(site.id) + if rec == nil || rec.funcIndex == 0 || uintptr(rec.funcIndex) > runtimeFuncInfoCount { + continue + } + pc := site.pc + fn := funcInfoAt(uintptr(rec.funcIndex) - 1) + entry := symbolPC(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + if entry == 0 { + sym := addrInfoSymbol(pc) + entry = sym.entry + } + file := funcInfoPackedFile(rec.file) + if file == "" { + file = funcInfoJoinFile(fn.fileRoot, fn.fileName) + } + line := int(rec.line) + if line == 0 { + line = int(fn.line) + } + function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) + if function == "" { + function = publicFunctionName(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + } + frames = append(frames, runtimePCLineFrame{ pc: pc, - entry: frame.Entry, - function: frame.Function, - file: frame.File, - line: frame.Line, - startLine: frame.StartLine, - ok: true, + entry: entry, + function: function, + file: file, + line: line, + startLine: int(fn.line), + }) + } + sortRuntimePCLineFrames(frames) + runtimePCLineFrames = uniqueRuntimePCLineFrames(frames) +} + +func pcLineInfoForID(id uint64) *runtimePCLineRecord { + lo, hi := uintptr(0), runtimePCLineCount + for lo < hi { + mid := (lo + hi) >> 1 + rec := pcLineAt(mid) + if rec.id >= id { + hi = mid + } else { + lo = mid + 1 + } + } + if lo >= runtimePCLineCount { + return nil + } + rec := pcLineAt(lo) + if rec.id != id { + return nil + } + return rec +} + +func symbolPC(symbol string) uintptr { + if symbol == "" { + return 0 + } + buf := make([]byte, len(symbol)+1) + copy(buf, symbol) + return uintptr(clitedebug.Symbol((*c.Char)(unsafe.Pointer(&buf[0])))) +} + +func sortRuntimePCLineFrames(frames []runtimePCLineFrame) { + if len(frames) < 2 { + return + } + quickSortRuntimePCLineFrames(frames, 0, len(frames)-1) +} + +func quickSortRuntimePCLineFrames(frames []runtimePCLineFrame, lo, hi int) { + for hi-lo > 16 { + mid := int(uint(lo+hi) >> 1) + if frames[mid].pc < frames[lo].pc { + frames[mid], frames[lo] = frames[lo], frames[mid] + } + if frames[hi].pc < frames[mid].pc { + frames[hi], frames[mid] = frames[mid], frames[hi] + } + if frames[mid].pc < frames[lo].pc { + frames[mid], frames[lo] = frames[lo], frames[mid] + } + pivot := frames[mid].pc + i, j := lo, hi + for { + for frames[i].pc < pivot { + i++ + } + for frames[j].pc > pivot { + j-- + } + if i >= j { + break + } + frames[i], frames[j] = frames[j], frames[i] + i++ + j-- + } + if j-lo < hi-i { + quickSortRuntimePCLineFrames(frames, lo, j) + lo = i + } else { + quickSortRuntimePCLineFrames(frames, i, hi) + hi = j + } + } + for i := lo + 1; i <= hi; i++ { + x := frames[i] + j := i - 1 + for j >= lo && frames[j].pc > x.pc { + frames[j+1] = frames[j] + j-- + } + frames[j+1] = x + } +} + +func uniqueRuntimePCLineFrames(frames []runtimePCLineFrame) []runtimePCLineFrame { + if len(frames) < 2 { + return frames + } + out := frames[:1] + for i := 1; i < len(frames); i++ { + if frames[i].pc == out[len(out)-1].pc { + out[len(out)-1] = frames[i] + continue + } + out = append(out, frames[i]) + } + return out +} + +func pcLineFrameForPC(pc, entry uintptr) (pcSymbol, bool) { + if pc == 0 { + return pcSymbol{}, false + } + initRuntimePCLineFrames() + frames := runtimePCLineFrames + if len(frames) == 0 { + return pcSymbol{}, false + } + lo, hi := 0, len(frames) + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if frames[mid].pc > pc { + hi = mid + } else { + lo = mid + 1 + } + } + if lo == 0 { + return pcSymbol{}, false + } + frame := frames[lo-1] + if entry != 0 && frame.entry != 0 && frame.entry != entry { + return pcSymbol{}, false + } + return pcSymbol{ + pc: pc, + entry: frame.entry, + function: frame.function, + file: frame.file, + line: frame.line, + startLine: frame.startLine, + ok: true, + }, true +} + +func mergePCLineSymbol(base, line pcSymbol) pcSymbol { + if line.entry == 0 { + line.entry = base.entry + } + if line.function == "" { + line.function = base.function + } + if line.file == "" { + line.file = base.file + } + if line.line == 0 { + line.line = base.line + } + if line.startLine == 0 { + line.startLine = base.startLine + } + line.ok = true + return line +} + +func frameSymbol(pc uintptr) pcSymbol { + if pc&3 != 0 { + if frame, ok := rtdebug.FrameForPC(pc); ok { + return pcSymbol{ + pc: pc, + entry: frame.Entry, + function: frame.Function, + file: frame.File, + line: frame.Line, + startLine: frame.StartLine, + ok: true, + } } } sym := addrInfoSymbol(pc) if pc == 0 { + if frame, ok := rtdebug.FrameForPC(pc); ok { + return pcSymbol{ + pc: pc, + entry: frame.Entry, + function: frame.Function, + file: frame.File, + line: frame.Line, + startLine: frame.StartLine, + ok: true, + } + } return sym } + if lineSym, ok := pcLineFrameForPC(pc, sym.entry); ok { + return mergePCLineSymbol(sym, lineSym) + } if sym.entry == 0 || pc > sym.entry { if callSym := addrInfoSymbol(pc - 1); callSym.ok { + if lineSym, ok := pcLineFrameForPC(pc-1, callSym.entry); ok { + lineSym.pc = pc + return mergePCLineSymbol(callSym, lineSym) + } callSym.pc = pc return callSym } } + if frame, ok := rtdebug.FrameForPC(pc); ok { + return pcSymbol{ + pc: pc, + entry: frame.Entry, + function: frame.Function, + file: frame.File, + line: frame.Line, + startLine: frame.StartLine, + ok: true, + } + } return sym } diff --git a/ssa/funcinfo.go b/ssa/funcinfo.go index 734399093d..4dbc8f08e1 100644 --- a/ssa/funcinfo.go +++ b/ssa/funcinfo.go @@ -20,7 +20,9 @@ import "github.com/xgo-dev/llvm" const ( FuncInfoMetadataName = "llgo.funcinfo" + PCLineMetadataName = "llgo.pcline" funcInfoVersion = 1 + pcLineVersion = 1 ) // EnableFuncInfoMetadata controls emission of DCE-safe function source @@ -61,3 +63,31 @@ func (p Package) EmitFuncInfo(symbol, name, file string, line, column int) { }), ) } + +// EmitPCLineInfo records a PC label id and its source position. The id names a +// zero-byte label emitted in the function body; keeping the metadata string-only +// lets dead functions be removed without the line table holding address +// references to them. +func (p Package) EmitPCLineInfo(id uint64, symbol, file string, line, column int) { + if id == 0 || symbol == "" { + return + } + if line < 0 { + line = 0 + } + if column < 0 { + column = 0 + } + i32 := p.Prog.Int32().ll + i64 := p.Prog.Int64().ll + p.mod.AddNamedMetadataOperand(PCLineMetadataName, + p.Prog.ctx.MDNode([]llvm.Metadata{ + llvm.ConstInt(i32, pcLineVersion, false).ConstantAsMetadata(), + llvm.ConstInt(i64, id, false).ConstantAsMetadata(), + p.Prog.ctx.MDString(symbol), + p.Prog.ctx.MDString(file), + llvm.ConstInt(i32, uint64(line), false).ConstantAsMetadata(), + llvm.ConstInt(i32, uint64(column), false).ConstantAsMetadata(), + }), + ) +} From 0a7b3f4e09cfb58c20e2414a9407f1061037f485 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 22:46:23 +0800 Subject: [PATCH 09/23] runtime: optimize FuncForPC metadata lookup --- internal/build/funcinfo_table.go | 31 +- .../lib/runtime/pprof_runtime_stub_llgo.go | 37 +-- runtime/internal/lib/runtime/symtab.go | 284 +++++++++++++++++- 3 files changed, 318 insertions(+), 34 deletions(-) diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 83b22d8bd1..680fe604e3 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -18,6 +18,7 @@ package build import ( "sort" + "strings" "github.com/xgo-dev/llvm" @@ -310,8 +311,7 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord llvm.ConstInt(countType, 0, false), })) pcLineCount.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.PCLines)), false)) - if ctx.buildConf.Goos == "linux" && ctx.buildConf.Target == "" { - emitPCSiteSentinel(mod, ctx.prog.PointerSize()) + if shouldEmitRuntimeELFSites(ctx) { pcSiteStart := llvm.AddGlobal(mod, pcSiteRecordType, pcSiteStartSymbol) pcSiteEnd := llvm.AddGlobal(mod, pcSiteRecordType, pcSiteEndSymbol) pcSiteStartPtr.SetInitializer(pcSiteStart) @@ -321,6 +321,7 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) } } + emitRuntimeFuncInfoSentinels(mod, ctx.prog.PointerSize(), shouldEmitRuntimeELFSites(ctx) && len(pcLineValues) != 0) stringArrayType := llvm.ArrayType(i8Type, len(encoded.Strings)) stringData := llvm.AddGlobal(mod, stringArrayType, funcInfoStringsDataSymbol) @@ -378,19 +379,31 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord count.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Records)), false)) } -func emitPCSiteSentinel(mod llvm.Module, pointerSize int) { +func shouldEmitRuntimeELFSites(ctx *context) bool { + return ctx != nil && + ctx.buildConf != nil && + ctx.buildConf.Goos == "linux" && + ctx.buildConf.Target == "" +} + +func emitRuntimeFuncInfoSentinels(mod llvm.Module, pointerSize int, pcSite bool) { + if !pcSite { + return + } ptrDirective := ".quad" align := "3" if pointerSize == 4 { ptrDirective = ".long" align = "2" } - mod.SetInlineAsm( - ".section llgo_pcline,\"aR\",@progbits\n" + - ".p2align " + align + "\n" + - ptrDirective + " 0\n" + - ".quad 0\n", - ) + var asm strings.Builder + if pcSite { + asm.WriteString(".section llgo_pcline,\"aR\",@progbits\n") + asm.WriteString(".p2align " + align + "\n") + asm.WriteString(ptrDirective + " 0\n") + asm.WriteString(".quad 0\n") + } + mod.SetInlineAsm(asm.String()) } func toFuncInfoRecords(records []funcInfoRecord) []buildfuncinfo.Record { diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 8fac1ada4b..aa69889194 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -3,7 +3,6 @@ package runtime import ( - "github.com/goplus/llgo/runtime/internal/clite/tls" llrt "github.com/goplus/llgo/runtime/internal/runtime" ) @@ -87,23 +86,24 @@ func NumGoroutine() int { func SetCPUProfileRate(hz int) {} -const funcForPCCacheSize = 256 +const funcForPCCacheSize = 1024 type funcForPCCacheEntry struct { pc uintptr fn *Func } -type funcForPCCache struct { - entries [funcForPCCacheSize]funcForPCCacheEntry -} - -var funcForPCCacheTLS = tls.Alloc[*funcForPCCache](nil) +var funcForPCCache [funcForPCCacheSize]funcForPCCacheEntry func FuncForPC(pc uintptr) *Func { if fn := cachedFuncForPC(pc); fn != nil { return fn } + if sym, ok := funcPCFrameForPC(pc); ok { + fn := newFuncForPC(pc, sym) + cacheFuncForPC(pc, fn) + return fn + } sym := frameSymbol(pc) fn := newFuncForPC(pc, sym) cacheFuncForPC(pc, fn) @@ -132,27 +132,18 @@ func newFuncForPC(pc uintptr, sym pcSymbol) *Func { } func cachedFuncForPC(pc uintptr) *Func { - cache := funcForPCCacheTLS.Get() - if cache == nil { - return nil - } - entry := &cache.entries[funcForPCCacheIndex(pc)] - if entry.pc == pc && entry.fn != nil { - return entry.fn + entry := &funcForPCCache[funcForPCCacheIndex(pc)] + fn := entry.fn + if fn != nil && entry.pc == pc && fn.pc == pc { + return fn } return nil } func cacheFuncForPC(pc uintptr, fn *Func) { - cache := funcForPCCacheTLS.Get() - if cache == nil { - cache = new(funcForPCCache) - funcForPCCacheTLS.Set(cache) - } - cache.entries[funcForPCCacheIndex(pc)] = funcForPCCacheEntry{ - pc: pc, - fn: fn, - } + entry := &funcForPCCache[funcForPCCacheIndex(pc)] + entry.fn = fn + entry.pc = pc } func funcForPCCacheIndex(pc uintptr) uintptr { diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 543d395cd7..ce9ef6dc7f 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -198,6 +198,26 @@ type runtimePCLineFrame struct { var runtimePCLineInit bool var runtimePCLineFrames []runtimePCLineFrame +type runtimeFuncPCFrame struct { + entry uintptr + funcIndex uint32 + function string + file string + startLine int +} + +type runtimePCPageIndex struct { + base uintptr + pages []uint32 +} + +const runtimeFuncPCPageShift = 12 + +var runtimeFuncPCInit bool +var runtimeFuncPCFrames []runtimeFuncPCFrame +var runtimeFuncPCEntries []uintptr +var runtimeFuncPCIndex runtimePCPageIndex + func hasStringPrefix(s, prefix string) bool { if len(s) < len(prefix) { return false @@ -478,6 +498,217 @@ func addrInfoSymbol(pc uintptr) pcSymbol { return sym } +func initRuntimeFuncPCFrames() { + if runtimeFuncPCInit { + return + } + runtimeFuncPCInit = true + if runtimeFuncInfoTable == nil || + runtimeFuncInfoCount == 0 || + runtimeFuncInfoStrings == nil || + runtimeFuncInfoStringOffsets == nil { + return + } + if runtimeFuncInfoCount > 1<<20 { + return + } + frames := make([]runtimeFuncPCFrame, 0, runtimeFuncInfoCount) + entries := make([]uintptr, runtimeFuncInfoCount+1) + for i := uintptr(0); i < runtimeFuncInfoCount; i++ { + fn := funcInfoAt(i) + pc := symbolPC(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + if pc == 0 { + continue + } + index := uint32(i + 1) + function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) + if function == "" { + function = publicFunctionName(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + } + file := funcInfoJoinFile(fn.fileRoot, fn.fileName) + frames = append(frames, runtimeFuncPCFrame{ + entry: pc, + funcIndex: index, + function: function, + file: file, + startLine: int(fn.line), + }) + if entries[index] == 0 || pc < entries[index] { + entries[index] = pc + } + } + sortRuntimeFuncPCFrames(frames) + frames = uniqueRuntimeFuncPCFrames(frames) + runtimeFuncPCFrames = frames + runtimeFuncPCEntries = entries + runtimeFuncPCIndex = buildRuntimeFuncPCIndex(frames) +} + +func sortRuntimeFuncPCFrames(frames []runtimeFuncPCFrame) { + if len(frames) < 2 { + return + } + quickSortRuntimeFuncPCFrames(frames, 0, len(frames)-1) +} + +func quickSortRuntimeFuncPCFrames(frames []runtimeFuncPCFrame, lo, hi int) { + for hi-lo > 16 { + mid := int(uint(lo+hi) >> 1) + if frames[mid].entry < frames[lo].entry { + frames[mid], frames[lo] = frames[lo], frames[mid] + } + if frames[hi].entry < frames[mid].entry { + frames[hi], frames[mid] = frames[mid], frames[hi] + } + if frames[mid].entry < frames[lo].entry { + frames[mid], frames[lo] = frames[lo], frames[mid] + } + pivot := frames[mid].entry + i, j := lo, hi + for { + for frames[i].entry < pivot { + i++ + } + for frames[j].entry > pivot { + j-- + } + if i >= j { + break + } + frames[i], frames[j] = frames[j], frames[i] + i++ + j-- + } + if j-lo < hi-i { + quickSortRuntimeFuncPCFrames(frames, lo, j) + lo = i + } else { + quickSortRuntimeFuncPCFrames(frames, i, hi) + hi = j + } + } + for i := lo + 1; i <= hi; i++ { + x := frames[i] + j := i - 1 + for j >= lo && frames[j].entry > x.entry { + frames[j+1] = frames[j] + j-- + } + frames[j+1] = x + } +} + +func uniqueRuntimeFuncPCFrames(frames []runtimeFuncPCFrame) []runtimeFuncPCFrame { + if len(frames) < 2 { + return frames + } + out := frames[:1] + for i := 1; i < len(frames); i++ { + if frames[i].entry == out[len(out)-1].entry { + out[len(out)-1] = frames[i] + continue + } + out = append(out, frames[i]) + } + return out +} + +func buildRuntimeFuncPCIndex(frames []runtimeFuncPCFrame) runtimePCPageIndex { + if len(frames) == 0 { + return runtimePCPageIndex{} + } + base := frames[0].entry >> runtimeFuncPCPageShift + last := frames[len(frames)-1].entry >> runtimeFuncPCPageShift + if last < base { + return runtimePCPageIndex{} + } + npages := last - base + 2 + if npages > 1<<20 && npages > uintptr(len(frames))*64 { + return runtimePCPageIndex{} + } + pages := make([]uint32, npages) + next := 0 + for page := range pages { + limit := (base + uintptr(page)) << runtimeFuncPCPageShift + for next < len(frames) && frames[next].entry < limit { + next++ + } + pages[page] = uint32(next) + } + return runtimePCPageIndex{base: base, pages: pages} +} + +func runtimeFuncPCFrameIndex(pc uintptr) int { + frames := runtimeFuncPCFrames + if len(frames) == 0 { + return -1 + } + lo, hi := 0, len(frames) + if pages := runtimeFuncPCIndex.pages; len(pages) != 0 { + page := pc >> runtimeFuncPCPageShift + if page >= runtimeFuncPCIndex.base { + off := page - runtimeFuncPCIndex.base + if off < uintptr(len(pages)) { + lo = int(pages[off]) + if off+1 < uintptr(len(pages)) { + hi = int(pages[off+1]) + } + if lo > 0 { + lo-- + } + if hi < len(frames) { + hi++ + } + } + } + } + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if frames[mid].entry > pc { + hi = mid + } else { + lo = mid + 1 + } + } + idx := lo - 1 + if idx < 0 { + return -1 + } + return idx +} + +func funcEntryForIndex(index uint32) uintptr { + if index == 0 { + return 0 + } + initRuntimeFuncPCFrames() + if uintptr(index) >= uintptr(len(runtimeFuncPCEntries)) { + return 0 + } + return runtimeFuncPCEntries[index] +} + +func funcPCFrameForPC(pc uintptr) (pcSymbol, bool) { + if pc == 0 { + return pcSymbol{}, false + } + initRuntimeFuncPCFrames() + idx := runtimeFuncPCFrameIndex(pc) + if idx < 0 { + return pcSymbol{}, false + } + frame := runtimeFuncPCFrames[idx] + return pcSymbol{ + pc: pc, + entry: frame.entry, + function: frame.function, + file: frame.file, + line: frame.startLine, + startLine: frame.startLine, + ok: true, + }, true +} + func initRuntimePCLineFrames() { if runtimePCLineInit { return @@ -518,7 +749,10 @@ func initRuntimePCLineFrames() { } pc := site.pc fn := funcInfoAt(uintptr(rec.funcIndex) - 1) - entry := symbolPC(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + entry := funcEntryForIndex(rec.funcIndex) + if entry == 0 { + entry = symbolPC(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + } if entry == 0 { sym := addrInfoSymbol(pc) entry = sym.entry @@ -683,6 +917,39 @@ func pcLineFrameForPC(pc, entry uintptr) (pcSymbol, bool) { }, true } +func pcLineFrameForExactPC(pc uintptr) (pcSymbol, bool) { + if pc == 0 { + return pcSymbol{}, false + } + initRuntimePCLineFrames() + frames := runtimePCLineFrames + if len(frames) == 0 { + return pcSymbol{}, false + } + lo, hi := 0, len(frames) + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if frames[mid].pc >= pc { + hi = mid + } else { + lo = mid + 1 + } + } + if lo >= len(frames) || frames[lo].pc != pc { + return pcSymbol{}, false + } + frame := frames[lo] + return pcSymbol{ + pc: pc, + entry: frame.entry, + function: frame.function, + file: frame.file, + line: frame.line, + startLine: frame.startLine, + ok: true, + }, true +} + func mergePCLineSymbol(base, line pcSymbol) pcSymbol { if line.entry == 0 { line.entry = base.entry @@ -717,8 +984,8 @@ func frameSymbol(pc uintptr) pcSymbol { } } } - sym := addrInfoSymbol(pc) if pc == 0 { + sym := addrInfoSymbol(pc) if frame, ok := rtdebug.FrameForPC(pc); ok { return pcSymbol{ pc: pc, @@ -732,6 +999,14 @@ func frameSymbol(pc uintptr) pcSymbol { } return sym } + if lineSym, ok := pcLineFrameForExactPC(pc); ok { + return lineSym + } + if lineSym, ok := pcLineFrameForExactPC(pc - 1); ok { + lineSym.pc = pc + return lineSym + } + sym := addrInfoSymbol(pc) if lineSym, ok := pcLineFrameForPC(pc, sym.entry); ok { return mergePCLineSymbol(sym, lineSym) } @@ -745,6 +1020,11 @@ func frameSymbol(pc uintptr) pcSymbol { return callSym } } + if !sym.ok { + if funcSym, ok := funcPCFrameForPC(pc); ok { + return funcSym + } + } if frame, ok := rtdebug.FrameForPC(pc); ok { return pcSymbol{ pc: pc, From 4237d69564bbd56248b3fe2e2d01fe0919290b74 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 23:20:25 +0800 Subject: [PATCH 10/23] runtime: slim FuncForPC cache hot path --- .../lib/runtime/pprof_runtime_stub_llgo.go | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index aa69889194..ec0b91983e 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -3,6 +3,8 @@ package runtime import ( + "unsafe" + llrt "github.com/goplus/llgo/runtime/internal/runtime" ) @@ -94,11 +96,24 @@ type funcForPCCacheEntry struct { } var funcForPCCache [funcForPCCacheSize]funcForPCCacheEntry +var funcForPCLast funcForPCCacheEntry func FuncForPC(pc uintptr) *Func { - if fn := cachedFuncForPC(pc); fn != nil { + if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { + return fn + } + entry := (*funcForPCCacheEntry)(unsafe.Add( + unsafe.Pointer(&funcForPCCache[0]), + funcForPCCacheIndex(pc)*unsafe.Sizeof(funcForPCCacheEntry{}), + )) + if fn := entry.fn; fn != nil && entry.pc == pc { + funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} return fn } + return funcForPCSlow(pc) +} + +func funcForPCSlow(pc uintptr) *Func { if sym, ok := funcPCFrameForPC(pc); ok { fn := newFuncForPC(pc, sym) cacheFuncForPC(pc, fn) @@ -131,19 +146,14 @@ func newFuncForPC(pc uintptr, sym pcSymbol) *Func { } } -func cachedFuncForPC(pc uintptr) *Func { - entry := &funcForPCCache[funcForPCCacheIndex(pc)] - fn := entry.fn - if fn != nil && entry.pc == pc && fn.pc == pc { - return fn - } - return nil -} - func cacheFuncForPC(pc uintptr, fn *Func) { - entry := &funcForPCCache[funcForPCCacheIndex(pc)] + entry := (*funcForPCCacheEntry)(unsafe.Add( + unsafe.Pointer(&funcForPCCache[0]), + funcForPCCacheIndex(pc)*unsafe.Sizeof(funcForPCCacheEntry{}), + )) entry.fn = fn entry.pc = pc + funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} } func funcForPCCacheIndex(pc uintptr) uintptr { From 5a78b12783711e4e1429821d250310b6ab22ed8d Mon Sep 17 00:00:00 2001 From: Li Jie Date: Tue, 30 Jun 2026 23:51:24 +0800 Subject: [PATCH 11/23] cl: make pc-line labels clone-safe --- cl/caller_frame_test.go | 1 + cl/instr.go | 5 +++-- ssa/ssa_test.go | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index d3b53cf582..00bad87fed 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -363,6 +363,7 @@ func leaf() {} `!"example.com/foo.top"`, `!"caller_frame_compile.go"`, "__llgo_pcsite_", + "${:uid}", `.pushsection llgo_pcline`, `.quad __llgo_pcsite_`, } { diff --git a/cl/instr.go b/cl/instr.go index be19636a60..6db43beaea 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -1381,6 +1381,7 @@ func (p *context) emitPCLineLabel(b llssa.Builder, pos token.Pos) { p.pcLineSeq++ id := pcLineID(p.fn.Name(), p.pcLineSeq) label := pcLineLabelName(id) + asmLabel := label + "_${:uid}" ptrDirective := ".quad" align := "3" if p.prog.PointerSize() == 4 { @@ -1388,10 +1389,10 @@ func (p *context) emitPCLineLabel(b llssa.Builder, pos token.Pos) { align = "2" } b.InlineAsm( - label + ":\n" + + asmLabel + ":\n" + ".pushsection llgo_pcline,\"ao\",@progbits," + asmQuoteSymbol(p.fn.Name()) + "\n" + ".p2align " + align + "\n" + - ptrDirective + " " + label + "\n" + + ptrDirective + " " + asmLabel + "\n" + ".quad " + uint64Hex(id) + "\n" + ".popsection", ) diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 378bde70ff..1adb41cf88 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -204,6 +204,40 @@ func TestFuncInfoMetadataDoesNotPreserveFunctions(t *testing.T) { testFuncInfoMetadataDoesNotPreserveFunctions(t) } +func TestPCLineMetadataEmission(t *testing.T) { + prog := NewProgram(nil) + pkg := prog.NewPackage("main", "main") + + pkg.EmitPCLineInfo(0, "ignored", "ignored.go", -1, -1) + pkg.EmitPCLineInfo(0x1234, "", "ignored.go", -1, -1) + if ir := pkg.String(); strings.Contains(ir, PCLineMetadataName) { + t.Fatalf("invalid pcline rows should not emit metadata:\n%s", ir) + } + + pkg.EmitPCLineInfo(0x1234, "main.live", "call.go", 23, 5) + pkg.EmitPCLineInfo(0x5678, "main.negative", "negative.go", -7, -1) + ir := pkg.String() + for _, want := range []string{ + `!llgo.pcline = !{!`, + `i64 4660`, + `!"main.live"`, + `!"call.go"`, + `i32 23`, + `i32 5`, + `i64 22136`, + `!"main.negative"`, + `!"negative.go"`, + `i32 0`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("missing pcline field %s:\n%s", want, ir) + } + } + if strings.Contains(ir, `ptr @main.live`) || strings.Contains(ir, `ptr @"main.live"`) { + t.Fatalf("pcline metadata must not contain function pointer operands:\n%s", ir) + } +} + func testFuncInfoMetadataDoesNotPreserveFunctions(t *testing.T) { t.Helper() From cffaa09c139deb598f0551fb35a3ea654f59f87c Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 00:37:54 +0800 Subject: [PATCH 12/23] ci: extend macos coverage timeout --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index abf6b3bb67..98388e9012 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -75,7 +75,7 @@ jobs: - name: Test with coverage if: startsWith(matrix.os, 'macos') - run: go test -timeout 30m -coverprofile="coverage.txt" -covermode=atomic ./... + run: go test -timeout 45m -coverprofile="coverage.txt" -covermode=atomic ./... - name: Test with embedded emulator env env: From 9d0f94b582c70014548f0f79b0b3c049e1f584e2 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 08:39:42 +0800 Subject: [PATCH 13/23] runtime: guard funcinfo table initialization --- internal/build/funcinfo/funcinfo.go | 2 + internal/build/funcinfo_table.go | 5 ++ internal/build/funcinfo_table_test.go | 3 + runtime/internal/lib/runtime/symtab.go | 65 ++++++++++++++++--- test/go/runtime_lineinfo_stack_test.go | 87 ++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 7 deletions(-) diff --git a/internal/build/funcinfo/funcinfo.go b/internal/build/funcinfo/funcinfo.go index 10dc6aab6c..76c7cc123c 100644 --- a/internal/build/funcinfo/funcinfo.go +++ b/internal/build/funcinfo/funcinfo.go @@ -253,6 +253,8 @@ func buildHash(records []Record) ([]uint16, error) { return nil, nil } if len(records) > math.MaxUint16 { + // Runtime hash slots store 1-based uint16 record indexes. Larger + // tables remain correct by omitting the hash and using linear lookup. return nil, nil } buckets := 1 diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 680fe604e3..727f064b3b 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -31,6 +31,7 @@ const ( funcInfoCountSymbol = "__llgo_funcinfo_count" funcInfoStringsSymbol = "__llgo_funcinfo_strings" funcInfoStringOffsetsSymbol = "__llgo_funcinfo_string_offsets" + funcInfoStringCountSymbol = "__llgo_funcinfo_string_count" funcInfoHashSymbol = "__llgo_funcinfo_hash" funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" pcLineTableSymbol = "__llgo_pcline_table" @@ -228,6 +229,7 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcSiteEndPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteEndPtrSymbol) stringsPtr := llvm.AddGlobal(mod, llvm.PointerType(i8Type, 0), funcInfoStringsSymbol) stringOffsetsPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoStringOffsetsSymbol) + stringCount := llvm.AddGlobal(mod, countType, funcInfoStringCountSymbol) hashPtr := llvm.AddGlobal(mod, llvm.PointerType(i16Type, 0), funcInfoHashSymbol) count := llvm.AddGlobal(mod, countType, funcInfoCountSymbol) pcLineCount := llvm.AddGlobal(mod, countType, pcLineCountSymbol) @@ -239,6 +241,7 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) + stringCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) count.SetInitializer(llvm.ConstInt(countType, 0, false)) pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) @@ -257,6 +260,7 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) + stringCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) count.SetInitializer(llvm.ConstInt(countType, 0, false)) pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) @@ -355,6 +359,7 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord llvm.ConstInt(countType, 0, false), llvm.ConstInt(countType, 0, false), })) + stringCount.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.StringOffsets)), false)) if len(encoded.Hash) == 0 { hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index 6371a0ec4e..8f749818e6 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -60,6 +60,7 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) "@__llgo_pcsite_end = global ptr null", "@__llgo_funcinfo_strings = global ptr", "@__llgo_funcinfo_string_offsets = global ptr", + "@__llgo_funcinfo_string_count = global i64 5", "@__llgo_funcinfo_hash = global ptr", "@__llgo_funcinfo_count = global i64 1", "@__llgo_pcline_count = global i64 0", @@ -116,6 +117,7 @@ func TestFuncInfoTableMaterializesPCLineMetadata(t *testing.T) { "@__llgo_pcsite_start = global ptr @__start_llgo_pcline", "@__llgo_pcsite_end = global ptr @__stop_llgo_pcline", "@__llgo_pcline_count = global i64 1", + "@__llgo_funcinfo_string_count = global i64 6", "module asm \".section llgo_pcline", `@"__llgo_pcline_table$data" = private unnamed_addr constant [1 x { i64, i32, i32, i32 }]`, "i64 4660", @@ -199,6 +201,7 @@ func TestFuncInfoTableEmptyDefinitions(t *testing.T) { "@__llgo_pcsite_end = global ptr null", "@__llgo_funcinfo_strings = global ptr null", "@__llgo_funcinfo_string_offsets = global ptr null", + "@__llgo_funcinfo_string_count = global i64 0", "@__llgo_funcinfo_hash = global ptr null", "@__llgo_funcinfo_count = global i64 0", "@__llgo_pcline_count = global i64 0", diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index ce9ef6dc7f..1936d5fdd3 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -9,6 +9,7 @@ import ( c "github.com/goplus/llgo/runtime/internal/clite" clitedebug "github.com/goplus/llgo/runtime/internal/clite/debug" + latomic "github.com/goplus/llgo/runtime/internal/lib/sync/atomic" rtdebug "github.com/goplus/llgo/runtime/internal/runtime" ) @@ -153,6 +154,9 @@ var runtimeFuncInfoStrings *c.Char //go:linkname runtimeFuncInfoStringOffsets __llgo_funcinfo_string_offsets var runtimeFuncInfoStringOffsets *uint32 +//go:linkname runtimeFuncInfoStringCount __llgo_funcinfo_string_count +var runtimeFuncInfoStringCount uintptr + //go:linkname runtimeFuncInfoHash __llgo_funcinfo_hash var runtimeFuncInfoHash *uint16 @@ -195,7 +199,7 @@ type runtimePCLineFrame struct { startLine int } -var runtimePCLineInit bool +var runtimePCLineInitState uint32 var runtimePCLineFrames []runtimePCLineFrame type runtimeFuncPCFrame struct { @@ -213,11 +217,17 @@ type runtimePCPageIndex struct { const runtimeFuncPCPageShift = 12 -var runtimeFuncPCInit bool +var runtimeFuncPCInitState uint32 var runtimeFuncPCFrames []runtimeFuncPCFrame var runtimeFuncPCEntries []uintptr var runtimeFuncPCIndex runtimePCPageIndex +const ( + runtimeFuncInfoInitUninit uint32 = iota + runtimeFuncInfoInitDone + runtimeFuncInfoInitBusy +) + func hasStringPrefix(s, prefix string) bool { if len(s) < len(prefix) { return false @@ -305,7 +315,8 @@ func cStringAppend(dst []byte, cstr *c.Char) []byte { } func funcInfoCString(id uint16) *c.Char { - if runtimeFuncInfoStrings == nil || runtimeFuncInfoStringOffsets == nil { + if runtimeFuncInfoStrings == nil || runtimeFuncInfoStringOffsets == nil || + uintptr(id) >= runtimeFuncInfoStringCount { return nil } off := *(*uint32)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoStringOffsets), uintptr(id)*unsafe.Sizeof(*runtimeFuncInfoStringOffsets))) @@ -499,10 +510,30 @@ func addrInfoSymbol(pc uintptr) pcSymbol { } func initRuntimeFuncPCFrames() { - if runtimeFuncPCInit { + if latomic.LoadUint32(&runtimeFuncPCInitState) == runtimeFuncInfoInitDone { return } - runtimeFuncPCInit = true + initRuntimeFuncPCFramesSlow() +} + +func initRuntimeFuncPCFramesSlow() { + for { + state := latomic.LoadUint32(&runtimeFuncPCInitState) + switch state { + case runtimeFuncInfoInitDone: + return + case runtimeFuncInfoInitUninit: + if latomic.CompareAndSwapUint32(&runtimeFuncPCInitState, runtimeFuncInfoInitUninit, runtimeFuncInfoInitBusy) { + initRuntimeFuncPCFramesOnce() + latomic.StoreUint32(&runtimeFuncPCInitState, runtimeFuncInfoInitDone) + return + } + } + c.Usleep(1) + } +} + +func initRuntimeFuncPCFramesOnce() { if runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 || runtimeFuncInfoStrings == nil || @@ -710,10 +741,30 @@ func funcPCFrameForPC(pc uintptr) (pcSymbol, bool) { } func initRuntimePCLineFrames() { - if runtimePCLineInit { + if latomic.LoadUint32(&runtimePCLineInitState) == runtimeFuncInfoInitDone { return } - runtimePCLineInit = true + initRuntimePCLineFramesSlow() +} + +func initRuntimePCLineFramesSlow() { + for { + state := latomic.LoadUint32(&runtimePCLineInitState) + switch state { + case runtimeFuncInfoInitDone: + return + case runtimeFuncInfoInitUninit: + if latomic.CompareAndSwapUint32(&runtimePCLineInitState, runtimeFuncInfoInitUninit, runtimeFuncInfoInitBusy) { + initRuntimePCLineFramesOnce() + latomic.StoreUint32(&runtimePCLineInitState, runtimeFuncInfoInitDone) + return + } + } + c.Usleep(1) + } +} + +func initRuntimePCLineFramesOnce() { if runtimePCLineTable == nil || runtimePCLineCount == 0 || runtimePCSiteStart == nil || diff --git a/test/go/runtime_lineinfo_stack_test.go b/test/go/runtime_lineinfo_stack_test.go index e9c9bf7334..a9e95bdbc3 100644 --- a/test/go/runtime_lineinfo_stack_test.go +++ b/test/go/runtime_lineinfo_stack_test.go @@ -175,6 +175,93 @@ func TestRuntimeLineInfoAndStack(t *testing.T) { } } +const runtimeFuncInfoConcurrentFirstUseProbe = `package main + +import ( + "runtime" + "strconv" + "strings" + "sync" +) + +func main() { + const n = 32 + start := make(chan struct{}) + errc := make(chan string, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + errc <- checkRuntimeInfo() + }() + } + close(start) + wg.Wait() + close(errc) + for err := range errc { + if err != "" { + panic(err) + } + } +} + +//go:noinline +func checkRuntimeInfo() string { + pc, file, line, ok := runtime.Caller(0) // CONCURRENT_CALLER_MARK + if !ok || !strings.HasSuffix(file, "main.go") || line != CONCURRENT_CALLER_LINE { + return "bad caller: " + file + ":" + strconv.Itoa(line) + } + fn := runtime.FuncForPC(pc) + if fn == nil || fn.Name() != "main.checkRuntimeInfo" { + name := "" + if fn != nil { + name = fn.Name() + } + return "bad func: " + name + } + file, line = fn.FileLine(pc) + if !strings.HasSuffix(file, "main.go") || line != CONCURRENT_CALLER_LINE { + return "bad fileline: " + file + ":" + strconv.Itoa(line) + } + var pcs [8]uintptr + n := runtime.Callers(0, pcs[:]) + frames := runtime.CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if frame.Function == "main.checkRuntimeInfo" { + if !strings.HasSuffix(frame.File, "main.go") || frame.Line == 0 { + return "bad frame: " + frame.File + ":" + strconv.Itoa(frame.Line) + } + return "" + } + if !more { + return "missing frame" + } + } +} +` + +func TestRuntimeFuncInfoConcurrentFirstUse(t *testing.T) { + source := runtimeFuncInfoConcurrentFirstUseProbe + source = strings.ReplaceAll(source, "CONCURRENT_CALLER_LINE", strconv.Itoa(markerLine(source, "CONCURRENT_CALLER_MARK"))) + + dir := t.TempDir() + file := filepath.Join(dir, "main.go") + if err := os.WriteFile(file, []byte(source), 0644); err != nil { + t.Fatal(err) + } + + repoRoot := findStringConversionRepoRoot(t) + t.Setenv("LLGO_ROOT", repoRoot) + cmd := exec.Command("go", "run", "./cmd/llgo", "run", "-a", file) + cmd.Dir = repoRoot + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("llgo concurrent funcinfo probe failed: %v\n%s", err, out) + } +} + func markerLine(source, marker string) int { line := 1 for _, part := range strings.SplitAfter(source, "\n") { From e804579c7dbf8885c88c8aa255bcd54ae0e30db2 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 11:24:28 +0800 Subject: [PATCH 14/23] runtime: fix funcinfo entry pc line metadata --- cl/compile.go | 19 ++++++- cl/funcinfo_metadata_test.go | 9 ++++ internal/build/build.go | 2 + internal/build/funcinfo_table.go | 75 +++++++++++++++++++++++++- internal/build/funcinfo_table_test.go | 45 ++++++++++++++++ internal/build/main_module.go | 3 +- runtime/internal/lib/runtime/symtab.go | 61 ++++++++++++++++++++- test/go/runtime_lineinfo_stack_test.go | 54 +++++++++++++++++-- 8 files changed, 259 insertions(+), 9 deletions(-) diff --git a/cl/compile.go b/cl/compile.go index 443a0cc9cb..f3d2c21338 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -565,7 +565,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun if pkgTypes != nil { goName = funcName(pkgTypes, f, false) } - pos := p.goProg.Fset.Position(f.Pos()) + pos := p.funcInfoPosition(f) pkg.EmitFuncInfo(fn.Name(), goName, pos.Filename, pos.Line, pos.Column) } var childInits []func() @@ -701,6 +701,23 @@ func (p *context) getFuncBodyPos(f *ssa.Function) token.Position { return p.goProg.Fset.Position(f.Pos()) } +func (p *context) funcInfoPosition(f *ssa.Function) token.Position { + if f != nil { + switch syntax := f.Syntax().(type) { + case *ast.FuncDecl: + if syntax.Body != nil && len(syntax.Body.List) != 0 { + return p.goProg.Fset.Position(syntax.Body.List[0].Pos()) + } + case *ast.FuncLit: + if syntax.Body != nil && len(syntax.Body.List) != 0 { + return p.goProg.Fset.Position(syntax.Body.List[0].Pos()) + } + } + return p.goProg.Fset.Position(f.Pos()) + } + return token.Position{} +} + func isGlobal(v *types.Var) bool { // TODO(lijie): better implementation return strings.HasPrefix(v.Parent().String(), "package ") diff --git a/cl/funcinfo_metadata_test.go b/cl/funcinfo_metadata_test.go index 5319b16751..902813cfad 100644 --- a/cl/funcinfo_metadata_test.go +++ b/cl/funcinfo_metadata_test.go @@ -92,6 +92,15 @@ func (T) method() {} if got := records["foo.top"].name; got != "foo.top" { t.Fatalf("caller stack frame name = %q, want foo.top", got) } + if got := records["foo.top"].line; got != 6 { + t.Fatalf("top funcinfo line = %d, want first body statement line 6", got) + } + if got := records["foo.leaf"].line; got != 9 { + t.Fatalf("leaf funcinfo line = %d, want line 9", got) + } + if got := records["foo.T.method"].line; got != 11 { + t.Fatalf("empty method funcinfo line = %d, want declaration line 11", got) + } } func TestNoInlineDirectiveDisablesTailCalls(t *testing.T) { diff --git a/internal/build/build.go b/internal/build/build.go index ea58216d85..e6c2501361 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1046,6 +1046,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa // Use a stable synthetic name to avoid confusing it with the real main package in traces/logs. funcInfo := prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil) pcLineInfo := collectPCLineInfo(linkedOrder) + funcInfoStubs := collectFuncInfoStubIndexes(linkedOrder, funcInfo) entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{ rtInit: needRuntime, pyInit: needPyInit, @@ -1055,6 +1056,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa abiSymbols: linkedModuleGlobals(linkedOrder), funcInfo: funcInfo, pcLineInfo: pcLineInfo, + funcInfoStubs: funcInfoStubs, }) entryObjFile, err := exportObject(ctx, "entry_main", entryPkg.ExportFile, entryPkg.LPkg) if err != nil { diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 727f064b3b..64c2b12385 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -34,6 +34,8 @@ const ( funcInfoStringCountSymbol = "__llgo_funcinfo_string_count" funcInfoHashSymbol = "__llgo_funcinfo_hash" funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" + funcInfoStubIndexesSymbol = "__llgo_funcinfo_stub_indexes" + funcInfoStubCountSymbol = "__llgo_funcinfo_stub_count" pcLineTableSymbol = "__llgo_pcline_table" pcLineCountSymbol = "__llgo_pcline_count" pcSiteStartPtrSymbol = "__llgo_pcsite_start" @@ -45,6 +47,8 @@ const ( funcInfoStringsDataSymbol = "__llgo_funcinfo_strings$data" funcInfoStringOffsetsDataSymbol = "__llgo_funcinfo_string_offsets$data" funcInfoHashDataSymbol = "__llgo_funcinfo_hash$data" + funcInfoStubIndexesDataSymbol = "__llgo_funcinfo_stub_indexes$data" + closureStubPrefix = "__llgo_stub." ) type funcInfoRecord struct { @@ -121,6 +125,45 @@ func collectPCLineInfo(pkgs []Package) []pcLineRecord { return out } +func collectFuncInfoStubIndexes(pkgs []Package, records []funcInfoRecord) []uint32 { + if len(records) == 0 { + return nil + } + recordBySymbol := make(map[string]uint32, len(records)) + for i, rec := range records { + if rec.symbol != "" { + recordBySymbol[rec.symbol] = uint32(i + 1) + } + } + seen := make(map[uint32]none) + for _, pkg := range pkgs { + if pkg == nil || pkg.LPkg == nil { + continue + } + fn := pkg.LPkg.Module().FirstFunction() + for !fn.IsNil() { + name := fn.Name() + if target, ok := strings.CutPrefix(name, closureStubPrefix); ok { + if idx := recordBySymbol[target]; idx != 0 { + seen[idx] = none{} + } + } + fn = llvm.NextFunction(fn) + } + } + if len(seen) == 0 { + return nil + } + out := make([]uint32, 0, len(seen)) + for idx := range seen { + out = append(out, idx) + } + sort.Slice(out, func(i, j int) bool { + return out[i] < out[j] + }) + return out +} + func prepareFuncInfoTableRecords(records []funcInfoRecord, liveSymbols map[string]none) []funcInfoRecord { if len(records) == 0 { return nil @@ -195,7 +238,7 @@ func readPCLineInfo(mod llvm.Module) []pcLineRecord { return out } -func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord, pcLines []pcLineRecord) { +func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord, pcLines []pcLineRecord, stubIndexes []uint32) { mod := pkg.Module() llvmCtx := mod.Context() i8Type := llvmCtx.Int8Type() @@ -232,6 +275,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord stringCount := llvm.AddGlobal(mod, countType, funcInfoStringCountSymbol) hashPtr := llvm.AddGlobal(mod, llvm.PointerType(i16Type, 0), funcInfoHashSymbol) count := llvm.AddGlobal(mod, countType, funcInfoCountSymbol) + stubIndexesPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoStubIndexesSymbol) + stubCount := llvm.AddGlobal(mod, countType, funcInfoStubCountSymbol) pcLineCount := llvm.AddGlobal(mod, countType, pcLineCountSymbol) hashMask := llvm.AddGlobal(mod, countType, funcInfoHashMaskSymbol) if len(records) == 0 && len(pcLines) == 0 { @@ -244,6 +289,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord stringCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) count.SetInitializer(llvm.ConstInt(countType, 0, false)) + stubIndexesPtr.SetInitializer(llvm.ConstPointerNull(stubIndexesPtr.GlobalValueType())) + stubCount.SetInitializer(llvm.ConstInt(countType, 0, false)) pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) return @@ -263,6 +310,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord stringCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashPtr.SetInitializer(llvm.ConstPointerNull(hashPtr.GlobalValueType())) count.SetInitializer(llvm.ConstInt(countType, 0, false)) + stubIndexesPtr.SetInitializer(llvm.ConstPointerNull(stubIndexesPtr.GlobalValueType())) + stubCount.SetInitializer(llvm.ConstInt(countType, 0, false)) pcLineCount.SetInitializer(llvm.ConstInt(countType, 0, false)) hashMask.SetInitializer(llvm.ConstInt(countType, 0, false)) return @@ -382,6 +431,30 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord hashMask.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Hash)-1), false)) } count.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Records)), false)) + stubIndexValues := make([]llvm.Value, 0, len(stubIndexes)) + for _, idx := range stubIndexes { + if idx == 0 || int(idx) > len(encoded.Records) { + continue + } + stubIndexValues = append(stubIndexValues, llvm.ConstInt(i32Type, uint64(idx), false)) + } + if len(stubIndexValues) == 0 { + stubIndexesPtr.SetInitializer(llvm.ConstPointerNull(stubIndexesPtr.GlobalValueType())) + stubCount.SetInitializer(llvm.ConstInt(countType, 0, false)) + } else { + stubIndexArrayType := llvm.ArrayType(i32Type, len(stubIndexValues)) + stubIndexData := llvm.AddGlobal(mod, stubIndexArrayType, funcInfoStubIndexesDataSymbol) + stubIndexData.SetInitializer(llvm.ConstArray(i32Type, stubIndexValues)) + stubIndexData.SetLinkage(llvm.PrivateLinkage) + stubIndexData.SetGlobalConstant(true) + stubIndexData.SetUnnamedAddr(true) + stubIndexData.SetAlignment(4) + stubIndexesPtr.SetInitializer(llvm.ConstInBoundsGEP(stubIndexArrayType, stubIndexData, []llvm.Value{ + llvm.ConstInt(countType, 0, false), + llvm.ConstInt(countType, 0, false), + })) + stubCount.SetInitializer(llvm.ConstInt(countType, uint64(len(stubIndexValues)), false)) + } } func shouldEmitRuntimeELFSites(ctx *context) bool { diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index 8f749818e6..1a58825e51 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -63,6 +63,8 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) "@__llgo_funcinfo_string_count = global i64 5", "@__llgo_funcinfo_hash = global ptr", "@__llgo_funcinfo_count = global i64 1", + "@__llgo_funcinfo_stub_indexes = global ptr null", + "@__llgo_funcinfo_stub_count = global i64 0", "@__llgo_pcline_count = global i64 0", "@__llgo_funcinfo_hash_mask = global i64 1", `@"__llgo_funcinfo_table$data" = private unnamed_addr constant [1 x { i16, i16, i16, i16, i16, i16, i32 }]`, @@ -83,6 +85,47 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) } } +func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { + prog := llssa.NewProgram(nil) + src := prog.NewPackage("example.com/p", "example.com/p") + src.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "live.go", 17, 3) + src.EmitFuncInfo("example.com/p.other", "example.com/p.Other", "other.go", 23, 1) + src.NewFunc(closureStubPrefix+"example.com/p.live", llssa.NoArgsNoRet, llssa.InC) + + records := collectFuncInfo([]Package{{LPkg: src}}) + stubs := collectFuncInfoStubIndexes([]Package{{LPkg: src}}, records) + if len(stubs) != 1 || records[stubs[0]-1].symbol != "example.com/p.live" { + t.Fatalf("stub indexes = %+v for records %+v, want live", stubs, records) + } + + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records, funcInfoStubs: stubs}) + ir := entry.LPkg.String() + for _, want := range []string{ + "@__llgo_funcinfo_stub_indexes = global ptr", + "@__llgo_funcinfo_stub_count = global i64 1", + `@"__llgo_funcinfo_stub_indexes$data" = private unnamed_addr constant [1 x i32]`, + "@__llgo_funcinfo_count = global i64 2", + } { + if !strings.Contains(ir, want) { + t.Fatalf("funcinfo stub index table IR missing %q:\n%s", want, ir) + } + } + if strings.Contains(ir, closureStubPrefix) { + t.Fatalf("stub index table should not add stub symbol strings:\n%s", ir) + } +} + func TestFuncInfoTableMaterializesPCLineMetadata(t *testing.T) { prog := llssa.NewProgram(nil) src := prog.NewPackage("example.com/p", "example.com/p") @@ -204,6 +247,8 @@ func TestFuncInfoTableEmptyDefinitions(t *testing.T) { "@__llgo_funcinfo_string_count = global i64 0", "@__llgo_funcinfo_hash = global ptr null", "@__llgo_funcinfo_count = global i64 0", + "@__llgo_funcinfo_stub_indexes = global ptr null", + "@__llgo_funcinfo_stub_count = global i64 0", "@__llgo_pcline_count = global i64 0", "@__llgo_funcinfo_hash_mask = global i64 0", } { diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 83289dd23f..6992bcec73 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -45,6 +45,7 @@ type genConfig struct { abiSymbols map[string]none funcInfo []funcInfoRecord pcLineInfo []pcLineRecord + funcInfoStubs []uint32 } // genMainModule generates the main entry module for an llgo program. @@ -62,7 +63,7 @@ func genMainModule(ctx *context, rtPkgPath string, pkg *packages.Package, cfg *g argvValueType := prog.Pointer(prog.CStr()) argvVar := mainPkg.NewVarEx("__llgo_argv", prog.Pointer(argvValueType)) argvVar.InitNil() - emitFuncInfoTable(ctx, mainPkg, cfg.funcInfo, cfg.pcLineInfo) + emitFuncInfoTable(ctx, mainPkg, cfg.funcInfo, cfg.pcLineInfo, cfg.funcInfoStubs) exportFile := pkg.ExportFile if exportFile == "" { diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 1936d5fdd3..5dac41d161 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -166,6 +166,12 @@ var runtimeFuncInfoCount uintptr //go:linkname runtimeFuncInfoHashMask __llgo_funcinfo_hash_mask var runtimeFuncInfoHashMask uintptr +//go:linkname runtimeFuncInfoStubIndexes __llgo_funcinfo_stub_indexes +var runtimeFuncInfoStubIndexes *uint32 + +//go:linkname runtimeFuncInfoStubCount __llgo_funcinfo_stub_count +var runtimeFuncInfoStubCount uintptr + type runtimePCLineRecord struct { id uint64 funcIndex uint32 @@ -226,6 +232,8 @@ const ( runtimeFuncInfoInitUninit uint32 = iota runtimeFuncInfoInitDone runtimeFuncInfoInitBusy + runtimeClosureStubPrefix = "__llgo_stub." + runtimePublicClosureStubPrefix = "_llgo_stub." ) func hasStringPrefix(s, prefix string) bool { @@ -333,6 +341,11 @@ func pcLineAt(i uintptr) *runtimePCLineRecord { return (*runtimePCLineRecord)(unsafe.Add(unsafe.Pointer(runtimePCLineTable), i*size)) } +func funcInfoStubIndexAt(i uintptr) uint32 { + size := unsafe.Sizeof(*runtimeFuncInfoStubIndexes) + return *(*uint32)(unsafe.Add(unsafe.Pointer(runtimeFuncInfoStubIndexes), i*size)) +} + func funcInfoHashString(s string) uintptr { const ( offset = uint32(2166136261) @@ -435,12 +448,25 @@ func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { return nil } +func funcInfoForRuntimeSymbol(symbol string) *runtimeFuncInfoRecord { + if rec := funcInfoForSymbol(symbol); rec != nil { + return rec + } + if hasStringPrefix(symbol, runtimeClosureStubPrefix) { + return funcInfoForSymbol(symbol[len(runtimeClosureStubPrefix):]) + } + if hasStringPrefix(symbol, runtimePublicClosureStubPrefix) { + return funcInfoForSymbol(symbol[len(runtimePublicClosureStubPrefix):]) + } + return nil +} + func applyFuncInfo(sym *pcSymbol, rawFunction string) { - rec := funcInfoForSymbol(rawFunction) + rec := funcInfoForRuntimeSymbol(rawFunction) if rec == nil { public := publicFunctionName(rawFunction) if public != rawFunction { - rec = funcInfoForSymbol(public) + rec = funcInfoForRuntimeSymbol(public) } } if rec == nil { @@ -568,6 +594,37 @@ func initRuntimeFuncPCFramesOnce() { entries[index] = pc } } + // Closure stubs are an ABI adapter and may go away in a future closure + // lowering. Keep the compatibility table light: it stores only target + // funcinfo record indexes, and live stub PCs are resolved lazily here. + if runtimeFuncInfoStubIndexes != nil && runtimeFuncInfoStubCount != 0 && runtimeFuncInfoStubCount <= runtimeFuncInfoCount { + for i := uintptr(0); i < runtimeFuncInfoStubCount; i++ { + index := funcInfoStubIndexAt(i) + if index == 0 || uintptr(index) > runtimeFuncInfoCount { + continue + } + fn := funcInfoAt(uintptr(index) - 1) + symbol := funcInfoJoinName(fn.symbolPkg, fn.symbolName) + if symbol == "" { + continue + } + pc := symbolPC(runtimeClosureStubPrefix + symbol) + if pc == 0 { + continue + } + function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) + if function == "" { + function = publicFunctionName(symbol) + } + frames = append(frames, runtimeFuncPCFrame{ + entry: pc, + funcIndex: index, + function: function, + file: funcInfoJoinFile(fn.fileRoot, fn.fileName), + startLine: int(fn.line), + }) + } + } sortRuntimeFuncPCFrames(frames) frames = uniqueRuntimeFuncPCFrames(frames) runtimeFuncPCFrames = frames diff --git a/test/go/runtime_lineinfo_stack_test.go b/test/go/runtime_lineinfo_stack_test.go index a9e95bdbc3..52fffcabef 100644 --- a/test/go/runtime_lineinfo_stack_test.go +++ b/test/go/runtime_lineinfo_stack_test.go @@ -28,6 +28,7 @@ import ( const runtimeLineInfoProbe = `package main import ( + "reflect" "strconv" "runtime" "runtime/debug" @@ -38,8 +39,9 @@ import ( func main() { checkCaller() checkCallerSkip() - checkFrames() + checkFrames() // FRAMES_MAIN_MARK checkFuncForPC() + checkFuncForPCFunctionValue() checkFuncInfoRename() checkRuntimeStack() checkPanicStack() @@ -69,14 +71,25 @@ func helperCallerSkip() { //go:noinline func checkFrames() { var pcs [8]uintptr - n := runtime.Callers(0, pcs[:]) + n := runtime.Callers(0, pcs[:]) // FRAMES_CHECK_MARK frames := runtime.CallersFrames(pcs[:n]) + seenCheckFrames := false + seenMain := false for { frame, more := frames.Next() if frame.Function == "main.checkFrames" { - if !strings.HasSuffix(frame.File, "main.go") || frame.Line == 0 { - panic("bad frame") + if !strings.HasSuffix(frame.File, "main.go") || frame.Line != FRAMES_CHECK_LINE { + panic("bad checkFrames frame: " + frame.File + ":" + strconv.Itoa(frame.Line)) + } + seenCheckFrames = true + } + if frame.Function == "main.main" { + if !strings.HasSuffix(frame.File, "main.go") || frame.Line != FRAMES_MAIN_LINE { + panic("bad main frame: " + frame.File + ":" + strconv.Itoa(frame.Line)) } + seenMain = true + } + if seenCheckFrames && seenMain { return } if !more { @@ -108,6 +121,36 @@ func checkFuncForPC() { } } +//go:noinline +func entryPCTarget() int { + return 7 // FUNC_ENTRY_TARGET_MARK +} + +//go:noinline +func checkFuncForPCFunctionValue() { + if entryPCTarget() != 7 { + panic("bad target") + } + pc := reflect.ValueOf(entryPCTarget).Pointer() + if pc == 0 { + panic("missing function value pc") + } + fn := runtime.FuncForPC(pc) + if fn == nil { + panic("missing function value func") + } + if name := fn.Name(); name != "main.entryPCTarget" { + panic("bad function value func: " + name) + } + if entry := fn.Entry(); entry == 0 { + panic("missing function value entry") + } + file, line := fn.FileLine(pc) + if !strings.HasSuffix(file, "main.go") || line != FUNC_ENTRY_TARGET_LINE { + panic("bad function value fileline: " + file + ":" + strconv.Itoa(line)) + } +} + //go:noinline func checkFuncInfoRename() { pc := renamedPC() @@ -156,7 +199,10 @@ func TestRuntimeLineInfoAndStack(t *testing.T) { source := runtimeLineInfoProbe source = strings.ReplaceAll(source, "CALLER_LINE", strconv.Itoa(markerLine(source, "CALLER_MARK"))) source = strings.ReplaceAll(source, "CALLER_SKIP_LINE", strconv.Itoa(markerLine(source, "CALLER_SKIP_MARK"))) + source = strings.ReplaceAll(source, "FRAMES_MAIN_LINE", strconv.Itoa(markerLine(source, "FRAMES_MAIN_MARK"))) + source = strings.ReplaceAll(source, "FRAMES_CHECK_LINE", strconv.Itoa(markerLine(source, "FRAMES_CHECK_MARK"))) source = strings.ReplaceAll(source, "FUNC_FILELINE_LINE", strconv.Itoa(markerLine(source, "FUNC_FILELINE_MARK"))) + source = strings.ReplaceAll(source, "FUNC_ENTRY_TARGET_LINE", strconv.Itoa(markerLine(source, "FUNC_ENTRY_TARGET_MARK"))) source = strings.ReplaceAll(source, "RUNTIME_STACK_LINE", strconv.Itoa(markerLine(source, "RUNTIME_STACK_MARK"))) source = strings.ReplaceAll(source, "DEBUG_STACK_LINE", strconv.Itoa(markerLine(source, "DEBUG_STACK_CALL_MARK"))) From 06356679e085fb786d1b1fbfb075fdc74d3a3c9f Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 12:50:07 +0800 Subject: [PATCH 15/23] runtime: publish funcinfo records for live stubs --- internal/build/build.go | 3 +- internal/build/funcinfo_table.go | 160 +++++++++++++++++++++++-- internal/build/funcinfo_table_test.go | 37 ++++-- internal/build/main_module.go | 2 +- runtime/internal/lib/runtime/symtab.go | 103 +++++++++++++++- 5 files changed, 279 insertions(+), 26 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index e6c2501361..7d348651e5 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1046,7 +1046,7 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, outputPa // Use a stable synthetic name to avoid confusing it with the real main package in traces/logs. funcInfo := prepareFuncInfoTableRecords(collectFuncInfo(linkedOrder), nil) pcLineInfo := collectPCLineInfo(linkedOrder) - funcInfoStubs := collectFuncInfoStubIndexes(linkedOrder, funcInfo) + funcInfoStubs := collectFuncInfoStubRecords(linkedOrder, funcInfo) entryPkg := genMainModule(ctx, llssa.PkgRuntime, pkg, &genConfig{ rtInit: needRuntime, pyInit: needPyInit, @@ -1345,6 +1345,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { return fmt.Errorf("run LLVM passes failed for %v: %v", pkgPath, err) } } + emitFuncInfoStubSites(ctx, ret) printCmds := ctx.shouldPrintCommands(verbose) cgoLLFiles, cgoLdflags, err := buildCgo(ctx, aPkg, aPkg.Package.Syntax, externs, printCmds) diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 64c2b12385..36e3ab9b1f 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -36,10 +36,14 @@ const ( funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" funcInfoStubIndexesSymbol = "__llgo_funcinfo_stub_indexes" funcInfoStubCountSymbol = "__llgo_funcinfo_stub_count" + funcInfoStubSiteStartPtrSymbol = "__llgo_funcinfo_stubsite_start" + funcInfoStubSiteEndPtrSymbol = "__llgo_funcinfo_stubsite_end" pcLineTableSymbol = "__llgo_pcline_table" pcLineCountSymbol = "__llgo_pcline_count" pcSiteStartPtrSymbol = "__llgo_pcsite_start" pcSiteEndPtrSymbol = "__llgo_pcsite_end" + funcInfoStubSiteStartSymbol = "__start_llgo_funcinfo_stubsite" + funcInfoStubSiteEndSymbol = "__stop_llgo_funcinfo_stubsite" pcSiteStartSymbol = "__start_llgo_pcline" pcSiteEndSymbol = "__stop_llgo_pcline" funcInfoDataSymbol = "__llgo_funcinfo_table$data" @@ -67,6 +71,11 @@ type pcLineRecord struct { column uint32 } +type funcInfoStubRecord struct { + symbol string + funcIndex uint32 +} + func collectFuncInfo(pkgs []Package) []funcInfoRecord { seen := make(map[string]funcInfoRecord) for _, pkg := range pkgs { @@ -125,7 +134,7 @@ func collectPCLineInfo(pkgs []Package) []pcLineRecord { return out } -func collectFuncInfoStubIndexes(pkgs []Package, records []funcInfoRecord) []uint32 { +func collectFuncInfoStubRecords(pkgs []Package, records []funcInfoRecord) []funcInfoStubRecord { if len(records) == 0 { return nil } @@ -135,17 +144,21 @@ func collectFuncInfoStubIndexes(pkgs []Package, records []funcInfoRecord) []uint recordBySymbol[rec.symbol] = uint32(i + 1) } } - seen := make(map[uint32]none) + seen := make(map[string]funcInfoStubRecord) for _, pkg := range pkgs { if pkg == nil || pkg.LPkg == nil { continue } fn := pkg.LPkg.Module().FirstFunction() for !fn.IsNil() { + if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { + fn = llvm.NextFunction(fn) + continue + } name := fn.Name() if target, ok := strings.CutPrefix(name, closureStubPrefix); ok { if idx := recordBySymbol[target]; idx != 0 { - seen[idx] = none{} + seen[name] = funcInfoStubRecord{symbol: name, funcIndex: idx} } } fn = llvm.NextFunction(fn) @@ -154,12 +167,12 @@ func collectFuncInfoStubIndexes(pkgs []Package, records []funcInfoRecord) []uint if len(seen) == 0 { return nil } - out := make([]uint32, 0, len(seen)) - for idx := range seen { - out = append(out, idx) + out := make([]funcInfoStubRecord, 0, len(seen)) + for _, rec := range seen { + out = append(out, rec) } sort.Slice(out, func(i, j int) bool { - return out[i] < out[j] + return out[i].symbol < out[j].symbol }) return out } @@ -238,7 +251,7 @@ func readPCLineInfo(mod llvm.Module) []pcLineRecord { return out } -func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord, pcLines []pcLineRecord, stubIndexes []uint32) { +func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord, pcLines []pcLineRecord, stubRecords []funcInfoStubRecord) { mod := pkg.Module() llvmCtx := mod.Context() i8Type := llvmCtx.Int8Type() @@ -261,6 +274,10 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord i32Type, i32Type, }, false) + stubSiteRecordType := llvmCtx.StructType([]llvm.Type{ + llvm.PointerType(i8Type, 0), + i64Type, + }, false) pcSiteRecordType := llvmCtx.StructType([]llvm.Type{ llvm.PointerType(i8Type, 0), i64Type, @@ -270,6 +287,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcLinePtr := llvm.AddGlobal(mod, llvm.PointerType(pcLineRecordType, 0), pcLineTableSymbol) pcSiteStartPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteStartPtrSymbol) pcSiteEndPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteEndPtrSymbol) + stubSiteStartPtr := llvm.AddGlobal(mod, llvm.PointerType(stubSiteRecordType, 0), funcInfoStubSiteStartPtrSymbol) + stubSiteEndPtr := llvm.AddGlobal(mod, llvm.PointerType(stubSiteRecordType, 0), funcInfoStubSiteEndPtrSymbol) stringsPtr := llvm.AddGlobal(mod, llvm.PointerType(i8Type, 0), funcInfoStringsSymbol) stringOffsetsPtr := llvm.AddGlobal(mod, llvm.PointerType(i32Type, 0), funcInfoStringOffsetsSymbol) stringCount := llvm.AddGlobal(mod, countType, funcInfoStringCountSymbol) @@ -284,6 +303,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + stubSiteStartPtr.SetInitializer(llvm.ConstPointerNull(stubSiteStartPtr.GlobalValueType())) + stubSiteEndPtr.SetInitializer(llvm.ConstPointerNull(stubSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) stringCount.SetInitializer(llvm.ConstInt(countType, 0, false)) @@ -305,6 +326,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + stubSiteStartPtr.SetInitializer(llvm.ConstPointerNull(stubSiteStartPtr.GlobalValueType())) + stubSiteEndPtr.SetInitializer(llvm.ConstPointerNull(stubSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) stringOffsetsPtr.SetInitializer(llvm.ConstPointerNull(stringOffsetsPtr.GlobalValueType())) stringCount.SetInitializer(llvm.ConstInt(countType, 0, false)) @@ -374,7 +397,17 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) } } - emitRuntimeFuncInfoSentinels(mod, ctx.prog.PointerSize(), shouldEmitRuntimeELFSites(ctx) && len(pcLineValues) != 0) + emitELFSites := shouldEmitRuntimeELFSites(ctx) + emitRuntimeFuncInfoELFSites(mod, ctx.prog.PointerSize(), emitELFSites && len(pcLineValues) != 0, emitELFSites && len(stubRecords) != 0) + if emitELFSites && len(stubRecords) != 0 { + stubSiteStart := llvm.AddGlobal(mod, stubSiteRecordType, funcInfoStubSiteStartSymbol) + stubSiteEnd := llvm.AddGlobal(mod, stubSiteRecordType, funcInfoStubSiteEndSymbol) + stubSiteStartPtr.SetInitializer(stubSiteStart) + stubSiteEndPtr.SetInitializer(stubSiteEnd) + } else { + stubSiteStartPtr.SetInitializer(llvm.ConstPointerNull(stubSiteStartPtr.GlobalValueType())) + stubSiteEndPtr.SetInitializer(llvm.ConstPointerNull(stubSiteEndPtr.GlobalValueType())) + } stringArrayType := llvm.ArrayType(i8Type, len(encoded.Strings)) stringData := llvm.AddGlobal(mod, stringArrayType, funcInfoStringsDataSymbol) @@ -431,11 +464,17 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord hashMask.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Hash)-1), false)) } count.SetInitializer(llvm.ConstInt(countType, uint64(len(encoded.Records)), false)) - stubIndexValues := make([]llvm.Value, 0, len(stubIndexes)) - for _, idx := range stubIndexes { + stubIndexSeen := make(map[uint32]none, len(stubRecords)) + stubIndexValues := make([]llvm.Value, 0, len(stubRecords)) + for _, stub := range stubRecords { + idx := stub.funcIndex if idx == 0 || int(idx) > len(encoded.Records) { continue } + if _, ok := stubIndexSeen[idx]; ok { + continue + } + stubIndexSeen[idx] = none{} stubIndexValues = append(stubIndexValues, llvm.ConstInt(i32Type, uint64(idx), false)) } if len(stubIndexValues) == 0 { @@ -464,8 +503,80 @@ func shouldEmitRuntimeELFSites(ctx *context) bool { ctx.buildConf.Target == "" } -func emitRuntimeFuncInfoSentinels(mod llvm.Module, pointerSize int, pcSite bool) { - if !pcSite { +func emitFuncInfoStubSites(ctx *context, pkg llssa.Package) { + if !shouldEmitRuntimeELFSites(ctx) || pkg == nil || !ctx.prog.FuncInfoMetadataEnabled() { + return + } + mod := pkg.Module() + llvmCtx := mod.Context() + builder := llvmCtx.NewBuilder() + defer builder.Dispose() + asmType := llvm.FunctionType(llvmCtx.VoidType(), nil, false) + ptrDirective := ".quad" + align := "3" + if ctx.prog.PointerSize() == 4 { + ptrDirective = ".long" + align = "2" + } + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { + continue + } + symbol := fn.Name() + target, ok := strings.CutPrefix(symbol, closureStubPrefix) + if !ok || target == "" { + continue + } + entry := fn.EntryBasicBlock() + if entry.IsNil() { + continue + } + first := entry.FirstInstruction() + if first.IsNil() { + builder.SetInsertPointAtEnd(entry) + } else { + builder.SetInsertPointBefore(first) + } + instruction := ".pushsection llgo_funcinfo_stubsite,\"ao\",@progbits," + asmQuoteELFSymbol(symbol) + "\n" + + ".p2align " + align + "\n" + + ptrDirective + " " + asmQuoteELFSymbol(symbol) + "\n" + + ".quad " + uint64Hex(funcInfoSymbolID(target)) + "\n" + + ".popsection" + asm := llvm.InlineAsm(asmType, instruction, "", true, false, llvm.InlineAsmDialectATT, false) + builder.CreateCall(asmType, asm, nil, "") + } +} + +func funcInfoSymbolID(symbol string) uint64 { + const ( + offset = uint64(14695981039346656037) + prime = uint64(1099511628211) + ) + h := offset + for i := 0; i < len(symbol); i++ { + h ^= uint64(symbol[i]) + h *= prime + } + if h == 0 { + return 1 + } + return h +} + +func uint64Hex(v uint64) string { + const hexdigits = "0123456789abcdef" + var buf [18]byte + buf[0] = '0' + buf[1] = 'x' + for i := len(buf) - 1; i >= 2; i-- { + buf[i] = hexdigits[v&0xf] + v >>= 4 + } + return string(buf[:]) +} + +func emitRuntimeFuncInfoELFSites(mod llvm.Module, pointerSize int, pcSite bool, stubSite bool) { + if !pcSite && !stubSite { return } ptrDirective := ".quad" @@ -481,9 +592,32 @@ func emitRuntimeFuncInfoSentinels(mod llvm.Module, pointerSize int, pcSite bool) asm.WriteString(ptrDirective + " 0\n") asm.WriteString(".quad 0\n") } + if stubSite { + asm.WriteString(".section llgo_funcinfo_stubsite,\"aR\",@progbits\n") + asm.WriteString(".p2align " + align + "\n") + asm.WriteString(ptrDirective + " 0\n") + asm.WriteString(".quad 0\n") + } mod.SetInlineAsm(asm.String()) } +func asmQuoteELFSymbol(symbol string) string { + var b strings.Builder + b.Grow(len(symbol) + 2) + b.WriteByte('"') + for i := 0; i < len(symbol); i++ { + switch symbol[i] { + case '\\', '"': + b.WriteByte('\\') + case '$': + b.WriteByte('$') + } + b.WriteByte(symbol[i]) + } + b.WriteByte('"') + return b.String() +} + func toFuncInfoRecords(records []funcInfoRecord) []buildfuncinfo.Record { out := make([]buildfuncinfo.Record, len(records)) for i, rec := range records { diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index 1a58825e51..31fc6838bc 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -90,14 +90,8 @@ func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { src := prog.NewPackage("example.com/p", "example.com/p") src.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "live.go", 17, 3) src.EmitFuncInfo("example.com/p.other", "example.com/p.Other", "other.go", 23, 1) - src.NewFunc(closureStubPrefix+"example.com/p.live", llssa.NoArgsNoRet, llssa.InC) - - records := collectFuncInfo([]Package{{LPkg: src}}) - stubs := collectFuncInfoStubIndexes([]Package{{LPkg: src}}, records) - if len(stubs) != 1 || records[stubs[0]-1].symbol != "example.com/p.live" { - t.Fatalf("stub indexes = %+v for records %+v, want live", stubs, records) - } - + stubFn := src.NewFunc(closureStubPrefix+"example.com/p.live", llssa.NoArgsNoRet, llssa.InC) + stubFn.MakeBody(1).Return() ctx := &context{ prog: prog, buildConf: &Config{ @@ -106,6 +100,27 @@ func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { Goarch: "amd64", }, } + prog.EnableFuncInfoMetadata(true) + emitFuncInfoStubSites(ctx, src) + srcIR := src.String() + for _, want := range []string{ + "call void asm sideeffect", + ".pushsection llgo_funcinfo_stubsite", + `.quad \22__llgo_stub.example.com/p.live\22`, + ".quad 0x", + } { + if !strings.Contains(srcIR, want) { + t.Fatalf("package stub site IR missing %q:\n%s", want, srcIR) + } + } + + records := collectFuncInfo([]Package{{LPkg: src}}) + stubs := collectFuncInfoStubRecords([]Package{{LPkg: src}}, records) + if len(stubs) != 1 || records[stubs[0].funcIndex-1].symbol != "example.com/p.live" || + stubs[0].symbol != closureStubPrefix+"example.com/p.live" { + t.Fatalf("stub indexes = %+v for records %+v, want live", stubs, records) + } + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ PkgPath: "example.com/main", ExportFile: "main.a", @@ -114,14 +129,18 @@ func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { for _, want := range []string{ "@__llgo_funcinfo_stub_indexes = global ptr", "@__llgo_funcinfo_stub_count = global i64 1", + "@__llgo_funcinfo_stubsite_start = global ptr @__start_llgo_funcinfo_stubsite", + "@__llgo_funcinfo_stubsite_end = global ptr @__stop_llgo_funcinfo_stubsite", `@"__llgo_funcinfo_stub_indexes$data" = private unnamed_addr constant [1 x i32]`, "@__llgo_funcinfo_count = global i64 2", + "module asm \".section llgo_funcinfo_stubsite", + ".quad 0", } { if !strings.Contains(ir, want) { t.Fatalf("funcinfo stub index table IR missing %q:\n%s", want, ir) } } - if strings.Contains(ir, closureStubPrefix) { + if strings.Contains(ir, closureStubPrefix+"example.com/p.live\\00") { t.Fatalf("stub index table should not add stub symbol strings:\n%s", ir) } } diff --git a/internal/build/main_module.go b/internal/build/main_module.go index 6992bcec73..67378f6e2e 100644 --- a/internal/build/main_module.go +++ b/internal/build/main_module.go @@ -45,7 +45,7 @@ type genConfig struct { abiSymbols map[string]none funcInfo []funcInfoRecord pcLineInfo []pcLineRecord - funcInfoStubs []uint32 + funcInfoStubs []funcInfoStubRecord } // genMainModule generates the main entry module for an llgo program. diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 5dac41d161..0d425ecf0a 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -172,6 +172,17 @@ var runtimeFuncInfoStubIndexes *uint32 //go:linkname runtimeFuncInfoStubCount __llgo_funcinfo_stub_count var runtimeFuncInfoStubCount uintptr +type runtimeFuncInfoStubSiteRecord struct { + pc uintptr + symbolID uint64 +} + +//go:linkname runtimeFuncInfoStubSiteStart __llgo_funcinfo_stubsite_start +var runtimeFuncInfoStubSiteStart *runtimeFuncInfoStubSiteRecord + +//go:linkname runtimeFuncInfoStubSiteEnd __llgo_funcinfo_stubsite_end +var runtimeFuncInfoStubSiteEnd *runtimeFuncInfoStubSiteRecord + type runtimePCLineRecord struct { id uint64 funcIndex uint32 @@ -594,9 +605,11 @@ func initRuntimeFuncPCFramesOnce() { entries[index] = pc } } + frames = appendRuntimeFuncInfoStubSiteFrames(frames) // Closure stubs are an ABI adapter and may go away in a future closure - // lowering. Keep the compatibility table light: it stores only target - // funcinfo record indexes, and live stub PCs are resolved lazily here. + // lowering. Keep the fallback compatibility table light: it stores only + // target funcinfo record indexes. On ELF we prefer the associated stub-site + // section above because linkers do not expose local stubs through dlsym. if runtimeFuncInfoStubIndexes != nil && runtimeFuncInfoStubCount != 0 && runtimeFuncInfoStubCount <= runtimeFuncInfoCount { for i := uintptr(0); i < runtimeFuncInfoStubCount; i++ { index := funcInfoStubIndexAt(i) @@ -632,6 +645,92 @@ func initRuntimeFuncPCFramesOnce() { runtimeFuncPCIndex = buildRuntimeFuncPCIndex(frames) } +func appendRuntimeFuncInfoStubSiteFrames(frames []runtimeFuncPCFrame) []runtimeFuncPCFrame { + if runtimeFuncInfoStubSiteStart == nil || runtimeFuncInfoStubSiteEnd == nil { + return frames + } + start := uintptr(unsafe.Pointer(runtimeFuncInfoStubSiteStart)) + end := uintptr(unsafe.Pointer(runtimeFuncInfoStubSiteEnd)) + size := unsafe.Sizeof(*runtimeFuncInfoStubSiteStart) + if end <= start || size == 0 || (end-start)%size != 0 { + return frames + } + nsite := (end - start) / size + if nsite > runtimeFuncInfoCount*16 || nsite > 1<<20 { + return frames + } + for i := uintptr(0); i < nsite; i++ { + site := (*runtimeFuncInfoStubSiteRecord)(unsafe.Pointer(start + i*size)) + if site == nil || site.pc == 0 || site.symbolID == 0 { + continue + } + funcIndex := funcInfoIndexForSymbolID(site.symbolID) + if funcIndex == 0 || uintptr(funcIndex) > runtimeFuncInfoCount { + continue + } + fn := funcInfoAt(uintptr(funcIndex) - 1) + symbol := funcInfoJoinName(fn.symbolPkg, fn.symbolName) + function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) + if function == "" { + function = publicFunctionName(symbol) + } + frames = append(frames, runtimeFuncPCFrame{ + entry: site.pc, + funcIndex: funcIndex, + function: function, + file: funcInfoJoinFile(fn.fileRoot, fn.fileName), + startLine: int(fn.line), + }) + } + return frames +} + +func funcInfoIndexForSymbolID(id uint64) uint32 { + if id == 0 || runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 { + return 0 + } + for i := uintptr(0); i < runtimeFuncInfoCount; i++ { + rec := funcInfoAt(i) + if funcInfoSymbolIDFromRecord(rec) == id { + return uint32(i + 1) + } + } + return 0 +} + +func funcInfoSymbolIDFromRecord(rec *runtimeFuncInfoRecord) uint64 { + const ( + offset = uint64(14695981039346656037) + prime = uint64(1099511628211) + ) + if rec == nil { + return 0 + } + h := offset + h = funcInfoHashCString(h, funcInfoCString(rec.symbolPkg)) + pkgLen := cStringLen(funcInfoCString(rec.symbolPkg)) + name := funcInfoCString(rec.symbolName) + if pkgLen != 0 && cStringLen(name) != 0 { + h ^= uint64('.') + h *= prime + } + h = funcInfoHashCString(h, name) + if h == 0 { + return 1 + } + return h +} + +func funcInfoHashCString(h uint64, s *c.Char) uint64 { + const prime = uint64(1099511628211) + for s != nil && *s != 0 { + h ^= uint64(byte(*s)) + h *= prime + s = (*c.Char)(unsafe.Add(unsafe.Pointer(s), 1)) + } + return h +} + func sortRuntimeFuncPCFrames(frames []runtimeFuncPCFrame) { if len(frames) < 2 { return From 7f3c0510ce75f319345384b8502056c18fc75a27 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 13:12:00 +0800 Subject: [PATCH 16/23] runtime: skip ELF stub-site records during LTO --- internal/build/funcinfo_table.go | 11 ++++++++--- internal/build/funcinfo_table_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index 36e3ab9b1f..f7ffc57900 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -398,8 +398,9 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord } } emitELFSites := shouldEmitRuntimeELFSites(ctx) - emitRuntimeFuncInfoELFSites(mod, ctx.prog.PointerSize(), emitELFSites && len(pcLineValues) != 0, emitELFSites && len(stubRecords) != 0) - if emitELFSites && len(stubRecords) != 0 { + emitStubSites := shouldEmitRuntimeStubELFSites(ctx) + emitRuntimeFuncInfoELFSites(mod, ctx.prog.PointerSize(), emitELFSites && len(pcLineValues) != 0, emitStubSites && len(stubRecords) != 0) + if emitStubSites && len(stubRecords) != 0 { stubSiteStart := llvm.AddGlobal(mod, stubSiteRecordType, funcInfoStubSiteStartSymbol) stubSiteEnd := llvm.AddGlobal(mod, stubSiteRecordType, funcInfoStubSiteEndSymbol) stubSiteStartPtr.SetInitializer(stubSiteStart) @@ -503,8 +504,12 @@ func shouldEmitRuntimeELFSites(ctx *context) bool { ctx.buildConf.Target == "" } +func shouldEmitRuntimeStubELFSites(ctx *context) bool { + return shouldEmitRuntimeELFSites(ctx) && !ctx.buildConf.ltoEnabled() +} + func emitFuncInfoStubSites(ctx *context, pkg llssa.Package) { - if !shouldEmitRuntimeELFSites(ctx) || pkg == nil || !ctx.prog.FuncInfoMetadataEnabled() { + if !shouldEmitRuntimeStubELFSites(ctx) || pkg == nil || !ctx.prog.FuncInfoMetadataEnabled() { return } mod := pkg.Module() diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index 31fc6838bc..9ed6739cad 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -22,6 +22,7 @@ import ( "github.com/xgo-dev/llvm" + "github.com/goplus/llgo/internal/lto" "github.com/goplus/llgo/internal/packages" llssa "github.com/goplus/llgo/ssa" ) @@ -143,6 +144,30 @@ func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { if strings.Contains(ir, closureStubPrefix+"example.com/p.live\\00") { t.Fatalf("stub index table should not add stub symbol strings:\n%s", ir) } + + ltoCtx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + LTO: lto.Full, + }, + } + ltoEntry := genMainModule(ltoCtx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records, funcInfoStubs: stubs}) + ltoIR := ltoEntry.LPkg.String() + for _, bad := range []string{ + "@__llgo_funcinfo_stubsite_start = global ptr @__start_llgo_funcinfo_stubsite", + "@__llgo_funcinfo_stubsite_end = global ptr @__stop_llgo_funcinfo_stubsite", + "module asm \".section llgo_funcinfo_stubsite", + } { + if strings.Contains(ltoIR, bad) { + t.Fatalf("full LTO funcinfo table should not emit stub site %q:\n%s", bad, ltoIR) + } + } } func TestFuncInfoTableMaterializesPCLineMetadata(t *testing.T) { From e93b2324198be859a5561a8631de624bbc035153 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 14:40:21 +0800 Subject: [PATCH 17/23] runtime: reduce funcinfo lookup initialization cost --- internal/build/build.go | 3 + internal/build/build_test.go | 20 +++ .../lib/runtime/pprof_runtime_stub_llgo.go | 7 + runtime/internal/lib/runtime/symtab.go | 127 ++++++++++++------ 4 files changed, 119 insertions(+), 38 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 7d348651e5..97c68389dc 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1107,6 +1107,9 @@ func linkedModuleGlobals(pkgs []Package) map[string]none { continue } for g := pkg.LPkg.Module().FirstGlobal(); !g.IsNil(); g = gllvm.NextGlobal(g) { + if g.IsDeclaration() { + continue + } seen[g.Name()] = none{} } } diff --git a/internal/build/build_test.go b/internal/build/build_test.go index bc6f89d785..c8ce08e9e6 100644 --- a/internal/build/build_test.go +++ b/internal/build/build_test.go @@ -22,6 +22,7 @@ import ( "github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/packages" llssa "github.com/goplus/llgo/ssa" + "github.com/xgo-dev/llvm" ) func TestMain(m *testing.M) { @@ -98,6 +99,25 @@ func TestIsFuncInfoEnabled(t *testing.T) { } } +func TestLinkedModuleGlobalsSkipsDeclarations(t *testing.T) { + prog := llssa.NewProgram(nil) + lpkg := prog.NewPackage("example.com/p", "example.com/p") + mod := lpkg.Module() + i32 := mod.Context().Int32Type() + + defined := llvm.AddGlobal(mod, i32, "example.com/p.defined") + defined.SetInitializer(llvm.ConstInt(i32, 1, false)) + llvm.AddGlobal(mod, i32, "example.com/p.declared") + + got := linkedModuleGlobals([]Package{{LPkg: lpkg}}) + if _, ok := got["example.com/p.defined"]; !ok { + t.Fatalf("linkedModuleGlobals missing defined global: %#v", got) + } + if _, ok := got["example.com/p.declared"]; ok { + t.Fatalf("linkedModuleGlobals should skip external declarations: %#v", got) + } +} + func mockRun(args []string, cfg *Config) { defer mockable.DisableMock() mockable.EnableMock() diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index ec0b91983e..a0bae4d160 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -114,6 +114,13 @@ func FuncForPC(pc uintptr) *Func { } func funcForPCSlow(pc uintptr) *Func { + if pc&3 != 0 { + if sym := frameSymbol(pc); sym.ok { + fn := newFuncForPC(pc, sym) + cacheFuncForPC(pc, fn) + return fn + } + } if sym, ok := funcPCFrameForPC(pc); ok { fn := newFuncForPC(pc, sym) cacheFuncForPC(pc, fn) diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 0d425ecf0a..bdfebb167d 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -222,9 +222,6 @@ var runtimePCLineFrames []runtimePCLineFrame type runtimeFuncPCFrame struct { entry uintptr funcIndex uint32 - function string - file string - startLine int } type runtimePCPageIndex struct { @@ -405,6 +402,34 @@ func funcInfoJoinName(pkgID, nameID uint16) string { return string(buf) } +func funcInfoNameLen(pkgID, nameID uint16) int { + pkgLen := cStringLen(funcInfoCString(pkgID)) + nameLen := cStringLen(funcInfoCString(nameID)) + if pkgLen == 0 { + return nameLen + } + if nameLen == 0 { + return pkgLen + } + return pkgLen + 1 + nameLen +} + +func appendFuncInfoName(dst []byte, pkgID, nameID uint16) []byte { + pkg := funcInfoCString(pkgID) + name := funcInfoCString(nameID) + pkgLen := cStringLen(pkg) + nameLen := cStringLen(name) + if pkgLen == 0 { + return cStringAppend(dst, name) + } + if nameLen == 0 { + return cStringAppend(dst, pkg) + } + dst = cStringAppend(dst, pkg) + dst = append(dst, '.') + return cStringAppend(dst, name) +} + func funcInfoJoinFile(rootID, nameID uint16) string { root := funcInfoCString(rootID) name := funcInfoCString(nameID) @@ -426,6 +451,53 @@ func funcInfoPackedFile(file uint32) string { return funcInfoJoinFile(uint16(file>>16), uint16(file)) } +func maxFuncInfoSymbolLen() int { + maxLen := 0 + for i := uintptr(0); i < runtimeFuncInfoCount; i++ { + fn := funcInfoAt(i) + if n := funcInfoNameLen(fn.symbolPkg, fn.symbolName); n > maxLen { + maxLen = n + } + } + return maxLen +} + +func symbolPCBytes(name []byte) uintptr { + if len(name) == 0 { + return 0 + } + name = append(name, 0) + return uintptr(clitedebug.Symbol((*c.Char)(unsafe.Pointer(&name[0])))) +} + +func symbolPCFuncInfoName(buf []byte, pkgID, nameID uint16) uintptr { + name := appendFuncInfoName(buf[:0], pkgID, nameID) + return symbolPCBytes(name) +} + +func symbolPCPrefixedFuncInfoName(buf []byte, prefix string, pkgID, nameID uint16) uintptr { + name := append(buf[:0], prefix...) + name = appendFuncInfoName(name, pkgID, nameID) + return symbolPCBytes(name) +} + +func funcInfoFunctionName(fn *runtimeFuncInfoRecord) string { + if fn == nil { + return "" + } + if function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)); function != "" { + return function + } + return publicFunctionName(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) +} + +func funcInfoFileName(fn *runtimeFuncInfoRecord) string { + if fn == nil { + return "" + } + return funcInfoJoinFile(fn.fileRoot, fn.fileName) +} + func funcInfoForSymbol(symbol string) *runtimeFuncInfoRecord { if symbol == "" || runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 { return nil @@ -582,24 +654,17 @@ func initRuntimeFuncPCFramesOnce() { } frames := make([]runtimeFuncPCFrame, 0, runtimeFuncInfoCount) entries := make([]uintptr, runtimeFuncInfoCount+1) + symbolBuf := make([]byte, 0, maxFuncInfoSymbolLen()+len(runtimeClosureStubPrefix)+1) for i := uintptr(0); i < runtimeFuncInfoCount; i++ { fn := funcInfoAt(i) - pc := symbolPC(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + pc := symbolPCFuncInfoName(symbolBuf, fn.symbolPkg, fn.symbolName) if pc == 0 { continue } index := uint32(i + 1) - function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) - if function == "" { - function = publicFunctionName(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) - } - file := funcInfoJoinFile(fn.fileRoot, fn.fileName) frames = append(frames, runtimeFuncPCFrame{ entry: pc, funcIndex: index, - function: function, - file: file, - startLine: int(fn.line), }) if entries[index] == 0 || pc < entries[index] { entries[index] = pc @@ -617,24 +682,13 @@ func initRuntimeFuncPCFramesOnce() { continue } fn := funcInfoAt(uintptr(index) - 1) - symbol := funcInfoJoinName(fn.symbolPkg, fn.symbolName) - if symbol == "" { - continue - } - pc := symbolPC(runtimeClosureStubPrefix + symbol) + pc := symbolPCPrefixedFuncInfoName(symbolBuf, runtimeClosureStubPrefix, fn.symbolPkg, fn.symbolName) if pc == 0 { continue } - function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) - if function == "" { - function = publicFunctionName(symbol) - } frames = append(frames, runtimeFuncPCFrame{ entry: pc, funcIndex: index, - function: function, - file: funcInfoJoinFile(fn.fileRoot, fn.fileName), - startLine: int(fn.line), }) } } @@ -668,18 +722,9 @@ func appendRuntimeFuncInfoStubSiteFrames(frames []runtimeFuncPCFrame) []runtimeF if funcIndex == 0 || uintptr(funcIndex) > runtimeFuncInfoCount { continue } - fn := funcInfoAt(uintptr(funcIndex) - 1) - symbol := funcInfoJoinName(fn.symbolPkg, fn.symbolName) - function := publicFunctionName(funcInfoJoinName(fn.namePkg, fn.nameName)) - if function == "" { - function = publicFunctionName(symbol) - } frames = append(frames, runtimeFuncPCFrame{ entry: site.pc, funcIndex: funcIndex, - function: function, - file: funcInfoJoinFile(fn.fileRoot, fn.fileName), - startLine: int(fn.line), }) } return frames @@ -885,13 +930,18 @@ func funcPCFrameForPC(pc uintptr) (pcSymbol, bool) { return pcSymbol{}, false } frame := runtimeFuncPCFrames[idx] + if frame.funcIndex == 0 || uintptr(frame.funcIndex) > runtimeFuncInfoCount { + return pcSymbol{}, false + } + fn := funcInfoAt(uintptr(frame.funcIndex) - 1) + line := int(fn.line) return pcSymbol{ pc: pc, entry: frame.entry, - function: frame.function, - file: frame.file, - line: frame.startLine, - startLine: frame.startLine, + function: funcInfoFunctionName(fn), + file: funcInfoFileName(fn), + line: line, + startLine: line, ok: true, }, true } @@ -945,6 +995,7 @@ func initRuntimePCLineFramesOnce() { return } frames := make([]runtimePCLineFrame, 0, nsite) + symbolBuf := make([]byte, 0, maxFuncInfoSymbolLen()+1) for i := uintptr(0); i < nsite; i++ { site := (*runtimePCSiteRecord)(unsafe.Pointer(start + i*size)) if site == nil || site.id == 0 || site.pc == 0 { @@ -958,7 +1009,7 @@ func initRuntimePCLineFramesOnce() { fn := funcInfoAt(uintptr(rec.funcIndex) - 1) entry := funcEntryForIndex(rec.funcIndex) if entry == 0 { - entry = symbolPC(funcInfoJoinName(fn.symbolPkg, fn.symbolName)) + entry = symbolPCFuncInfoName(symbolBuf, fn.symbolPkg, fn.symbolName) } if entry == 0 { sym := addrInfoSymbol(pc) From 7b244429df5324a2f779996398aa3ffd5fd29b1b Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 17:21:54 +0800 Subject: [PATCH 18/23] test: cover runtime caller metadata edges --- cl/caller_frame_test.go | 219 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index 00bad87fed..b2a63fe47b 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -5,6 +5,7 @@ package cl import ( "go/ast" + "go/importer" "go/parser" "go/token" "go/types" @@ -129,6 +130,34 @@ func buildCallerFrameSSAPackage(t *testing.T, pkgPath, src string) (*gossa.Packa return ssapkg, files } +func newLLSSAProgForTarget(t *testing.T, target *llssa.Target) llssa.Program { + t.Helper() + prog := llssa.NewProgram(target) + prog.SetRuntime(func() *types.Package { + rt, err := importer.For("source", nil).Import(llssa.PkgRuntime) + if err != nil { + t.Fatal("load runtime failed:", err) + } + return rt + }) + if target != nil && target.GOARCH != "" { + prog.TypeSizes(types.SizesFor("gc", target.GOARCH)) + } + return prog +} + +func newRuntimeCallerAnalysis(pkg *gossa.Package) *runtimeCallerAnalysis { + funcs, trackable := collectRuntimeCallerFunctions(pkg) + return &runtimeCallerAnalysis{ + pkg: pkg, + funcs: funcs, + trackable: trackable, + callsites: collectRuntimeCallerCallsites(funcs), + memo: make(map[*gossa.Function]bool), + visiting: make(map[*gossa.Function]bool), + } +} + func TestRuntimeCallerPackageDetection(t *testing.T) { ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" @@ -215,6 +244,127 @@ func FuncForPC(pc uintptr) uintptr { return 0 } } } +func TestRuntimeCallerAnalysisEdgeCases(t *testing.T) { + if fnUsesRuntimeCaller(nil) { + t.Fatal("nil function should not use runtime caller metadata") + } + if fnUsesRuntimeCaller(&gossa.Function{}) { + t.Fatal("function without a package should not use runtime caller metadata") + } + if runtimeCallerFuncSet(nil) != nil { + t.Fatal("nil package should have no runtime caller set") + } + if fnHasDirectRuntimeCaller(nil) { + t.Fatal("nil function should not have direct runtime caller use") + } + if functionBelongsToPackage(nil, nil) { + t.Fatal("nil function/package should not belong to a package") + } + if typeBelongsToPackage(types.Typ[types.Int], nil) { + t.Fatal("types should not belong to a nil package") + } + if isRuntimeCallerLookupFunc(nil) { + t.Fatal("nil function should not be a runtime caller lookup") + } + called := false + forEachCall(nil, func(*gossa.CallCommon) { + called = true + }) + if called { + t.Fatal("forEachCall should ignore nil functions") + } + + ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +type I interface { Call() } +type J interface { Call() } +type T struct{} + +func target() { runtime.Caller(0) } +func plain() {} +func call(fn func()) { fn() } +func callRuntime() { call(target) } +func (T) Call() { runtime.Caller(0) } +func viaStatic() { var i I = T{}; i.Call() } +func viaChange(j J) { var i I = j; i.Call() } +func viaParam(i I) { i.Call() } +func passInterface() { var i I = T{}; viaParam(i) } +`) + analysis := newRuntimeCallerAnalysis(ssapkg) + if analysis.fnMayReachRuntimeCaller(nil) { + t.Fatal("nil function should not reach runtime caller metadata") + } + if targets, ok := analysis.functionValueTargets(ssapkg.Func("callRuntime"), ssapkg.Func("target")); !ok || !targets[ssapkg.Func("target")] { + t.Fatal("static function value should resolve to its target") + } + if _, ok := analysis.functionValueTargets(ssapkg.Func("target"), nil); ok { + t.Fatal("nil function value should be unresolved") + } + if _, ok := analysis.functionParamTargets(ssapkg.Func("call"), 99); ok { + t.Fatal("out-of-range function argument should be unresolved") + } + callFn := ssapkg.Func("call") + callParam := callFn.Params[0] + callParams := callFn.Params + callFn.Params = nil + if _, ok := analysis.functionValueTargets(callFn, callParam); ok { + t.Fatal("function parameter missing from Params should be unresolved") + } + callFn.Params = callParams + + iface := ssapkg.Pkg.Scope().Lookup("I").Type().Underlying().(*types.Interface) + method := iface.Method(0) + if !analysis.fnMayReachRuntimeCaller(ssapkg.Func("viaStatic")) { + t.Fatal("static interface dispatch should reach runtime caller metadata") + } + if !analysis.fnMayReachRuntimeCaller(ssapkg.Func("viaChange")) { + t.Fatal("changed interface dispatch should conservatively reach runtime caller metadata") + } + if targets, ok := analysis.interfaceMethodTargets(ssapkg.Func("viaParam"), ssapkg.Func("viaParam").Params[0], method); !ok || len(targets) == 0 { + t.Fatal("interface parameter callsites should resolve concrete method targets") + } + analysis.callsites[ssapkg.Func("viaParam")] = []*gossa.CallCommon{{}} + if _, ok := analysis.interfaceMethodTargets(ssapkg.Func("viaParam"), ssapkg.Func("viaParam").Params[0], method); ok { + t.Fatal("out-of-range interface argument should be unresolved") + } + if _, ok := analysis.interfaceMethodTargets(ssapkg.Func("viaStatic"), nil, method); ok { + t.Fatal("nil interface receiver should be unresolved") + } + if _, ok := analysis.staticInterfaceMethodTargets(&gossa.ChangeInterface{}, method); ok { + t.Fatal("empty interface conversion should be unresolved") + } + viaParam := ssapkg.Func("viaParam") + interfaceParam := viaParam.Params[0] + interfaceParams := viaParam.Params + viaParam.Params = nil + if _, ok := analysis.interfaceMethodTargets(viaParam, interfaceParam, method); ok { + t.Fatal("interface parameter missing from Params should be unresolved") + } + viaParam.Params = interfaceParams + if _, ok := analysis.methodTargetsForType(nil, nil); ok { + t.Fatal("nil method lookup should be unresolved") + } + other := types.NewFunc(token.NoPos, ssapkg.Pkg, "Other", nil) + if _, ok := analysis.methodTargetsForType(ssapkg.Type("T").Type(), other); ok { + t.Fatal("missing interface method should be unresolved") + } + if idx, ok := parameterIndex(ssapkg.Func("target"), nil); ok || idx != 0 { + t.Fatal("nil parameter should not be found") + } + + methodOnlyPkg, _ := buildCallerFrameSSAPackage(t, "example.com/methodonly", `package methodonly +import "runtime" + +type T struct{} +func (T) Call() { runtime.Caller(0) } +var _ = T{} +`) + if runtimeCallerFuncSet(methodOnlyPkg) != nil { + t.Fatal("method-only runtime caller use should not mark top-level functions") + } +} + func TestCallerFrameTrackingEligibility(t *testing.T) { if (&context{}).shouldTrackCallerFrames() { t.Fatal("missing compiler state should not track caller frames") @@ -380,6 +530,31 @@ func leaf() {} } } +func TestCompileRuntimeCallerPCLineMetadata32Bit(t *testing.T) { + ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func top() { + runtime.Caller(0) +} +`) + prog := newLLSSAProgForTarget(t, &llssa.Target{GOOS: "linux", GOARCH: "386"}) + prog.EnableFuncInfoMetadata(true) + pkg, err := NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + for _, want := range []string{ + `.p2align 2`, + `.long __llgo_pcsite_`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("missing 32-bit pcline asm %q:\n%s", want, ir) + } + } +} + func TestCompileRuntimeCallerPCLineEscapesDollarInInlineAsm(t *testing.T) { ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" @@ -412,6 +587,50 @@ func top() { } } +func TestRuntimeCallerInstrumentationEdgeCases(t *testing.T) { + ssapkg, _ := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" + +func top() { + runtime.Caller(0) +} +`) + prog := newLLSSAProgForTarget(t, &llssa.Target{GOOS: "linux", GOARCH: "amd64"}) + prog.EnableFuncInfoMetadata(true) + pkg := prog.NewPackage("foo", "example.com/foo") + fn := pkg.NewFunc("example.com/foo.top", llssa.NoArgsNoRet, llssa.InGo) + ctx := &context{ + prog: prog, + pkg: pkg, + fn: fn, + goFn: ssapkg.Func("top"), + fset: token.NewFileSet(), + trackCallerFrames: true, + runtimeCallerFuncs: runtimeCallerFuncSet(ssapkg), + } + var b llssa.Builder + ctx.pushCallerLocationFrame(b, nil) + ctx.recordRuntimeLocation(b, token.NoPos, "RecordCallerLocation") + ctx.emitPCLineLabel(b, token.NoPos) + ctx.popCallerLocationFrame(b) + + if pos := (&context{}).funcInfoPosition(nil); pos.IsValid() { + t.Fatal("nil function should have no funcinfo position") + } + if canEmitPCLineLabelsForTarget(nil) { + t.Fatal("nil target should not emit pc-line labels") + } + if canEmitPCLineLabelsForTarget(&llssa.Target{GOOS: "linux", GOARCH: "wasm"}) { + t.Fatal("wasm target should not emit pc-line labels") + } + if canEmitPCLineLabelsForTarget(&llssa.Target{GOOS: "linux", GOARCH: "amd64", Target: "esp32"}) { + t.Fatal("named target should not emit pc-line labels") + } + if got, want := asmQuoteSymbol(`a\b"c$d`), `"a\\b\"c$$d"`; got != want { + t.Fatalf("asmQuoteSymbol() = %q, want %q", got, want) + } +} + func TestCompileRuntimeCallerPCLineMetadataSkippedOnDarwin(t *testing.T) { ssapkg, files := buildCallerFrameSSAPackage(t, "example.com/foo", `package foo import "runtime" From 3323f2e676e80a354997b67f802332f6962b8992 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 21:24:18 +0800 Subject: [PATCH 19/23] runtime: speed up funcinfo entry lookup --- internal/build/build.go | 1 + internal/build/funcinfo_table.go | 107 ++++++++++++- internal/build/funcinfo_table_test.go | 99 +++++++++++- .../lib/runtime/pprof_runtime_stub_llgo.go | 16 ++ runtime/internal/lib/runtime/symtab.go | 151 ++++++++++++++---- 5 files changed, 336 insertions(+), 38 deletions(-) diff --git a/internal/build/build.go b/internal/build/build.go index 97c68389dc..f5a7c87f8a 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -1348,6 +1348,7 @@ func buildPkg(ctx *context, aPkg *aPackage, verbose bool) error { return fmt.Errorf("run LLVM passes failed for %v: %v", pkgPath, err) } } + emitFuncInfoEntrySites(ctx, ret) emitFuncInfoStubSites(ctx, ret) printCmds := ctx.shouldPrintCommands(verbose) diff --git a/internal/build/funcinfo_table.go b/internal/build/funcinfo_table.go index f7ffc57900..c7a6d1498c 100644 --- a/internal/build/funcinfo_table.go +++ b/internal/build/funcinfo_table.go @@ -36,12 +36,16 @@ const ( funcInfoHashMaskSymbol = "__llgo_funcinfo_hash_mask" funcInfoStubIndexesSymbol = "__llgo_funcinfo_stub_indexes" funcInfoStubCountSymbol = "__llgo_funcinfo_stub_count" + funcInfoEntryStartPtrSymbol = "__llgo_funcinfo_entry_start" + funcInfoEntryEndPtrSymbol = "__llgo_funcinfo_entry_end" funcInfoStubSiteStartPtrSymbol = "__llgo_funcinfo_stubsite_start" funcInfoStubSiteEndPtrSymbol = "__llgo_funcinfo_stubsite_end" pcLineTableSymbol = "__llgo_pcline_table" pcLineCountSymbol = "__llgo_pcline_count" pcSiteStartPtrSymbol = "__llgo_pcsite_start" pcSiteEndPtrSymbol = "__llgo_pcsite_end" + funcInfoEntryStartSymbol = "__start_llgo_funcinfo_entry" + funcInfoEntryEndSymbol = "__stop_llgo_funcinfo_entry" funcInfoStubSiteStartSymbol = "__start_llgo_funcinfo_stubsite" funcInfoStubSiteEndSymbol = "__stop_llgo_funcinfo_stubsite" pcSiteStartSymbol = "__start_llgo_pcline" @@ -274,6 +278,10 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord i32Type, i32Type, }, false) + funcEntryRecordType := llvmCtx.StructType([]llvm.Type{ + llvm.PointerType(i8Type, 0), + i64Type, + }, false) stubSiteRecordType := llvmCtx.StructType([]llvm.Type{ llvm.PointerType(i8Type, 0), i64Type, @@ -287,6 +295,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcLinePtr := llvm.AddGlobal(mod, llvm.PointerType(pcLineRecordType, 0), pcLineTableSymbol) pcSiteStartPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteStartPtrSymbol) pcSiteEndPtr := llvm.AddGlobal(mod, llvm.PointerType(pcSiteRecordType, 0), pcSiteEndPtrSymbol) + entryStartPtr := llvm.AddGlobal(mod, llvm.PointerType(funcEntryRecordType, 0), funcInfoEntryStartPtrSymbol) + entryEndPtr := llvm.AddGlobal(mod, llvm.PointerType(funcEntryRecordType, 0), funcInfoEntryEndPtrSymbol) stubSiteStartPtr := llvm.AddGlobal(mod, llvm.PointerType(stubSiteRecordType, 0), funcInfoStubSiteStartPtrSymbol) stubSiteEndPtr := llvm.AddGlobal(mod, llvm.PointerType(stubSiteRecordType, 0), funcInfoStubSiteEndPtrSymbol) stringsPtr := llvm.AddGlobal(mod, llvm.PointerType(i8Type, 0), funcInfoStringsSymbol) @@ -303,6 +313,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + entryStartPtr.SetInitializer(llvm.ConstPointerNull(entryStartPtr.GlobalValueType())) + entryEndPtr.SetInitializer(llvm.ConstPointerNull(entryEndPtr.GlobalValueType())) stubSiteStartPtr.SetInitializer(llvm.ConstPointerNull(stubSiteStartPtr.GlobalValueType())) stubSiteEndPtr.SetInitializer(llvm.ConstPointerNull(stubSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) @@ -326,6 +338,8 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord pcLinePtr.SetInitializer(llvm.ConstPointerNull(pcLinePtr.GlobalValueType())) pcSiteStartPtr.SetInitializer(llvm.ConstPointerNull(pcSiteStartPtr.GlobalValueType())) pcSiteEndPtr.SetInitializer(llvm.ConstPointerNull(pcSiteEndPtr.GlobalValueType())) + entryStartPtr.SetInitializer(llvm.ConstPointerNull(entryStartPtr.GlobalValueType())) + entryEndPtr.SetInitializer(llvm.ConstPointerNull(entryEndPtr.GlobalValueType())) stubSiteStartPtr.SetInitializer(llvm.ConstPointerNull(stubSiteStartPtr.GlobalValueType())) stubSiteEndPtr.SetInitializer(llvm.ConstPointerNull(stubSiteEndPtr.GlobalValueType())) stringsPtr.SetInitializer(llvm.ConstPointerNull(stringsPtr.GlobalValueType())) @@ -398,8 +412,18 @@ func emitFuncInfoTable(ctx *context, pkg llssa.Package, records []funcInfoRecord } } emitELFSites := shouldEmitRuntimeELFSites(ctx) + emitEntrySites := shouldEmitRuntimeEntryELFSites(ctx) && len(encoded.Records) != 0 emitStubSites := shouldEmitRuntimeStubELFSites(ctx) - emitRuntimeFuncInfoELFSites(mod, ctx.prog.PointerSize(), emitELFSites && len(pcLineValues) != 0, emitStubSites && len(stubRecords) != 0) + emitRuntimeFuncInfoELFSites(mod, ctx.prog.PointerSize(), emitELFSites && len(pcLineValues) != 0, emitEntrySites, emitStubSites && len(stubRecords) != 0) + if emitEntrySites { + entryStart := llvm.AddGlobal(mod, funcEntryRecordType, funcInfoEntryStartSymbol) + entryEnd := llvm.AddGlobal(mod, funcEntryRecordType, funcInfoEntryEndSymbol) + entryStartPtr.SetInitializer(entryStart) + entryEndPtr.SetInitializer(entryEnd) + } else { + entryStartPtr.SetInitializer(llvm.ConstPointerNull(entryStartPtr.GlobalValueType())) + entryEndPtr.SetInitializer(llvm.ConstPointerNull(entryEndPtr.GlobalValueType())) + } if emitStubSites && len(stubRecords) != 0 { stubSiteStart := llvm.AddGlobal(mod, stubSiteRecordType, funcInfoStubSiteStartSymbol) stubSiteEnd := llvm.AddGlobal(mod, stubSiteRecordType, funcInfoStubSiteEndSymbol) @@ -505,7 +529,70 @@ func shouldEmitRuntimeELFSites(ctx *context) bool { } func shouldEmitRuntimeStubELFSites(ctx *context) bool { - return shouldEmitRuntimeELFSites(ctx) && !ctx.buildConf.ltoEnabled() + return shouldEmitRuntimeELFSites(ctx) +} + +func shouldEmitRuntimeEntryELFSites(ctx *context) bool { + return shouldEmitRuntimeELFSites(ctx) +} + +func emitFuncInfoEntrySites(ctx *context, pkg llssa.Package) { + if !shouldEmitRuntimeEntryELFSites(ctx) || pkg == nil || !ctx.prog.FuncInfoMetadataEnabled() { + return + } + mod := pkg.Module() + records := readFuncInfo(mod) + if len(records) == 0 { + return + } + symbolIDs := make(map[string]uint64, len(records)) + for _, rec := range records { + if rec.symbol != "" { + symbolIDs[rec.symbol] = funcInfoSymbolID(rec.symbol) + } + } + if len(symbolIDs) == 0 { + return + } + llvmCtx := mod.Context() + builder := llvmCtx.NewBuilder() + defer builder.Dispose() + asmType := llvm.FunctionType(llvmCtx.VoidType(), nil, false) + ptrDirective := ".quad" + align := "3" + if ctx.prog.PointerSize() == 4 { + ptrDirective = ".long" + align = "2" + } + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if fn.IsDeclaration() || fn.BasicBlocksCount() == 0 { + continue + } + symbol := fn.Name() + symbolID := symbolIDs[symbol] + if symbolID == 0 { + continue + } + entry := fn.EntryBasicBlock() + if entry.IsNil() { + continue + } + first := entry.FirstInstruction() + if first.IsNil() { + builder.SetInsertPointAtEnd(entry) + } else { + builder.SetInsertPointBefore(first) + } + anchor := ".Lllgo_funcinfo_entry_anchor_${:uid}" + instruction := anchor + ":\n" + + ".pushsection llgo_funcinfo_entry,\"ao\",@progbits," + anchor + "\n" + + ".p2align " + align + "\n" + + ptrDirective + " " + anchor + "\n" + + ".quad " + uint64Hex(symbolID) + "\n" + + ".popsection" + asm := llvm.InlineAsm(asmType, instruction, "", true, false, llvm.InlineAsmDialectATT, false) + builder.CreateCall(asmType, asm, nil, "") + } } func emitFuncInfoStubSites(ctx *context, pkg llssa.Package) { @@ -542,9 +629,11 @@ func emitFuncInfoStubSites(ctx *context, pkg llssa.Package) { } else { builder.SetInsertPointBefore(first) } - instruction := ".pushsection llgo_funcinfo_stubsite,\"ao\",@progbits," + asmQuoteELFSymbol(symbol) + "\n" + + anchor := ".Lllgo_funcinfo_stubsite_anchor_${:uid}" + instruction := anchor + ":\n" + + ".pushsection llgo_funcinfo_stubsite,\"ao\",@progbits," + anchor + "\n" + ".p2align " + align + "\n" + - ptrDirective + " " + asmQuoteELFSymbol(symbol) + "\n" + + ptrDirective + " " + anchor + "\n" + ".quad " + uint64Hex(funcInfoSymbolID(target)) + "\n" + ".popsection" asm := llvm.InlineAsm(asmType, instruction, "", true, false, llvm.InlineAsmDialectATT, false) @@ -580,8 +669,8 @@ func uint64Hex(v uint64) string { return string(buf[:]) } -func emitRuntimeFuncInfoELFSites(mod llvm.Module, pointerSize int, pcSite bool, stubSite bool) { - if !pcSite && !stubSite { +func emitRuntimeFuncInfoELFSites(mod llvm.Module, pointerSize int, pcSite bool, entrySite bool, stubSite bool) { + if !pcSite && !entrySite && !stubSite { return } ptrDirective := ".quad" @@ -597,6 +686,12 @@ func emitRuntimeFuncInfoELFSites(mod llvm.Module, pointerSize int, pcSite bool, asm.WriteString(ptrDirective + " 0\n") asm.WriteString(".quad 0\n") } + if entrySite { + asm.WriteString(".section llgo_funcinfo_entry,\"aR\",@progbits\n") + asm.WriteString(".p2align " + align + "\n") + asm.WriteString(ptrDirective + " 0\n") + asm.WriteString(".quad 0\n") + } if stubSite { asm.WriteString(".section llgo_funcinfo_stubsite,\"aR\",@progbits\n") asm.WriteString(".p2align " + align + "\n") diff --git a/internal/build/funcinfo_table_test.go b/internal/build/funcinfo_table_test.go index 9ed6739cad..1c7e3cb3aa 100644 --- a/internal/build/funcinfo_table_test.go +++ b/internal/build/funcinfo_table_test.go @@ -64,10 +64,13 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) "@__llgo_funcinfo_string_count = global i64 5", "@__llgo_funcinfo_hash = global ptr", "@__llgo_funcinfo_count = global i64 1", + "@__llgo_funcinfo_entry_start = global ptr @__start_llgo_funcinfo_entry", + "@__llgo_funcinfo_entry_end = global ptr @__stop_llgo_funcinfo_entry", "@__llgo_funcinfo_stub_indexes = global ptr null", "@__llgo_funcinfo_stub_count = global i64 0", "@__llgo_pcline_count = global i64 0", "@__llgo_funcinfo_hash_mask = global i64 1", + "module asm \".section llgo_funcinfo_entry", `@"__llgo_funcinfo_table$data" = private unnamed_addr constant [1 x { i16, i16, i16, i16, i16, i16, i32 }]`, `@"__llgo_funcinfo_string_offsets$data" = private unnamed_addr constant`, `@"__llgo_funcinfo_hash$data" = private unnamed_addr constant [2 x i16]`, @@ -86,6 +89,88 @@ func TestFuncInfoTableMaterializesMetadataWithoutFunctionPointers(t *testing.T) } } +func TestFuncInfoTableMaterializesEntrySites(t *testing.T) { + prog := llssa.NewProgram(nil) + src := prog.NewPackage("example.com/p", "example.com/p") + src.EmitFuncInfo("example.com/p.live", "example.com/p.Live", "live.go", 17, 3) + src.EmitFuncInfo("example.com/p.missing", "example.com/p.Missing", "missing.go", 19, 1) + liveFn := src.NewFunc("example.com/p.live", llssa.NoArgsNoRet, llssa.InC) + liveFn.MakeBody(1).Return() + otherFn := src.NewFunc("example.com/p.other", llssa.NoArgsNoRet, llssa.InC) + otherFn.MakeBody(1).Return() + ctx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + }, + } + prog.EnableFuncInfoMetadata(true) + emitFuncInfoEntrySites(ctx, src) + srcIR := src.String() + for _, want := range []string{ + "call void asm sideeffect", + ".pushsection llgo_funcinfo_entry", + ".Lllgo_funcinfo_entry_anchor_", + ".quad .Lllgo_funcinfo_entry_anchor_", + ".quad 0x", + } { + if !strings.Contains(srcIR, want) { + t.Fatalf("package entry site IR missing %q:\n%s", want, srcIR) + } + } + for _, bad := range []string{ + `.quad \22example.com/p.live\22`, + `.quad \22example.com/p.other\22`, + `.quad \22example.com/p.missing\22`, + } { + if strings.Contains(srcIR, bad) { + t.Fatalf("package entry site IR should not contain %q:\n%s", bad, srcIR) + } + } + + records := collectFuncInfo([]Package{{LPkg: src}}) + entry := genMainModule(ctx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records}) + ir := entry.LPkg.String() + for _, want := range []string{ + "@__llgo_funcinfo_entry_start = global ptr @__start_llgo_funcinfo_entry", + "@__llgo_funcinfo_entry_end = global ptr @__stop_llgo_funcinfo_entry", + "module asm \".section llgo_funcinfo_entry", + } { + if !strings.Contains(ir, want) { + t.Fatalf("funcinfo entry table IR missing %q:\n%s", want, ir) + } + } + + ltoCtx := &context{ + prog: prog, + buildConf: &Config{ + BuildMode: BuildModeExe, + Goos: "linux", + Goarch: "amd64", + LTO: lto.Full, + }, + } + ltoEntry := genMainModule(ltoCtx, llssa.PkgRuntime, &packages.Package{ + PkgPath: "example.com/main", + ExportFile: "main.a", + }, &genConfig{funcInfo: records}) + ltoIR := ltoEntry.LPkg.String() + for _, want := range []string{ + "@__llgo_funcinfo_entry_start = global ptr @__start_llgo_funcinfo_entry", + "@__llgo_funcinfo_entry_end = global ptr @__stop_llgo_funcinfo_entry", + "module asm \".section llgo_funcinfo_entry", + } { + if !strings.Contains(ltoIR, want) { + t.Fatalf("full LTO funcinfo table IR missing %q:\n%s", want, ltoIR) + } + } +} + func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { prog := llssa.NewProgram(nil) src := prog.NewPackage("example.com/p", "example.com/p") @@ -107,13 +192,17 @@ func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { for _, want := range []string{ "call void asm sideeffect", ".pushsection llgo_funcinfo_stubsite", - `.quad \22__llgo_stub.example.com/p.live\22`, + ".Lllgo_funcinfo_stubsite_anchor_", + ".quad .Lllgo_funcinfo_stubsite_anchor_", ".quad 0x", } { if !strings.Contains(srcIR, want) { t.Fatalf("package stub site IR missing %q:\n%s", want, srcIR) } } + if strings.Contains(srcIR, `.quad \22__llgo_stub.example.com/p.live\22`) { + t.Fatalf("package stub site must not reference stub function symbols:\n%s", srcIR) + } records := collectFuncInfo([]Package{{LPkg: src}}) stubs := collectFuncInfoStubRecords([]Package{{LPkg: src}}, records) @@ -159,13 +248,13 @@ func TestFuncInfoTableMaterializesClosureStubIndexes(t *testing.T) { ExportFile: "main.a", }, &genConfig{funcInfo: records, funcInfoStubs: stubs}) ltoIR := ltoEntry.LPkg.String() - for _, bad := range []string{ + for _, want := range []string{ "@__llgo_funcinfo_stubsite_start = global ptr @__start_llgo_funcinfo_stubsite", "@__llgo_funcinfo_stubsite_end = global ptr @__stop_llgo_funcinfo_stubsite", "module asm \".section llgo_funcinfo_stubsite", } { - if strings.Contains(ltoIR, bad) { - t.Fatalf("full LTO funcinfo table should not emit stub site %q:\n%s", bad, ltoIR) + if !strings.Contains(ltoIR, want) { + t.Fatalf("full LTO funcinfo stub site table IR missing %q:\n%s", want, ltoIR) } } } @@ -291,6 +380,8 @@ func TestFuncInfoTableEmptyDefinitions(t *testing.T) { "@__llgo_funcinfo_string_count = global i64 0", "@__llgo_funcinfo_hash = global ptr null", "@__llgo_funcinfo_count = global i64 0", + "@__llgo_funcinfo_entry_start = global ptr null", + "@__llgo_funcinfo_entry_end = global ptr null", "@__llgo_funcinfo_stub_indexes = global ptr null", "@__llgo_funcinfo_stub_count = global i64 0", "@__llgo_pcline_count = global i64 0", diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index a0bae4d160..4548771399 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -120,6 +120,22 @@ func funcForPCSlow(pc uintptr) *Func { cacheFuncForPC(pc, fn) return fn } + } else if pc != 0 { + // Function-value PCs point at the real function entry. ELF funcinfo + // entry-site anchors are emitted from LLVM IR and can land after the + // backend prologue, so an exact entry PC may sort before its anchor. + // Prefer native symbol info only when it is an exact entry match; the + // section table below remains the normal fast fallback. + if sym := addrInfoSymbol(pc); sym.ok && sym.entry == pc && sym.function != "" { + fn := newFuncForPC(pc, sym) + cacheFuncForPC(pc, fn) + return fn + } + if sym, ok := funcPCFrameForEntryPC(pc); ok { + fn := newFuncForPC(pc, sym) + cacheFuncForPC(pc, fn) + return fn + } } if sym, ok := funcPCFrameForPC(pc); ok { fn := newFuncForPC(pc, sym) diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index bdfebb167d..098c42fbef 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -172,6 +172,17 @@ var runtimeFuncInfoStubIndexes *uint32 //go:linkname runtimeFuncInfoStubCount __llgo_funcinfo_stub_count var runtimeFuncInfoStubCount uintptr +type runtimeFuncInfoEntryRecord struct { + pc uintptr + symbolID uint64 +} + +//go:linkname runtimeFuncInfoEntryStart __llgo_funcinfo_entry_start +var runtimeFuncInfoEntryStart *runtimeFuncInfoEntryRecord + +//go:linkname runtimeFuncInfoEntryEnd __llgo_funcinfo_entry_end +var runtimeFuncInfoEntryEnd *runtimeFuncInfoEntryRecord + type runtimeFuncInfoStubSiteRecord struct { pc uintptr symbolID uint64 @@ -230,6 +241,7 @@ type runtimePCPageIndex struct { } const runtimeFuncPCPageShift = 12 +const runtimeFuncPCEntrySlack = 64 var runtimeFuncPCInitState uint32 var runtimeFuncPCFrames []runtimeFuncPCFrame @@ -654,28 +666,39 @@ func initRuntimeFuncPCFramesOnce() { } frames := make([]runtimeFuncPCFrame, 0, runtimeFuncInfoCount) entries := make([]uintptr, runtimeFuncInfoCount+1) - symbolBuf := make([]byte, 0, maxFuncInfoSymbolLen()+len(runtimeClosureStubPrefix)+1) - for i := uintptr(0); i < runtimeFuncInfoCount; i++ { - fn := funcInfoAt(i) - pc := symbolPCFuncInfoName(symbolBuf, fn.symbolPkg, fn.symbolName) - if pc == 0 { - continue - } - index := uint32(i + 1) - frames = append(frames, runtimeFuncPCFrame{ - entry: pc, - funcIndex: index, - }) - if entries[index] == 0 || pc < entries[index] { - entries[index] = pc + var indexBySymbolID map[uint64]uint32 + if runtimeFuncInfoEntryStart != nil || runtimeFuncInfoStubSiteStart != nil { + indexBySymbolID = funcInfoIndexBySymbolID() + } + frames, usedEntrySites := appendRuntimeFuncInfoEntryFrames(frames, entries, indexBySymbolID) + symbolBuf := []byte(nil) + if !usedEntrySites { + symbolBuf = make([]byte, 0, maxFuncInfoSymbolLen()+len(runtimeClosureStubPrefix)+1) + for i := uintptr(0); i < runtimeFuncInfoCount; i++ { + fn := funcInfoAt(i) + pc := symbolPCFuncInfoName(symbolBuf, fn.symbolPkg, fn.symbolName) + if pc == 0 { + continue + } + index := uint32(i + 1) + frames = append(frames, runtimeFuncPCFrame{ + entry: pc, + funcIndex: index, + }) + if entries[index] == 0 || pc < entries[index] { + entries[index] = pc + } } } - frames = appendRuntimeFuncInfoStubSiteFrames(frames) + frames = appendRuntimeFuncInfoStubSiteFrames(frames, indexBySymbolID) // Closure stubs are an ABI adapter and may go away in a future closure // lowering. Keep the fallback compatibility table light: it stores only // target funcinfo record indexes. On ELF we prefer the associated stub-site // section above because linkers do not expose local stubs through dlsym. if runtimeFuncInfoStubIndexes != nil && runtimeFuncInfoStubCount != 0 && runtimeFuncInfoStubCount <= runtimeFuncInfoCount { + if symbolBuf == nil { + symbolBuf = make([]byte, 0, maxFuncInfoSymbolLen()+len(runtimeClosureStubPrefix)+1) + } for i := uintptr(0); i < runtimeFuncInfoStubCount; i++ { index := funcInfoStubIndexAt(i) if index == 0 || uintptr(index) > runtimeFuncInfoCount { @@ -699,7 +722,43 @@ func initRuntimeFuncPCFramesOnce() { runtimeFuncPCIndex = buildRuntimeFuncPCIndex(frames) } -func appendRuntimeFuncInfoStubSiteFrames(frames []runtimeFuncPCFrame) []runtimeFuncPCFrame { +func appendRuntimeFuncInfoEntryFrames(frames []runtimeFuncPCFrame, entries []uintptr, indexBySymbolID map[uint64]uint32) ([]runtimeFuncPCFrame, bool) { + if runtimeFuncInfoEntryStart == nil || runtimeFuncInfoEntryEnd == nil { + return frames, false + } + start := uintptr(unsafe.Pointer(runtimeFuncInfoEntryStart)) + end := uintptr(unsafe.Pointer(runtimeFuncInfoEntryEnd)) + size := unsafe.Sizeof(*runtimeFuncInfoEntryStart) + if end <= start || size == 0 || (end-start)%size != 0 { + return frames, false + } + nsite := (end - start) / size + if nsite > runtimeFuncInfoCount*16 || nsite > 1<<20 { + return frames, false + } + used := false + for i := uintptr(0); i < nsite; i++ { + site := (*runtimeFuncInfoEntryRecord)(unsafe.Pointer(start + i*size)) + if site == nil || site.pc == 0 || site.symbolID == 0 { + continue + } + funcIndex := indexBySymbolID[site.symbolID] + if funcIndex == 0 || uintptr(funcIndex) > runtimeFuncInfoCount { + continue + } + frames = append(frames, runtimeFuncPCFrame{ + entry: site.pc, + funcIndex: funcIndex, + }) + if entries[funcIndex] == 0 || site.pc < entries[funcIndex] { + entries[funcIndex] = site.pc + } + used = true + } + return frames, used +} + +func appendRuntimeFuncInfoStubSiteFrames(frames []runtimeFuncPCFrame, indexBySymbolID map[uint64]uint32) []runtimeFuncPCFrame { if runtimeFuncInfoStubSiteStart == nil || runtimeFuncInfoStubSiteEnd == nil { return frames } @@ -718,7 +777,7 @@ func appendRuntimeFuncInfoStubSiteFrames(frames []runtimeFuncPCFrame) []runtimeF if site == nil || site.pc == 0 || site.symbolID == 0 { continue } - funcIndex := funcInfoIndexForSymbolID(site.symbolID) + funcIndex := indexBySymbolID[site.symbolID] if funcIndex == 0 || uintptr(funcIndex) > runtimeFuncInfoCount { continue } @@ -730,17 +789,21 @@ func appendRuntimeFuncInfoStubSiteFrames(frames []runtimeFuncPCFrame) []runtimeF return frames } -func funcInfoIndexForSymbolID(id uint64) uint32 { - if id == 0 || runtimeFuncInfoTable == nil || runtimeFuncInfoCount == 0 { - return 0 - } +func funcInfoIndexBySymbolID() map[uint64]uint32 { + indexBySymbolID := make(map[uint64]uint32, runtimeFuncInfoCount) for i := uintptr(0); i < runtimeFuncInfoCount; i++ { - rec := funcInfoAt(i) - if funcInfoSymbolIDFromRecord(rec) == id { - return uint32(i + 1) + id := funcInfoSymbolIDFromRecord(funcInfoAt(i)) + if id == 0 { + continue } + index := uint32(i + 1) + if prev, ok := indexBySymbolID[id]; ok && prev != index { + indexBySymbolID[id] = 0 + continue + } + indexBySymbolID[id] = index } - return 0 + return indexBySymbolID } func funcInfoSymbolIDFromRecord(rec *runtimeFuncInfoRecord) uint64 { @@ -930,14 +993,46 @@ func funcPCFrameForPC(pc uintptr) (pcSymbol, bool) { return pcSymbol{}, false } frame := runtimeFuncPCFrames[idx] - if frame.funcIndex == 0 || uintptr(frame.funcIndex) > runtimeFuncInfoCount { + return pcSymbolForFuncInfoIndex(pc, frame.entry, frame.funcIndex) +} + +func funcPCFrameForEntryPC(pc uintptr) (pcSymbol, bool) { + if pc == 0 { + return pcSymbol{}, false + } + initRuntimeFuncPCFrames() + frames := runtimeFuncPCFrames + if len(frames) == 0 { return pcSymbol{}, false } - fn := funcInfoAt(uintptr(frame.funcIndex) - 1) + lo, hi := 0, len(frames) + for lo < hi { + mid := int(uint(lo+hi) >> 1) + if frames[mid].entry >= pc { + hi = mid + } else { + lo = mid + 1 + } + } + if lo >= len(frames) { + return pcSymbol{}, false + } + frame := frames[lo] + if frame.entry != pc && frame.entry-pc > runtimeFuncPCEntrySlack { + return pcSymbol{}, false + } + return pcSymbolForFuncInfoIndex(pc, pc, frame.funcIndex) +} + +func pcSymbolForFuncInfoIndex(pc, entry uintptr, funcIndex uint32) (pcSymbol, bool) { + if funcIndex == 0 || uintptr(funcIndex) > runtimeFuncInfoCount { + return pcSymbol{}, false + } + fn := funcInfoAt(uintptr(funcIndex) - 1) line := int(fn.line) return pcSymbol{ pc: pc, - entry: frame.entry, + entry: entry, function: funcInfoFunctionName(fn), file: funcInfoFileName(fn), line: line, From 1f48fc322fe8a52c4be65bfc4da1eebf10691f85 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 22:01:41 +0800 Subject: [PATCH 20/23] test: cover pcline metadata in dev lto coverage --- ssa/ssa_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ssa/ssa_test.go b/ssa/ssa_test.go index 1adb41cf88..3e81d24010 100644 --- a/ssa/ssa_test.go +++ b/ssa/ssa_test.go @@ -205,6 +205,12 @@ func TestFuncInfoMetadataDoesNotPreserveFunctions(t *testing.T) { } func TestPCLineMetadataEmission(t *testing.T) { + testPCLineMetadataEmission(t) +} + +func testPCLineMetadataEmission(t *testing.T) { + t.Helper() + prog := NewProgram(nil) pkg := prog.NewPackage("main", "main") @@ -332,6 +338,11 @@ func TestDevLTOGlobalDCEFuncInfoMetadata(t *testing.T) { testFuncInfoMetadataDoesNotBlockGlobalDCE(t) } +func TestDevLTOGlobalDCEPCLineMetadata(t *testing.T) { + requireGoGlobalDCE(t) + testPCLineMetadataEmission(t) +} + func requireGoGlobalDCE(t *testing.T) { t.Helper() } From f8cf01ea45f1852f833bf57d55a6b2b2121b29fb Mon Sep 17 00:00:00 2001 From: Li Jie Date: Wed, 1 Jul 2026 23:21:41 +0800 Subject: [PATCH 21/23] runtime: speed up funcinfo hot paths --- cl/caller_frame_test.go | 47 ++++++++++- cl/compile.go | 4 +- cl/instr.go | 29 ++++++- runtime/internal/lib/runtime/symtab.go | 105 +++++++++++++++++++------ 4 files changed, 154 insertions(+), 31 deletions(-) diff --git a/cl/caller_frame_test.go b/cl/caller_frame_test.go index b2a63fe47b..e5c9fd7f0a 100644 --- a/cl/caller_frame_test.go +++ b/cl/caller_frame_test.go @@ -61,10 +61,18 @@ func f() { _ = dbg.Stack() } name: "dot import", src: `package foo import . "runtime" -func f() { _ = FuncForPC(0) } +func f() { Caller(0) } `, want: true, }, + { + name: "runtime FuncForPC only", + src: `package foo +import "runtime" +func f() { _ = runtime.FuncForPC(0) } +`, + want: false, + }, { name: "blank import", src: `package foo @@ -178,7 +186,8 @@ func interfaceCaller(c callerIface) { interfaceDispatch(c) } func closureLayer(next func()) func() { return func() { next() } } func closureCaller() { closureLayer(closureLayer(direct))() } func stack() { _ = debug.Stack() } -func anonOnly() { func() { runtime.FuncForPC(0) }() } +func anonOnly() { func() { runtime.Caller(0) }() } +func funcForPCOnly() { _ = runtime.FuncForPC(0) } func leaf() {} func callFunc(f func()) { f() } func callFuncHot() { callFunc(leaf) } @@ -216,6 +225,9 @@ func plain() {} t.Fatalf("%s should not be tracked when resolved dynamic targets do not reach runtime stack APIs", name) } } + if runtimeCallerFuncs[ssapkg.Func("funcForPCOnly")] { + t.Fatal("FuncForPC-only function should not need caller frame tracking") + } if runtimeCallerFuncs[ssapkg.Func("plain")] { t.Fatal("plain function should not be tracked") } @@ -225,6 +237,12 @@ func plain() {} t.Fatalf("%s should be a runtime caller metadata function", name) } } + if isRuntimeCallerFrameName("FuncForPC") { + t.Fatal("FuncForPC should not require caller frame tracking") + } + if !isRuntimeCallerFrameName("Caller") { + t.Fatal("Caller should require caller frame tracking") + } if isRuntimeCallerName("Version") { t.Fatal("Version should not be a runtime caller metadata function") } @@ -239,6 +257,9 @@ func FuncForPC(pc uintptr) uintptr { return 0 } if !isRuntimeCallerFunc(rtpkg.Func("FuncForPC")) { t.Fatal("LLGo runtime lib FuncForPC should be treated as runtime metadata use") } + if isRuntimeCallerFrameFunc(rtpkg.Func("FuncForPC")) { + t.Fatal("FuncForPC should not require caller frame tracking") + } if isRuntimeCallerLookupFunc(rtpkg.Func("FuncForPC")) { t.Fatal("FuncForPC should not consume caller lookup tokens") } @@ -706,6 +727,28 @@ func f() {} if ir := pkg.Module().String(); strings.Contains(ir, "RecordCallerLocation") || strings.Contains(ir, "RecordPanicLocation") { t.Fatalf("packages without runtime stack APIs should not emit caller location tracking:\n%s", ir) } + + ssapkg, files = buildCallerFrameSSAPackage(t, "example.com/foo", `package foo +import "runtime" +func f() { _ = runtime.FuncForPC(0) } +`) + prog = newLLSSAProg(t) + prog.Target().GOOS = "linux" + prog.Target().GOARCH = "amd64" + prog.EnableFuncInfoMetadata(true) + pkg, err = NewPackage(prog, ssapkg, files) + if err != nil { + t.Fatal(err) + } + ir := pkg.Module().String() + for _, bad := range []string{"RecordCallerLocation", "RecordPanicLocation", "PushCallerLocationFrame", `!llgo.pcline`} { + if strings.Contains(ir, bad) { + t.Fatalf("FuncForPC-only packages should not emit caller frame tracking %q:\n%s", bad, ir) + } + } + if !strings.Contains(ir, `!llgo.funcinfo = !{!`) { + t.Fatalf("FuncForPC-only packages should still emit funcinfo metadata:\n%s", ir) + } } func TestCompileRuntimeCallerLocationOnlyForRuntimePaths(t *testing.T) { diff --git a/cl/compile.go b/cl/compile.go index f3d2c21338..ff3f31689f 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -266,7 +266,7 @@ func filesUseRuntimeCaller(files []*ast.File) bool { return false } case *ast.Ident: - if (dotImports["runtime"] && isRuntimeCallerName(n.Name)) || + if (dotImports["runtime"] && isRuntimeCallerFrameName(n.Name)) || (dotImports["runtime/debug"] && n.Name == "Stack") { found = true return false @@ -284,7 +284,7 @@ func filesUseRuntimeCaller(files []*ast.File) bool { func runtimeCallerSelector(path, name string) bool { switch path { case "runtime": - return isRuntimeCallerName(name) + return isRuntimeCallerFrameName(name) case "runtime/debug": return name == "Stack" default: diff --git a/cl/instr.go b/cl/instr.go index 6db43beaea..136b514a47 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -1040,7 +1040,7 @@ func fnHasDirectRuntimeCaller(fn *ssa.Function) bool { if !ok { continue } - if isRuntimeCallerFunc(call.Common().StaticCallee()) { + if isRuntimeCallerFrameFunc(call.Common().StaticCallee()) { return true } } @@ -1057,7 +1057,7 @@ func (a *runtimeCallerAnalysis) fnMayReachRuntimeCaller(fn *ssa.Function) bool { if fn == nil { return false } - if isRuntimeCallerFunc(fn) { + if isRuntimeCallerFrameFunc(fn) { return true } if !a.funcs[fn] { @@ -1078,7 +1078,7 @@ func (a *runtimeCallerAnalysis) fnMayReachRuntimeCaller(fn *ssa.Function) bool { } callee := call.StaticCallee() switch { - case isRuntimeCallerFunc(callee): + case isRuntimeCallerFrameFunc(callee): reaches = true case callee != nil: reaches = a.fnMayReachRuntimeCaller(callee) @@ -1276,6 +1276,20 @@ func isRuntimeCallerFunc(fn *ssa.Function) bool { } } +func isRuntimeCallerFrameFunc(fn *ssa.Function) bool { + if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { + return false + } + switch fn.Pkg.Pkg.Path() { + case "runtime", "github.com/goplus/llgo/runtime/internal/lib/runtime": + return isRuntimeCallerFrameName(fn.Name()) + case "runtime/debug": + return fn.Name() == "Stack" + default: + return false + } +} + func isRuntimeCallerLookupFunc(fn *ssa.Function) bool { if fn == nil || fn.Pkg == nil || fn.Pkg.Pkg == nil { return false @@ -1301,6 +1315,15 @@ func isRuntimeCallerName(name string) bool { } } +func isRuntimeCallerFrameName(name string) bool { + switch name { + case "Caller", "Callers", "CallersFrames", "Stack": + return true + default: + return false + } +} + func (p *context) runtimeCallerFrameName() string { if p == nil { return "" diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index 098c42fbef..8328262902 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -229,6 +229,7 @@ type runtimePCLineFrame struct { var runtimePCLineInitState uint32 var runtimePCLineFrames []runtimePCLineFrame +var runtimePCLineIndex runtimePCPageIndex type runtimeFuncPCFrame struct { entry uintptr @@ -1132,7 +1133,9 @@ func initRuntimePCLineFramesOnce() { }) } sortRuntimePCLineFrames(frames) - runtimePCLineFrames = uniqueRuntimePCLineFrames(frames) + frames = uniqueRuntimePCLineFrames(frames) + runtimePCLineFrames = frames + runtimePCLineIndex = buildRuntimePCLineIndex(frames) } func pcLineInfoForID(id uint64) *runtimePCLineRecord { @@ -1234,28 +1237,93 @@ func uniqueRuntimePCLineFrames(frames []runtimePCLineFrame) []runtimePCLineFrame return out } -func pcLineFrameForPC(pc, entry uintptr) (pcSymbol, bool) { - if pc == 0 { - return pcSymbol{}, false +func buildRuntimePCLineIndex(frames []runtimePCLineFrame) runtimePCPageIndex { + if len(frames) == 0 { + return runtimePCPageIndex{} } - initRuntimePCLineFrames() + base := frames[0].pc >> runtimeFuncPCPageShift + last := frames[len(frames)-1].pc >> runtimeFuncPCPageShift + if last < base { + return runtimePCPageIndex{} + } + npages := last - base + 2 + if npages > 1<<20 && npages > uintptr(len(frames))*64 { + return runtimePCPageIndex{} + } + pages := make([]uint32, npages) + next := 0 + for page := range pages { + limit := (base + uintptr(page)) << runtimeFuncPCPageShift + for next < len(frames) && frames[next].pc < limit { + next++ + } + pages[page] = uint32(next) + } + return runtimePCPageIndex{base: base, pages: pages} +} + +func runtimePCLineFrameRange(pc uintptr) (int, int) { + frames := runtimePCLineFrames + lo, hi := 0, len(frames) + if pages := runtimePCLineIndex.pages; len(pages) != 0 { + page := pc >> runtimeFuncPCPageShift + if page >= runtimePCLineIndex.base { + off := page - runtimePCLineIndex.base + if off < uintptr(len(pages)) { + lo = int(pages[off]) + if off+1 < uintptr(len(pages)) { + hi = int(pages[off+1]) + } + if lo > 0 { + lo-- + } + if hi < len(frames) { + hi++ + } + } + } + } + return lo, hi +} + +func runtimePCLineFrameIndex(pc uintptr, exact bool) int { frames := runtimePCLineFrames if len(frames) == 0 { - return pcSymbol{}, false + return -1 } - lo, hi := 0, len(frames) + lo, hi := runtimePCLineFrameRange(pc) for lo < hi { mid := int(uint(lo+hi) >> 1) - if frames[mid].pc > pc { + if frames[mid].pc > pc || (exact && frames[mid].pc == pc) { hi = mid } else { lo = mid + 1 } } - if lo == 0 { + if exact { + if lo >= len(frames) || frames[lo].pc != pc { + return -1 + } + return lo + } + idx := lo - 1 + if idx < 0 { + return -1 + } + return idx +} + +func pcLineFrameForPC(pc, entry uintptr) (pcSymbol, bool) { + if pc == 0 { + return pcSymbol{}, false + } + initRuntimePCLineFrames() + frames := runtimePCLineFrames + idx := runtimePCLineFrameIndex(pc, false) + if idx < 0 { return pcSymbol{}, false } - frame := frames[lo-1] + frame := frames[idx] if entry != 0 && frame.entry != 0 && frame.entry != entry { return pcSymbol{}, false } @@ -1276,22 +1344,11 @@ func pcLineFrameForExactPC(pc uintptr) (pcSymbol, bool) { } initRuntimePCLineFrames() frames := runtimePCLineFrames - if len(frames) == 0 { - return pcSymbol{}, false - } - lo, hi := 0, len(frames) - for lo < hi { - mid := int(uint(lo+hi) >> 1) - if frames[mid].pc >= pc { - hi = mid - } else { - lo = mid + 1 - } - } - if lo >= len(frames) || frames[lo].pc != pc { + idx := runtimePCLineFrameIndex(pc, true) + if idx < 0 { return pcSymbol{}, false } - frame := frames[lo] + frame := frames[idx] return pcSymbol{ pc: pc, entry: frame.entry, From 1f26921bfeed21610ea733bb6780dcf0a584ecf5 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 2 Jul 2026 01:51:55 +0800 Subject: [PATCH 22/23] benchmark: add runtime funcinfo comparison runner --- benchmark/runtime_funcinfo/.gitignore | 1 + benchmark/runtime_funcinfo/README.md | 52 ++ benchmark/runtime_funcinfo/main.go | 1130 +++++++++++++++++++++++++ 3 files changed, 1183 insertions(+) create mode 100644 benchmark/runtime_funcinfo/.gitignore create mode 100644 benchmark/runtime_funcinfo/README.md create mode 100644 benchmark/runtime_funcinfo/main.go diff --git a/benchmark/runtime_funcinfo/.gitignore b/benchmark/runtime_funcinfo/.gitignore new file mode 100644 index 0000000000..89f9ac04aa --- /dev/null +++ b/benchmark/runtime_funcinfo/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/benchmark/runtime_funcinfo/README.md b/benchmark/runtime_funcinfo/README.md new file mode 100644 index 0000000000..d06338a96b --- /dev/null +++ b/benchmark/runtime_funcinfo/README.md @@ -0,0 +1,52 @@ +# Runtime Funcinfo Benchmark + +This benchmark keeps runtime funcinfo measurements comparable across branches by +generating the same probe programs and rebuilding them with each compiler/root +pair in one run. + +It covers: + +- hot runtime metadata calls: `Caller`, `Callers`, `CallersFrames`, + `FuncForPC`, and `Func.FileLine`. +- deep stacks through direct calls, interface calls, and closures. +- many packages and methods, generated from configurable package/method counts. +- a stdlib-heavy program with `encoding/json`, `text/template`, `regexp`, + `go/parser`, `go/token`, and `net/netip` imports. + +Generated modules use `example.com/llgo-bench/...` import paths. This is +intentional: LLGo does not enable caller-frame tracking for stdlib-shaped paths +without a dot, and that would benchmark the fallback path instead of normal +third-party package behavior. + +Example: + +```sh +go run ./benchmark/runtime_funcinfo \ + -runs=11 \ + -llgo-opt=2 \ + -variant go=go \ + -variant main=llgo,/path/to/llgo-main,/path/to/llgo-main-root \ + -variant 2002=llgo,/path/to/llgo-2002,/path/to/llgo-2002-root \ + -variant 2009=llgo,/path/to/llgo-2009,/path/to/llgo-2009-root \ + -variant 2010=llgo,/path/to/llgo-2010,/path/to/llgo-2010-root +``` + +Add `-include-lto` to build an additional `+lto` variant for every LLGo +compiler. LLGo builds use `-O2` by default; pass `-llgo-opt=` to omit the +optimization flag. Output is written to `benchmark/runtime_funcinfo/out` by +default: + +- `summary.md`: markdown performance and size tables. +- `results.json`: raw build and run data. +- `work/`: generated probe modules. +- `bin/`: generated executables. + +Performance cells are `best/trimmed avg` from process-level runs. The trimmed +average drops one minimum and one maximum when at least three runs are present. +`-iters` is a base iteration count: `hot` uses the full count, `deep` uses a +quarter, and `multipkg`/`stdlib` use one twentieth because each operation does +substantially more work. + +`multipkg.FuncForPCMany` and `multipkg.FileLineMany` are batch metrics over all +generated target functions (`-packages * -methods`, 144 targets with the default +settings), not single-lookup timings. diff --git a/benchmark/runtime_funcinfo/main.go b/benchmark/runtime_funcinfo/main.go new file mode 100644 index 0000000000..48ef24d93a --- /dev/null +++ b/benchmark/runtime_funcinfo/main.go @@ -0,0 +1,1130 @@ +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type variantFlag []string + +func (v *variantFlag) String() string { + return strings.Join(*v, ";") +} + +func (v *variantFlag) Set(s string) error { + *v = append(*v, s) + return nil +} + +type variant struct { + Name string `json:"name"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Root string `json:"root,omitempty"` + LTO bool `json:"lto,omitempty"` +} + +type scenario struct { + Name string + Dir string +} + +type buildResult struct { + Variant string `json:"variant"` + Scenario string `json:"scenario"` + Binary string `json:"binary"` + Size int64 `json:"size_bytes"` + BuildMS int64 `json:"build_ms"` + Error string `json:"error,omitempty"` +} + +type runResult struct { + Variant string `json:"variant"` + Scenario string `json:"scenario"` + Metrics map[string][]int64 `json:"metrics_ns"` + Error string `json:"error,omitempty"` + Output string `json:"output,omitempty"` + Env map[string]string `json:"env,omitempty"` +} + +type resultFile struct { + GeneratedAt time.Time `json:"generated_at"` + PackageCount int `json:"package_count"` + MethodCount int `json:"method_count"` + Variants []variant `json:"variants"` + Scenarios []string `json:"scenarios"` + Builds []buildResult `json:"builds"` + Runs []runResult `json:"runs"` +} + +func main() { + var variants variantFlag + outDir := flag.String("out", filepath.Join("benchmark", "runtime_funcinfo", "out"), "output directory") + runs := flag.Int("runs", 11, "process runs per executable") + iters := flag.Int("iters", 200000, "inner benchmark iterations") + llgoOpt := flag.String("llgo-opt", "2", "LLGo optimization level passed as -O; empty disables the flag") + scenarioList := flag.String("scenarios", "hot,deep,multipkg,stdlib", "comma-separated scenarios") + includeLTO := flag.Bool("include-lto", false, "also build full-LTO variants for LLGo compilers") + pkgCount := flag.Int("packages", 12, "generated package count for multipkg") + methodCount := flag.Int("methods", 12, "generated functions and methods per generated package") + flag.Var(&variants, "variant", "variant definition: name=go or name=llgo,/path/to/llgo,/path/to/root") + flag.Parse() + + if len(variants) == 0 { + variants = append(variants, "go=go") + } + parsed, err := parseVariants(variants, *includeLTO) + if err != nil { + fatal(err) + } + if *runs <= 0 { + fatal(errors.New("-runs must be positive")) + } + if *iters <= 0 { + fatal(errors.New("-iters must be positive")) + } + if *pkgCount <= 0 || *methodCount <= 0 { + fatal(errors.New("-packages and -methods must be positive")) + } + + absOut, err := filepath.Abs(*outDir) + if err != nil { + fatal(err) + } + if err := os.RemoveAll(absOut); err != nil { + fatal(err) + } + for _, dir := range []string{"work", "bin"} { + if err := os.MkdirAll(filepath.Join(absOut, dir), 0755); err != nil { + fatal(err) + } + } + + scenarios, err := generateScenarios(filepath.Join(absOut, "work"), splitList(*scenarioList), *pkgCount, *methodCount) + if err != nil { + fatal(err) + } + + var builds []buildResult + var runsOut []runResult + for _, sc := range scenarios { + for _, v := range parsed { + br := buildScenario(absOut, sc, v, *llgoOpt) + builds = append(builds, br) + if br.Error != "" { + fmt.Fprintf(os.Stderr, "build failed: %s/%s: %s\n", v.Name, sc.Name, br.Error) + continue + } + rr := runScenario(sc, v, br.Binary, *runs, *iters) + runsOut = append(runsOut, rr) + if rr.Error != "" { + fmt.Fprintf(os.Stderr, "run failed: %s/%s: %s\n", v.Name, sc.Name, rr.Error) + } + } + } + + result := resultFile{ + GeneratedAt: time.Now(), + PackageCount: *pkgCount, + MethodCount: *methodCount, + Variants: parsed, + Scenarios: scenarioNames(scenarios), + Builds: builds, + Runs: runsOut, + } + if err := writeJSON(filepath.Join(absOut, "results.json"), result); err != nil { + fatal(err) + } + summary := renderSummary(result) + if err := os.WriteFile(filepath.Join(absOut, "summary.md"), []byte(summary), 0644); err != nil { + fatal(err) + } + fmt.Print(summary) +} + +func parseVariants(values []string, includeLTO bool) ([]variant, error) { + var out []variant + seen := map[string]bool{} + for _, raw := range values { + name, spec, ok := strings.Cut(raw, "=") + if !ok || name == "" || spec == "" { + return nil, fmt.Errorf("bad -variant %q", raw) + } + if seen[name] { + return nil, fmt.Errorf("duplicate variant %q", name) + } + seen[name] = true + var v variant + v.Name = name + switch { + case spec == "go": + v.Kind = "go" + v.Tool = "go" + case strings.HasPrefix(spec, "go,"): + parts := strings.Split(spec, ",") + if len(parts) != 2 || parts[1] == "" { + return nil, fmt.Errorf("bad go variant %q", raw) + } + v.Kind = "go" + v.Tool = parts[1] + case strings.HasPrefix(spec, "llgo,"): + parts := strings.Split(spec, ",") + if len(parts) != 3 || parts[1] == "" || parts[2] == "" { + return nil, fmt.Errorf("bad llgo variant %q", raw) + } + v.Kind = "llgo" + v.Tool = parts[1] + v.Root = parts[2] + default: + return nil, fmt.Errorf("unknown variant kind in %q", raw) + } + out = append(out, v) + if includeLTO && v.Kind == "llgo" { + lto := v + lto.Name = v.Name + "+lto" + lto.LTO = true + out = append(out, lto) + } + } + return out, nil +} + +func generateScenarios(workDir string, names []string, pkgCount, methodCount int) ([]scenario, error) { + var out []scenario + for _, name := range names { + dir := filepath.Join(workDir, name) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, err + } + var err error + switch name { + case "hot": + err = generateHot(dir) + case "deep": + err = generateDeep(dir) + case "multipkg": + err = generateMultipkg(dir, pkgCount, methodCount) + case "stdlib": + err = generateStdlib(dir) + default: + return nil, fmt.Errorf("unknown scenario %q", name) + } + if err != nil { + return nil, err + } + out = append(out, scenario{Name: name, Dir: dir}) + } + return out, nil +} + +func writeModule(dir, module string) error { + return os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module "+module+"\n\ngo 1.24\n"), 0644) +} + +func generateHot(dir string) error { + if err := writeModule(dir, "example.com/llgo-bench/hot"); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "main.go"), []byte(hotSource), 0644) +} + +func generateDeep(dir string) error { + if err := writeModule(dir, "example.com/llgo-bench/deep"); err != nil { + return err + } + var b strings.Builder + b.WriteString(deepPrefix) + for i := 0; i < 32; i++ { + fmt.Fprintf(&b, "//go:noinline\nfunc frame%d() { frame%d() }\n\n", i, i+1) + } + b.WriteString(`//go:noinline +func frame32() { + pc, file, line, ok := runtime.Caller(16) + if !ok || pc == 0 || file == "" || line == 0 { + panic("bad deep caller") + } + sinkPC = pc + sinkString = file + sinkInt += line +} + +`) + b.WriteString(deepSuffix) + return os.WriteFile(filepath.Join(dir, "main.go"), []byte(b.String()), 0644) +} + +func generateMultipkg(dir string, pkgCount, methodCount int) error { + if err := writeModule(dir, "example.com/llgo-bench/multipkg"); err != nil { + return err + } + for i := 0; i < pkgCount; i++ { + pkgName := fmt.Sprintf("p%02d", i) + pkgDir := filepath.Join(dir, pkgName) + if err := os.MkdirAll(pkgDir, 0755); err != nil { + return err + } + var b strings.Builder + fmt.Fprintf(&b, "package %s\n\n", pkgName) + b.WriteString("import (\n\t\"reflect\"\n\t\"runtime\"\n") + if i+1 < pkgCount { + fmt.Fprintf(&b, "\tnext \"example.com/llgo-bench/multipkg/p%02d\"\n", i+1) + } + b.WriteString(")\n\n") + fmt.Fprintf(&b, "type T%02d struct { V int }\n", i) + b.WriteString("type Worker interface { M00(int) int }\n\n") + for j := 0; j < methodCount; j++ { + fmt.Fprintf(&b, "//go:noinline\nfunc F%02d_%02d(x int) int { return x + %d }\n\n", i, j, i*100+j) + fmt.Fprintf(&b, "//go:noinline\nfunc (t T%02d) M%02d(x int) int { return t.V + x + %d }\n\n", i, j, j) + } + b.WriteString("//go:noinline\nfunc Targets() []uintptr {\n\treturn []uintptr{\n") + for j := 0; j < methodCount; j++ { + fmt.Fprintf(&b, "\t\treflect.ValueOf(F%02d_%02d).Pointer(),\n", i, j) + } + b.WriteString("\t}\n}\n\n") + b.WriteString("//go:noinline\nfunc Run(x int) int {\n") + b.WriteString("\tpc, _, line, ok := runtime.Caller(0)\n\tif !ok || pc == 0 || line == 0 { panic(\"bad caller\") }\n") + fmt.Fprintf(&b, "\tvar w Worker = T%02d{V: x}\n", i) + b.WriteString("\ttotal := w.M00(x)\n") + for j := 0; j < methodCount; j++ { + fmt.Fprintf(&b, "\ttotal += F%02d_%02d(x)\n", i, j) + fmt.Fprintf(&b, "\ttotal += (T%02d{V: total}).M%02d(x)\n", i, j) + } + if i+1 < pkgCount { + b.WriteString("\ttotal += next.Run(x+1)\n") + } + b.WriteString("\treturn total + line\n}\n") + if err := os.WriteFile(filepath.Join(pkgDir, pkgName+".go"), []byte(b.String()), 0644); err != nil { + return err + } + } + + var main strings.Builder + main.WriteString("package main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\t\"time\"\n") + for i := 0; i < pkgCount; i++ { + fmt.Fprintf(&main, "\tp%02d \"example.com/llgo-bench/multipkg/p%02d\"\n", i, i) + } + main.WriteString(")\n\nvar sinkInt int\nvar sinkString string\n\n") + main.WriteString(commonBenchHelpers) + main.WriteString("func main() {\n\titers := benchIters(10000)\n\tvar targets []uintptr\n") + for i := 0; i < pkgCount; i++ { + fmt.Fprintf(&main, "\ttargets = append(targets, p%02d.Targets()...)\n", i) + } + main.WriteString(` + if funcInfoReady(targets) { + measure("multipkg.FuncForPCMany", iters, func() { + total := 0 + for _, pc := range targets { + fn := runtime.FuncForPC(pc) + if fn == nil { + panic("missing func") + } + total += len(fn.Name()) + } + sinkInt += total + }) + measure("multipkg.FileLineMany", iters, func() { + total := 0 + for _, pc := range targets { + fn := runtime.FuncForPC(pc) + if fn == nil { + panic("missing func") + } + file, line := fn.FileLine(pc) + if file == "" || line == 0 { + panic("missing fileline") + } + total += line + len(file) + } + sinkInt += total + }) + } + measure("multipkg.DeepRun", iters, func() { + sinkInt += p00.Run(1) + }) + fmt.Println("sink=", sinkInt, sinkString) +} + +func funcInfoReady(targets []uintptr) bool { + for _, pc := range targets { + fn := runtime.FuncForPC(pc) + if fn == nil { + return false + } + if file, line := fn.FileLine(pc); file == "" || line == 0 { + return false + } + } + return len(targets) != 0 +} +`) + return os.WriteFile(filepath.Join(dir, "main.go"), []byte(main.String()), 0644) +} + +func generateStdlib(dir string) error { + if err := writeModule(dir, "example.com/llgo-bench/stdlib"); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "main.go"), []byte(stdlibSource), 0644) +} + +func buildScenario(outDir string, sc scenario, v variant, llgoOpt string) buildResult { + bin := filepath.Join(outDir, "bin", safeName(v.Name)+"_"+sc.Name) + if v.LTO { + bin += "_lto" + } + if exeSuffix := executableSuffix(); exeSuffix != "" { + bin += exeSuffix + } + start := time.Now() + var cmd *exec.Cmd + switch v.Kind { + case "go": + cmd = exec.Command(v.Tool, "build", "-trimpath", "-o", bin, ".") + case "llgo": + args := []string{"build", "-trimpath", "-a", "-o", bin} + if llgoOpt != "" { + args = append(args, "-O"+llgoOpt) + } + if v.LTO { + args = append(args, "-lto=full") + } + args = append(args, ".") + cmd = exec.Command(v.Tool, args...) + default: + return buildResult{Variant: v.Name, Scenario: sc.Name, Binary: bin, Error: "unknown variant kind"} + } + cmd.Dir = sc.Dir + cmd.Env = os.Environ() + if v.Kind == "llgo" { + cmd.Env = append(cmd.Env, "LLGO_ROOT="+v.Root, "LLGO_FUNCINFO=1") + } + out, err := cmd.CombinedOutput() + br := buildResult{Variant: v.Name, Scenario: sc.Name, Binary: bin, BuildMS: time.Since(start).Milliseconds()} + if err != nil { + br.Error = strings.TrimSpace(string(out)) + if br.Error == "" { + br.Error = err.Error() + } + return br + } + info, err := os.Stat(bin) + if err != nil { + br.Error = err.Error() + return br + } + br.Size = info.Size() + return br +} + +func runScenario(sc scenario, v variant, bin string, runs, iters int) runResult { + scenarioIters := iterationsForScenario(sc.Name, iters) + rr := runResult{ + Variant: v.Name, + Scenario: sc.Name, + Metrics: map[string][]int64{}, + Env: map[string]string{ + "BENCH_ITERS": strconv.Itoa(scenarioIters), + }, + } + for i := 0; i < runs; i++ { + cmd := exec.Command(bin) + cmd.Dir = sc.Dir + cmd.Env = append(os.Environ(), "BENCH_ITERS="+strconv.Itoa(scenarioIters)) + out, err := cmd.CombinedOutput() + if err != nil { + rr.Error = err.Error() + rr.Output = string(out) + return rr + } + metrics, err := parseMetrics(out) + if err != nil { + rr.Error = err.Error() + rr.Output = string(out) + return rr + } + for k, v := range metrics { + rr.Metrics[k] = append(rr.Metrics[k], v) + } + } + return rr +} + +func iterationsForScenario(name string, base int) int { + div := 1 + switch name { + case "deep": + div = 4 + case "multipkg", "stdlib": + div = 20 + } + n := base / div + if n < 1 { + return 1 + } + return n +} + +func parseMetrics(out []byte) (map[string]int64, error) { + metrics := map[string]int64{} + for _, raw := range strings.Split(string(out), "\n") { + line := strings.TrimSpace(raw) + if line == "" || !strings.Contains(line, "=") { + continue + } + name, value, _ := strings.Cut(line, "=") + if !strings.Contains(name, ".") { + continue + } + n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return nil, fmt.Errorf("parse metric %q: %w", line, err) + } + metrics[strings.TrimSpace(name)] = n + } + return metrics, nil +} + +func renderSummary(result resultFile) string { + var b strings.Builder + fmt.Fprintf(&b, "# Runtime Funcinfo Benchmark\n\nGenerated: `%s`\n\n", result.GeneratedAt.Format(time.RFC3339)) + b.WriteString("Cells are `best/trimmed avg`. Runtime metrics use `ns/op`; sizes use MiB.\n\n") + if result.PackageCount > 0 && result.MethodCount > 0 { + fmt.Fprintf(&b, "`multipkg.FuncForPCMany` and `multipkg.FileLineMany` are batch metrics over %d target functions (%d packages x %d functions).\n\n", + result.PackageCount*result.MethodCount, result.PackageCount, result.MethodCount) + } + for _, sc := range result.Scenarios { + metrics := metricsForScenario(result.Runs, sc) + if len(metrics) == 0 { + continue + } + fmt.Fprintf(&b, "## %s Performance\n\n", sc) + b.WriteString("| metric |") + for _, v := range result.Variants { + b.WriteString(" " + v.Name + " |") + } + b.WriteString("\n|---|") + for range result.Variants { + b.WriteString("---:|") + } + b.WriteString("\n") + for _, metric := range metrics { + b.WriteString("| " + metric + " |") + for _, v := range result.Variants { + rr, found := findRun(result.Runs, v.Name, sc) + cell := "FAIL" + if found && rr.Error == "" { + cell = "n/a" + } + if vals := rr.Metrics[metric]; len(vals) != 0 { + cell = formatPerf(vals) + } + b.WriteString(" " + cell + " |") + } + b.WriteString("\n") + } + b.WriteString("\n") + } + b.WriteString("## Binary Size\n\n| scenario |") + for _, v := range result.Variants { + b.WriteString(" " + v.Name + " |") + } + b.WriteString("\n|---|") + for range result.Variants { + b.WriteString("---:|") + } + b.WriteString("\n") + for _, sc := range result.Scenarios { + b.WriteString("| " + sc + " |") + for _, v := range result.Variants { + cell := "FAIL" + if br := findBuild(result.Builds, v.Name, sc); br.Error == "" && br.Size > 0 { + cell = formatMiB(br.Size) + } + b.WriteString(" " + cell + " |") + } + b.WriteString("\n") + } + b.WriteString("\n## Build Time\n\n| scenario |") + for _, v := range result.Variants { + b.WriteString(" " + v.Name + " |") + } + b.WriteString("\n|---|") + for range result.Variants { + b.WriteString("---:|") + } + b.WriteString("\n") + for _, sc := range result.Scenarios { + b.WriteString("| " + sc + " |") + for _, v := range result.Variants { + cell := "FAIL" + if br := findBuild(result.Builds, v.Name, sc); br.Error == "" { + cell = formatDurationMS(br.BuildMS) + } + b.WriteString(" " + cell + " |") + } + b.WriteString("\n") + } + return b.String() +} + +func metricsForScenario(runs []runResult, scenario string) []string { + set := map[string]bool{} + for _, rr := range runs { + if rr.Scenario != scenario || rr.Error != "" { + continue + } + for k := range rr.Metrics { + set[k] = true + } + } + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func findRun(runs []runResult, variant, scenario string) (runResult, bool) { + for _, rr := range runs { + if rr.Variant == variant && rr.Scenario == scenario { + return rr, true + } + } + return runResult{Metrics: map[string][]int64{}}, false +} + +func findBuild(builds []buildResult, variant, scenario string) buildResult { + for _, br := range builds { + if br.Variant == variant && br.Scenario == scenario { + return br + } + } + return buildResult{Error: "missing"} +} + +func formatPerf(values []int64) string { + if len(values) == 0 { + return "n/a" + } + sorted := append([]int64(nil), values...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + best := sorted[0] + avgVals := sorted + if len(sorted) >= 3 { + avgVals = sorted[1 : len(sorted)-1] + } + var sum int64 + for _, v := range avgVals { + sum += v + } + avg := float64(sum) / float64(len(avgVals)) + return formatNS(float64(best)) + "/" + formatNS(avg) +} + +func formatNS(ns float64) string { + switch { + case ns >= 1e6: + return trimFloat(ns/1e6) + "ms" + case ns >= 1e3: + return trimFloat(ns/1e3) + "us" + default: + return trimFloat(ns) + "ns" + } +} + +func formatMiB(bytes int64) string { + return trimFloat(float64(bytes)/(1024*1024)) + " MiB" +} + +func formatDurationMS(ms int64) string { + if ms >= 1000 { + return trimFloat(float64(ms)/1000) + "s" + } + return strconv.FormatInt(ms, 10) + "ms" +} + +func trimFloat(v float64) string { + if math.Abs(v-math.Round(v)) < 0.05 { + return strconv.FormatInt(int64(math.Round(v)), 10) + } + return strconv.FormatFloat(v, 'f', 1, 64) +} + +func writeJSON(path string, data any) error { + raw, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + raw = append(raw, '\n') + return os.WriteFile(path, raw, 0644) +} + +func splitList(s string) []string { + var out []string + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +func scenarioNames(scenarios []scenario) []string { + out := make([]string, len(scenarios)) + for i, sc := range scenarios { + out[i] = sc.Name + } + return out +} + +func safeName(s string) string { + replacer := strings.NewReplacer("/", "_", "\\", "_", ":", "_", "+", "_") + return replacer.Replace(s) +} + +func executableSuffix() string { + if os.PathSeparator == '\\' { + return ".exe" + } + return "" +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} + +const commonBenchHelpers = ` +func benchIters(def int) int { + if s := getenv("BENCH_ITERS"); s != "" { + n, err := atoi(s) + if err == nil && n > 0 { + return n + } + } + return def +} + +func measure(name string, n int, fn func()) { + fn() + start := time.Now() + for i := 0; i < n; i++ { + fn() + } + elapsed := time.Since(start).Nanoseconds() + if n <= 0 { + panic("bad iterations") + } + fmt.Printf("%s=%d\n", name, elapsed/int64(n)) +} + +func getenv(k string) string { + for _, kv := range os.Environ() { + if len(kv) > len(k) && kv[:len(k)] == k && kv[len(k)] == '=' { + return kv[len(k)+1:] + } + } + return "" +} + +func atoi(s string) (int, error) { + n := 0 + for _, r := range s { + if r < '0' || r > '9' { + return 0, fmt.Errorf("bad int") + } + n = n*10 + int(r-'0') + } + return n, nil +} +` + +const hotSource = `package main + +import ( + "fmt" + "os" + "reflect" + "runtime" + "time" +) + +var sinkInt int +var sinkPC uintptr +var sinkString string + +` + commonBenchHelpers + ` + +//go:noinline +func entryTarget(x int) int { + return x + 7 +} + +//go:noinline +func caller0() { + pc, file, line, ok := runtime.Caller(0) + if !ok || pc == 0 || file == "" || line == 0 { + panic("bad caller0") + } + sinkPC = pc + sinkString = file + sinkInt += line +} + +//go:noinline +func caller1() { + caller1Helper() +} + +//go:noinline +func caller1Helper() { + pc, file, line, ok := runtime.Caller(1) + if !ok || pc == 0 || file == "" || line == 0 { + panic("bad caller1") + } + sinkPC = pc + sinkString = file + sinkInt += line +} + +//go:noinline +func returnPC() uintptr { + pc, _, _, ok := runtime.Caller(0) + if !ok || pc == 0 { + panic("bad return pc") + } + return pc +} + +//go:noinline +func callersOnly() { + var pcs [16]uintptr + n := runtime.Callers(0, pcs[:]) + if n == 0 || pcs[0] == 0 { + panic("bad callers") + } + sinkPC = pcs[0] + sinkInt += n +} + +//go:noinline +func callersFramesFirst() { + var pcs [16]uintptr + n := runtime.Callers(0, pcs[:]) + frames := runtime.CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if frame.Function != "" && frame.File != "" && frame.Line != 0 { + sinkString = frame.Function + sinkInt += frame.Line + return + } + if !more { + break + } + } + panic("bad frame") +} + +func callersFramesReady() (ok bool) { + defer func() { + if recover() != nil { + ok = false + } + }() + callersFramesFirst() + return true +} + +func main() { + iters := benchIters(200000) + entryPC := reflect.ValueOf(entryTarget).Pointer() + returnedPC := returnPC() + measure("hot.Caller0", iters, caller0) + measure("hot.Caller1", iters, caller1) + measure("hot.CallersOnly", iters, callersOnly) + if callersFramesReady() { + measure("hot.CallersFramesFirst", iters, callersFramesFirst) + } + if entryFn := runtime.FuncForPC(entryPC); entryFn != nil && entryFn.Name() != "" { + measure("hot.FuncForPCEntry", iters, func() { + fn := runtime.FuncForPC(entryPC) + if fn == nil { + panic("missing entry func") + } + sinkString = fn.Name() + }) + if file, line := entryFn.FileLine(entryPC); file != "" && line != 0 { + measure("hot.FuncFileLineEntry", iters, func() { + file, line := entryFn.FileLine(entryPC) + if file == "" || line == 0 { + panic("missing entry fileline") + } + sinkString = file + sinkInt += line + }) + } + } + if returnFn := runtime.FuncForPC(returnedPC); returnFn != nil && returnFn.Name() != "" { + measure("hot.FuncForPCReturnPC", iters, func() { + fn := runtime.FuncForPC(returnedPC) + if fn == nil { + panic("missing return func") + } + sinkString = fn.Name() + }) + if file, line := returnFn.FileLine(returnedPC); file != "" && line != 0 { + measure("hot.FuncFileLineReturnPC", iters, func() { + file, line := returnFn.FileLine(returnedPC) + if file == "" || line == 0 { + panic("missing return fileline") + } + sinkString = file + sinkInt += line + }) + } + } + fmt.Println("sink=", sinkInt, sinkPC, sinkString) +} +` + +const deepPrefix = `package main + +import ( + "fmt" + "os" + "runtime" + "time" +) + +var sinkInt int +var sinkPC uintptr +var sinkString string + +` + commonBenchHelpers + ` + +type callerIface interface { + call() +} + +type callerImpl struct{} + +//go:noinline +func (callerImpl) call() { + frame0() +} + +//go:noinline +func closureLayer(next func()) func() { + return func() { + next() + } +} + +//go:noinline +func callInterface(c callerIface) { + c.call() +} + +//go:noinline +func callClosure() { + closureLayer(closureLayer(frame0))() +} + +` + +const deepSuffix = `//go:noinline +func framesAll() { + frame0() + var pcs [64]uintptr + n := runtime.Callers(0, pcs[:]) + frames := runtime.CallersFrames(pcs[:n]) + total := 0 + for { + frame, more := frames.Next() + if frame.Function != "" { + total += len(frame.Function) + frame.Line + } + if !more { + break + } + } + if total == 0 { + panic("bad frames") + } + sinkInt += total +} + +func deepReady(fn func()) (ok bool) { + defer func() { + if recover() != nil { + ok = false + } + }() + fn() + return true +} + +func main() { + iters := benchIters(50000) + if deepReady(frame0) { + measure("deep.Direct32", iters, frame0) + } + if deepReady(func() { callInterface(callerImpl{}) }) { + measure("deep.Interface32", iters, func() { callInterface(callerImpl{}) }) + } + if deepReady(callClosure) { + measure("deep.Closure32", iters, callClosure) + } + if deepReady(framesAll) { + measure("deep.CallersFramesAll", iters, framesAll) + } + fmt.Println("sink=", sinkInt, sinkPC, sinkString) +} +` + +const stdlibSource = `package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/parser" + "go/token" + "net/netip" + "os" + "reflect" + "regexp" + "runtime" + "strings" + "text/template" + "time" +) + +var sinkInt int +var sinkString string + +` + commonBenchHelpers + ` + +type payload struct { + Name string + Items []int + Addr string +} + +//go:noinline +func stdTarget(x int) int { + return x*3 + 1 +} + +//go:noinline +func stdWork() { + p := payload{Name: "llgo", Items: []int{1, 2, 3, 5, 8}, Addr: "127.0.0.1:8080"} + raw, err := json.Marshal(p) + if err != nil { + panic(err) + } + var out payload + if err := json.Unmarshal(raw, &out); err != nil { + panic(err) + } + tmpl := template.Must(template.New("x").Funcs(template.FuncMap{"join": strings.Join}).Parse("{{.Name}}:{{join .Words \",\"}}")) + var buf bytes.Buffer + if err := tmpl.Execute(&buf, map[string]any{"Name": out.Name, "Words": []string{"a", "b", "c"}}); err != nil { + panic(err) + } + re := regexp.MustCompile("[a-z]+") + matches := re.FindAllString(buf.String(), -1) + expr, err := parser.ParseExpr("1 + 2*3") + if err != nil || expr == nil { + panic("bad parser") + } + fs := token.NewFileSet() + file := fs.AddFile("bench.go", -1, 100) + file.AddLine(10) + addr := netip.MustParseAddrPort(out.Addr) + sinkInt += len(matches) + int(addr.Port()) + int(file.Line(token.Pos(11))) + sinkString = buf.String() +} + +//go:noinline +func stdCaller() { + pc, file, line, ok := runtime.Caller(0) + if !ok || pc == 0 || file == "" || line == 0 { + panic("bad caller") + } + sinkInt += line + sinkString = file +} + +//go:noinline +func stdFrames() { + var pcs [16]uintptr + n := runtime.Callers(0, pcs[:]) + frames := runtime.CallersFrames(pcs[:n]) + for { + frame, more := frames.Next() + if frame.Function != "" && frame.File != "" && frame.Line != 0 { + sinkInt += frame.Line + sinkString = frame.Function + return + } + if !more { + break + } + } + panic("bad frame") +} + +func stdFramesReady() (ok bool) { + defer func() { + if recover() != nil { + ok = false + } + }() + stdFrames() + return true +} + +func main() { + iters := benchIters(50000) + entryPC := reflect.ValueOf(stdTarget).Pointer() + measure("stdlib.Work", iters/10, stdWork) + measure("stdlib.Caller0", iters, stdCaller) + if stdFramesReady() { + measure("stdlib.CallersFramesFirst", iters, stdFrames) + } + if fn := runtime.FuncForPC(entryPC); fn != nil && fn.Name() != "" { + measure("stdlib.FuncForPCEntry", iters, func() { + fn := runtime.FuncForPC(entryPC) + if fn == nil { + panic("missing func") + } + sinkString = fn.Name() + }) + if file, line := fn.FileLine(entryPC); file != "" && line != 0 { + measure("stdlib.FuncFileLineEntry", iters, func() { + file, line := fn.FileLine(entryPC) + if file == "" || line == 0 { + panic("missing fileline") + } + sinkInt += line + sinkString = file + }) + } + } + fmt.Println("sink=", sinkInt, sinkString) +} +` From 89837e447a60d6d034e1dec426e8adb8117c5c39 Mon Sep 17 00:00:00 2001 From: Li Jie Date: Thu, 2 Jul 2026 02:20:39 +0800 Subject: [PATCH 23/23] runtime: avoid FuncForPC cache thrashing --- .../lib/runtime/pprof_runtime_stub_llgo.go | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go index 4548771399..efdc55c49c 100644 --- a/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go +++ b/runtime/internal/lib/runtime/pprof_runtime_stub_llgo.go @@ -2,11 +2,7 @@ package runtime -import ( - "unsafe" - - llrt "github.com/goplus/llgo/runtime/internal/runtime" -) +import llrt "github.com/goplus/llgo/runtime/internal/runtime" type StackRecord struct { Stack []uintptr @@ -88,27 +84,28 @@ func NumGoroutine() int { func SetCPUProfileRate(hz int) {} -const funcForPCCacheSize = 1024 +const funcForPCCacheSets = 1024 +const funcForPCCacheWays = 4 type funcForPCCacheEntry struct { pc uintptr fn *Func } -var funcForPCCache [funcForPCCacheSize]funcForPCCacheEntry +var funcForPCCache [funcForPCCacheSets][funcForPCCacheWays]funcForPCCacheEntry +var funcForPCCacheNext [funcForPCCacheSets]uint8 var funcForPCLast funcForPCCacheEntry func FuncForPC(pc uintptr) *Func { if fn := funcForPCLast.fn; fn != nil && funcForPCLast.pc == pc { return fn } - entry := (*funcForPCCacheEntry)(unsafe.Add( - unsafe.Pointer(&funcForPCCache[0]), - funcForPCCacheIndex(pc)*unsafe.Sizeof(funcForPCCacheEntry{}), - )) - if fn := entry.fn; fn != nil && entry.pc == pc { - funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} - return fn + set := &funcForPCCache[funcForPCCacheIndex(pc)] + for i := 0; i < funcForPCCacheWays; i++ { + if fn := set[i].fn; fn != nil && set[i].pc == pc { + funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} + return fn + } } return funcForPCSlow(pc) } @@ -124,14 +121,14 @@ func funcForPCSlow(pc uintptr) *Func { // Function-value PCs point at the real function entry. ELF funcinfo // entry-site anchors are emitted from LLVM IR and can land after the // backend prologue, so an exact entry PC may sort before its anchor. - // Prefer native symbol info only when it is an exact entry match; the - // section table below remains the normal fast fallback. - if sym := addrInfoSymbol(pc); sym.ok && sym.entry == pc && sym.function != "" { + // Prefer the section table when it can match within the entry slack; + // native symbol lookup is kept only as a fallback. + if sym, ok := funcPCFrameForEntryPC(pc); ok { fn := newFuncForPC(pc, sym) cacheFuncForPC(pc, fn) return fn } - if sym, ok := funcPCFrameForEntryPC(pc); ok { + if sym := addrInfoSymbol(pc); sym.ok && sym.entry == pc && sym.function != "" { fn := newFuncForPC(pc, sym) cacheFuncForPC(pc, fn) return fn @@ -170,15 +167,21 @@ func newFuncForPC(pc uintptr, sym pcSymbol) *Func { } func cacheFuncForPC(pc uintptr, fn *Func) { - entry := (*funcForPCCacheEntry)(unsafe.Add( - unsafe.Pointer(&funcForPCCache[0]), - funcForPCCacheIndex(pc)*unsafe.Sizeof(funcForPCCacheEntry{}), - )) - entry.fn = fn - entry.pc = pc - funcForPCLast = funcForPCCacheEntry{pc: pc, fn: fn} + setIndex := funcForPCCacheIndex(pc) + set := &funcForPCCache[setIndex] + for i := 0; i < funcForPCCacheWays; i++ { + if set[i].fn == nil || set[i].pc == pc { + set[i] = funcForPCCacheEntry{pc: pc, fn: fn} + funcForPCLast = set[i] + return + } + } + way := funcForPCCacheNext[setIndex] & (funcForPCCacheWays - 1) + funcForPCCacheNext[setIndex] = way + 1 + set[way] = funcForPCCacheEntry{pc: pc, fn: fn} + funcForPCLast = set[way] } func funcForPCCacheIndex(pc uintptr) uintptr { - return (pc >> 4) & (funcForPCCacheSize - 1) + return (pc >> 4) & (funcForPCCacheSets - 1) }