diff --git a/_pydemo/callpy/callpy b/_pydemo/callpy/callpy new file mode 100755 index 0000000000..dd9c3a45b0 Binary files /dev/null and b/_pydemo/callpy/callpy differ diff --git a/go.mod b/go.mod index c0be0ef720..004c236d27 100644 --- a/go.mod +++ b/go.mod @@ -21,3 +21,5 @@ require ( ) replace github.com/goplus/llgo/runtime => ./runtime + +replace github.com/goplus/llgo/runtimeext/pyenvrt => ./runtimeext/pyenvrt diff --git a/internal/build/build.go b/internal/build/build.go index 4c37457cc0..e4a6b349a3 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -43,6 +43,7 @@ import ( "github.com/goplus/llgo/internal/env" "github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/packages" + "github.com/goplus/llgo/internal/pyenv" "github.com/goplus/llgo/internal/typepatch" "github.com/goplus/llgo/ssa/abi" "github.com/goplus/llgo/xtool/clang" @@ -160,7 +161,6 @@ func Do(args []string, conf *Config) ([]Package, error) { if err != nil { return nil, fmt.Errorf("failed to setup crosscompile: %w", err) } - // Update GOOS/GOARCH from export if target was used if conf.Target != "" && export.GOOS != "" { conf.Goos = export.GOOS @@ -251,6 +251,14 @@ func Do(args []string, conf *Config) ([]Package, error) { altPkgs, err := packages.LoadEx(dedup, sizes, cfg, altPkgPaths...) check(err) + rootDir := filepath.Dir(env.LLGoRuntimeDir()) + cfgExt := *cfg + cfgExt.Dir = rootDir + altExtPkgs, err := packages.LoadEx(dedup, sizes, &cfgExt, "github.com/goplus/llgo/runtimeext/pyenvrt") + check(err) + + altPkgs = append(altPkgs, altExtPkgs...) + noRt := 1 prog.SetRuntime(func() *types.Package { noRt = 0 @@ -286,6 +294,7 @@ func Do(args []string, conf *Config) ([]Package, error) { isCheckLinkArgsEnabled: IsCheckLinkArgsEnabled(), cTransformer: cabi.NewTransformer(prog, conf.AbiMode), } + pkgs, err := buildAllPkgs(ctx, initial, verbose) check(err) if mode == ModeGen { @@ -308,7 +317,6 @@ func Do(args []string, conf *Config) ([]Package, error) { global, err := createGlobals(ctx, ctx.prog, pkgs) check(err) - for _, pkg := range initial { if needLink(pkg, mode) { linkMainPkg(ctx, pkg, allPkgs, global, conf, mode, verbose) @@ -449,6 +457,11 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs expdArgs := make([]string, 0, len(altParts)) for _, param := range altParts { param = strings.TrimSpace(param) + if param == "$LLGO_LIB_PYTHON" { + _ = pyenv.EnsureWithFetch("") + _ = pyenv.EnsureBuildEnv() + _ = pyenv.Verify() + } if strings.ContainsRune(param, '$') { expdArgs = append(expdArgs, xenv.ExpandEnvToArgs(param)...) ctx.nLibdir++ @@ -476,6 +489,7 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs ctx.nLibdir++ } } + if ctx.isCheckLinkArgsEnabled { if err := ctx.compiler().CheckLinkArgs(pkgLinkArgs, isWasmTarget(ctx.buildConf.Goos)); err != nil { panic(fmt.Sprintf("test link args '%s' failed\n\texpanded to: %v\n\tresolved to: %v\n\terror: %v", param, expdArgs, pkgLinkArgs, err)) @@ -595,6 +609,49 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l } } }) + + // Heuristic: if link args reference python libs, force python init + if !needPyInit { + for _, arg := range linkArgs { + if strings.Contains(arg, "python") { + needPyInit = true + break + } + } + } + + // Ensure python runtime cache when python initialization is needed + if needPyInit { + // Prepare Python environment for build/link + _ = pyenv.Ensure() + _ = pyenv.EnsureBuildEnv() + _ = pyenv.Verify() + // On macOS, avoid absolute rpath: copy libpython near the executable and use @executable_path + if ctx.buildConf.Goos == "darwin" { + if pyHome := pyenv.PythonHome(); pyHome != "" { + // Place copied libs next to the binary: /.llgo/lib + appDir := filepath.Dir(app) + appLibDir := filepath.Join(appDir, ".llgo", "lib") + _ = os.MkdirAll(appLibDir, 0755) + // Copy libpython*.dylib into appLibDir + if matches, _ := filepath.Glob(filepath.Join(pyHome, "lib", "libpython*.dylib")); len(matches) > 0 { + src := matches[0] + dst := filepath.Join(appLibDir, filepath.Base(src)) + if data, err := os.ReadFile(src); err == nil { + _ = os.WriteFile(dst, data, 0644) + } + // Normalize copied lib's install name to @rpath/ + _ = exec.Command("install_name_tool", "-id", "@rpath/"+filepath.Base(dst), dst).Run() + } + // Add relative rpath + relRPath := "-Wl,-rpath,@executable_path/.llgo/lib" + if !slices.Contains(linkArgs, relRPath) { + linkArgs = append(linkArgs, relRPath) + } + } + } + } + entryObjFile, err := genMainModuleFile(ctx, llssa.PkgRuntime, pkg, needRuntime, needPyInit) check(err) // defer os.Remove(entryLLFile) @@ -609,6 +666,14 @@ func linkMainPkg(ctx *context, pkg *packages.Package, pkgs []*aPackage, global l err = linkObjFiles(ctx, app, objFiles, linkArgs, verbose) check(err) + // After linking, ensure the executable references libpython via @rpath + if needPyInit && ctx.buildConf.Goos == "darwin" { + if dep := findDylibDep(app, "python3"); dep != "" && !strings.HasPrefix(dep, "@rpath") { + base := filepath.Base(dep) + _ = exec.Command("install_name_tool", "-change", dep, "@rpath/"+base, app).Run() + } + } + switch mode { case ModeTest: cmd := exec.Command(app, conf.RunArgs...) @@ -687,17 +752,22 @@ func isWasmTarget(goos string) bool { func genMainModuleFile(ctx *context, rtPkgPath string, pkg *packages.Package, needRuntime, needPyInit bool) (path string, err error) { var ( + pyEnvDecl string + pyEnvInit string pyInitDecl string pyInit string rtInitDecl string rtInit string ) + fmt.Print(rtPkgPath) mainPkgPath := pkg.PkgPath if needRuntime { rtInit = "call void @\"" + rtPkgPath + ".init\"()" rtInitDecl = "declare void @\"" + rtPkgPath + ".init\"()" } if needPyInit { + pyEnvDecl = "declare void @\"github.com/goplus/llgo/runtimeext/pyenvrt.init\"()" + pyEnvInit = "call void @\"github.com/goplus/llgo/runtimeext/pyenvrt.init\"()" pyInit = "call void @Py_Initialize()" pyInitDecl = "declare void @Py_Initialize()" } @@ -750,6 +820,7 @@ source_filename = "main" %s %s %s +%s declare void @"%s.init"() declare void @"%s.main"() define weak void @runtime.init() { @@ -770,15 +841,16 @@ _llgo_0: %s %s %s + %s call void @runtime.init() call void @"%s.init"() call void @"%s.main"() ret i32 0 } `, declSizeT, stdioDecl, - pyInitDecl, rtInitDecl, mainPkgPath, mainPkgPath, + pyEnvDecl, pyInitDecl, rtInitDecl, mainPkgPath, mainPkgPath, startDefine, mainDefine, stdioNobuf, - pyInit, rtInit, mainPkgPath, mainPkgPath) + pyEnvInit, pyInit, rtInit, mainPkgPath, mainPkgPath) return exportObject(ctx, pkg.PkgPath+".main", pkg.ExportFile+"-main", []byte(mainCode)) } diff --git a/internal/pyenv/fetch.go b/internal/pyenv/fetch.go new file mode 100644 index 0000000000..36dfe67b49 --- /dev/null +++ b/internal/pyenv/fetch.go @@ -0,0 +1,127 @@ +package pyenv + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +func downloadAndExtract(url, dir string) (err error) { + if _, err = os.Stat(dir); err == nil { + os.RemoveAll(dir) + } + tempDir := dir + ".temp" + os.RemoveAll(tempDir) + if err = os.MkdirAll(tempDir, 0755); err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + + urlPath := strings.Split(url, "/") + filename := urlPath[len(urlPath)-1] + localFile := filepath.Join(tempDir, filename) + if err = downloadFile(url, localFile); err != nil { + return fmt.Errorf("failed to download file: %w", err) + } + defer os.Remove(localFile) + + if strings.HasSuffix(filename, ".tar.gz") || strings.HasSuffix(filename, ".tgz") { + err = extractTarGz(localFile, tempDir) + } else { + return fmt.Errorf("unsupported archive format: %s", filename) + } + if err != nil { + return fmt.Errorf("failed to extract archive: %w", err) + } + if err = os.Rename(tempDir, dir); err != nil { + return fmt.Errorf("failed to rename directory: %w", err) + } + return nil +} + +func downloadFile(url, filepath string) error { + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("bad status: %s", resp.Status) + } + _, err = io.Copy(out, resp.Body) + return err +} + +func extractTarGz(tarGzFile, dest string) error { + file, err := os.Open(tarGzFile) + if err != nil { + return err + } + defer file.Close() + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + target := filepath.Join(dest, header.Name) + if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("%s: illegal file path", target) + } + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(target, 0755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return err + } + f.Close() + case tar.TypeSymlink: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + _ = os.Remove(target) + if err := os.Symlink(header.Linkname, target); err != nil { + return err + } + case tar.TypeLink: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return err + } + _ = os.Remove(target) + linkTarget := filepath.Join(dest, header.Linkname) + if err := os.Link(linkTarget, target); err != nil { + return err + } + + } + } + return nil +} diff --git a/internal/pyenv/pybuild.go b/internal/pyenv/pybuild.go new file mode 100644 index 0000000000..3638bdb1b0 --- /dev/null +++ b/internal/pyenv/pybuild.go @@ -0,0 +1,128 @@ +package pyenv + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/goplus/llgo/internal/env" +) + +// EnsureBuildEnv ensures the Python build environment by: +// - ensuring the cache directory {LLGoCacheDir}/python_env exists (atomic creation) +// - applying PATH/PYTHONHOME/DYLD_LIBRARY_PATH/PKG_CONFIG_PATH +// Priority: use LLPYG_PYHOME if set; otherwise default to the cache path. +func EnsureBuildEnv() error { + if err := Ensure(); err != nil { + return err + } + pyHome := getPyHome(filepath.Join(env.LLGoCacheDir(), "python_env", "python")) + return applyEnv(pyHome) +} + +// Verify runs a lightweight check to ensure a usable Python is available +// in current environment. It tries to execute: python -c "import sys; print('OK')". +func Verify() error { + exe, err := findPythonExec() + if err != nil { + return err + } + cmd := exec.Command(exe, "-c", "import sys; print('OK')") + cmd.Stdout, cmd.Stderr = nil, nil + return cmd.Run() +} + +// PythonHome returns the path that should be used as PYTHONHOME, +// preferring LLPYG_PYHOME if set; otherwise defaulting to the llgo cache path. +func PythonHome() string { + return getPyHome(filepath.Join(env.LLGoCacheDir(), "python_env", "python")) +} + +func findPythonExec() (string, error) { + if p, err := exec.LookPath("python"); err == nil { + return p, nil + } + if p, err := exec.LookPath("python3"); err == nil { + return p, nil + } + return "", exec.ErrNotFound +} + +func getPyHome(defaultPath string) string { + if v := os.Getenv("LLPYG_PYHOME"); v != "" { + return v + } + return defaultPath +} + +func applyEnv(pyHome string) error { + if pyHome == "" { + return nil + } + bin := filepath.Join(pyHome, "bin") + lib := filepath.Join(pyHome, "lib") + + // PATH: prepend pyHome/bin if not present + path := os.Getenv("PATH") + parts := strings.Split(path, string(os.PathListSeparator)) + hasBin := false + for _, p := range parts { + if p == bin { + hasBin = true + break + } + } + if !hasBin { + newPath := bin + if path != "" { + newPath += string(os.PathListSeparator) + path + } + if err := os.Setenv("PATH", newPath); err != nil { + return err + } + } + + // PYTHONHOME + if err := os.Setenv("PYTHONHOME", pyHome); err != nil { + return err + } + + // macOS: DYLD_LIBRARY_PATH append lib if missing + if runtime.GOOS == "darwin" { + dyld := os.Getenv("DYLD_LIBRARY_PATH") + if dyld == "" { + if err := os.Setenv("DYLD_LIBRARY_PATH", lib); err != nil { + return err + } + } else if !strings.Contains(dyld, lib) { + if err := os.Setenv("DYLD_LIBRARY_PATH", lib+string(os.PathListSeparator)+dyld); err != nil { + return err + } + } + } + + // PKG_CONFIG_PATH: add pyHome/lib/pkgconfig + pkgcfg := filepath.Join(pyHome, "lib", "pkgconfig") + pcp := os.Getenv("PKG_CONFIG_PATH") + if pcp == "" { + _ = os.Setenv("PKG_CONFIG_PATH", pkgcfg) + } else { + parts := strings.Split(pcp, string(os.PathListSeparator)) + found := false + for _, p := range parts { + if p == pkgcfg { + found = true + break + } + } + if !found { + _ = os.Setenv("PKG_CONFIG_PATH", pkgcfg+string(os.PathListSeparator)+pcp) + } + } + + // Avoid interference from custom PYTHONPATH + _ = os.Unsetenv("PYTHONPATH") + return nil +} diff --git a/internal/pyenv/pyenv.go b/internal/pyenv/pyenv.go new file mode 100644 index 0000000000..aa359ad2ae --- /dev/null +++ b/internal/pyenv/pyenv.go @@ -0,0 +1,74 @@ +package pyenv + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/goplus/llgo/internal/env" +) + +const pythonUrl = "https://github.com/astral-sh/python-build-standalone/releases/download/20250808/cpython-3.12.11+20250808-x86_64-apple-darwin-install_only.tar.gz" + +// Ensure makes sure the Python runtime cache directory exists under +// {LLGoCacheDir()}/python_env using an atomic temp-dir rename pattern. +// It is safe to call concurrently and is idempotent. +func Ensure() error { + root := filepath.Join(env.LLGoCacheDir(), "python_env") + return ensureDirAtomic(root) +} + +// EnsureWithFetch ensures the cache directory exists and, +// if it is empty and url is not empty, downloads and extracts +// assets from the given url into the cache directory. +func EnsureWithFetch(url string) error { + if url == "" { + url = pythonUrl + } + root := filepath.Join(env.LLGoCacheDir(), "python_env") + if err := ensureDirAtomic(root); err != nil { + return err + } + empty, err := isDirEmpty(root) + if err != nil { + return err + } + if empty && url != "" { + fmt.Println("downloading python assets from", url) + return downloadAndExtract(url, root) + } + return nil +} + +func ensureDirAtomic(dir string) error { + if st, err := os.Stat(dir); err == nil && st.IsDir() { + return nil + } + tmp := dir + ".temp" + _ = os.RemoveAll(tmp) + if err := os.MkdirAll(tmp, 0o755); err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + // // Optional marker to indicate successful initialization + // _ = os.WriteFile(filepath.Join(tmp, ".init_ok"), []byte(time.Now().Format(time.RFC3339)), 0o644) + // if err := os.Rename(tmp, dir); err != nil { + // _ = os.RemoveAll(tmp) + // // If another process won the race, treat as success + // if st, err2 := os.Stat(dir); err2 == nil && st.IsDir() { + // return nil + // } + // return fmt.Errorf("rename temp to final: %w", err) + // } + return nil +} + +func isDirEmpty(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return true, nil + } + return false, err + } + return len(entries) == 0, nil +} diff --git a/runtimeext/pyenvrt/pyenvrt.go b/runtimeext/pyenvrt/pyenvrt.go new file mode 100644 index 0000000000..3b6555f316 --- /dev/null +++ b/runtimeext/pyenvrt/pyenvrt.go @@ -0,0 +1,88 @@ +package pyenvrt + +/* +#include +#include + +#if defined(__APPLE__) +static int llgo_is_darwin() { return 1; } +#else +static int llgo_is_darwin() { return 0; } +#endif + +static void set_env(const char* k, const char* v) { if (k && v) setenv(k, v, 1); } +static void unset_env(const char* k) { if (k) unsetenv(k); } +static const char* get_env(const char* k) { return k ? getenv(k) : NULL; } + +static void prepend_path(const char* k, const char* v) { + if (!k || !v || !*v) return; + const char* cur = getenv(k); + if (!cur || !*cur) { setenv(k, v, 1); return; } + size_t lv = strlen(v), lc = strlen(cur); + char* buf = (char*)malloc(lv + 1 + lc + 1); + if (!buf) return; + memcpy(buf, v, lv); buf[lv] = ':'; memcpy(buf+lv+1, cur, lc); buf[lv+1+lc] = 0; + setenv(k, buf, 1); + free(buf); +} +*/ +import "C" +import "unsafe" + +func init() { + + pyHome := getenv("LLPYG_PYHOME") + if pyHome == "" { + if base := cacheBase(); base != "" { + pyHome = base + "/python_env/python" + } + } + if pyHome == "" { + return + } + setenv("PYTHONHOME", pyHome) + prependPath("PATH", pyHome+"/bin") +} + +func getenv(k string) string { + ck := C.CString(k) + defer C.free(unsafe.Pointer(ck)) + cv := C.get_env(ck) + if cv == nil { + return "" + } + return C.GoString(cv) +} + +func setenv(k, v string) { + ck, cv := C.CString(k), C.CString(v) + C.set_env(ck, cv) + C.free(unsafe.Pointer(ck)) + C.free(unsafe.Pointer(cv)) +} + +func unsetenv(k string) { + ck := C.CString(k) + C.unset_env(ck) + C.free(unsafe.Pointer(ck)) +} + +func prependPath(k, v string) { + ck, cv := C.CString(k), C.CString(v) + C.prepend_path(ck, cv) + C.free(unsafe.Pointer(ck)) + C.free(unsafe.Pointer(cv)) +} + +func cacheBase() string { + if v := getenv("LLGO_CACHE_DIR"); v != "" { + return v + } + if home := getenv("HOME"); home != "" { + return home + "/Library/Caches/llgo" + } + if xdg := getenv("XDG_CACHE_HOME"); xdg != "" { + return xdg + "/llgo" + } + return "" +}