diff --git a/compiler/internal/build/build.go b/compiler/internal/build/build.go index 75f2617deb..a44e05baaf 100644 --- a/compiler/internal/build/build.go +++ b/compiler/internal/build/build.go @@ -34,11 +34,15 @@ import ( "strings" "unsafe" + "golang.org/x/mod/module" "golang.org/x/tools/go/ssa" "github.com/goplus/llgo/compiler/cl" "github.com/goplus/llgo/compiler/internal/env" + "github.com/goplus/llgo/compiler/internal/installer" + "github.com/goplus/llgo/compiler/internal/installer/githubreleases" "github.com/goplus/llgo/compiler/internal/mockable" + "github.com/goplus/llgo/compiler/internal/mod" "github.com/goplus/llgo/compiler/internal/packages" "github.com/goplus/llgo/compiler/internal/typepatch" "github.com/goplus/llgo/compiler/ssa/abi" @@ -62,6 +66,9 @@ const ( ) const ( + defaultLLPkgOwner = "goplus" + defaultLLPkgRepo = "llpkg" + debugBuild = packages.DebugPackagesLoad ) @@ -112,6 +119,15 @@ func DefaultAppExt() string { return "" } +func defaultInstaller() installer.Installer { + return githubreleases.NewGHReleasesInstaller(map[string]string{ + "owner": defaultLLPkgOwner, + "repo": defaultLLPkgRepo, + "platform": runtime.GOOS, + "arch": runtime.GOARCH, + }) +} + // ----------------------------------------------------------------------------- const ( @@ -215,8 +231,10 @@ func Do(args []string, conf *Config) ([]Package, error) { env := llvm.New("") os.Setenv("PATH", env.BinDir()+":"+os.Getenv("PATH")) // TODO(xsw): check windows + installer := defaultInstaller() + output := conf.OutFile != "" - ctx := &context{env, cfg, progSSA, prog, dedup, patches, make(map[string]none), initial, mode, 0, output, make(map[*packages.Package]bool), make(map[*packages.Package]bool)} + ctx := &context{env, cfg, progSSA, prog, dedup, patches, make(map[string]none), initial, mode, 0, output, make(map[*packages.Package]bool), make(map[*packages.Package]bool), make(map[module.Version]string), installer} pkgs, err := buildAllPkgs(ctx, initial, verbose) check(err) if mode == ModeGen { @@ -281,6 +299,30 @@ type context struct { needRt map[*packages.Package]bool needPyInit map[*packages.Package]bool + + llpkgMod map[module.Version]string + installer installer.Installer +} + +// the pc for the deps of each package should be unique +// consider duplicate clib but different version +// exmaple: A requires zlib/1.3.1, B requires zlib/1.3.0 +func setDepsPC(ctx *context, pkg *packages.Package) { + var pcDir []string + ver := module.Version{ + Path: pkg.Module.Path, + Version: pkg.Module.Version, + } + if dir := ctx.llpkgMod[ver]; dir != "" { + pcDir = append(pcDir, dir) + } + + if len(pcDir) > 0 { + if pkgPath, ok := os.LookupEnv("PKG_CONFIG_PATH"); ok { + pcDir = append(pcDir, pkgPath) + } + os.Setenv("PKG_CONFIG_PATH", strings.Join(pcDir, ":")) + } } func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs []*aPackage, err error) { @@ -294,6 +336,7 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs if len(errPkgs) > 0 { return nil, fmt.Errorf("cannot build SSA for packages") } + built := ctx.built for _, aPkg := range pkgs { pkg := aPkg.Package @@ -302,6 +345,11 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs continue } built[pkg.ID] = none{} + + if isLLPkg(pkg) { + setDepsPC(ctx, pkg) + } + switch kind, param := cl.PkgKindOf(pkg.Types); kind { case cl.PkgDeclOnly: // skip packages that only contain declarations @@ -696,6 +744,36 @@ type aPackage struct { type Package = *aPackage +func isLLPkg(pkg *packages.Package) bool { + return pkg.Module != nil && strings.HasPrefix(pkg.Module.Path, mod.LLPkgPathPrefix) +} + +func fetchLLPkg(ctx *context, ver module.Version, verbose bool) string { + llpkg, err := mod.ParseLLPkg(ver) + if err != nil { + return "" + } + + dir, err := mod.LLPkgCacheDirByModule(ver) + check(err) + + pkgConfigDir := filepath.Join(dir, "lib", "pkgconfig") + + _, err = os.Stat(pkgConfigDir) + if os.IsNotExist(err) { + fmt.Printf("installing %s to %s\n", ver.String(), dir) + + err = os.MkdirAll(dir, 0777) + check(err) + _, err = ctx.installer.Install(llpkg, dir) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: installing %s fail: %v\n", ver.String(), err) + } + } + + return pkgConfigDir +} + func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aPackage, errs []*packages.Package) { prog := ctx.progSSA built := ctx.built @@ -705,6 +783,15 @@ func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aP if _, ok := built[pkgPath]; ok || strings.HasPrefix(pkgPath, altPkgPathPrefix) { return } + if isLLPkg(p) { + ver := module.Version{ + Path: p.Module.Path, + Version: p.Module.Version, + } + if _, ok := ctx.llpkgMod[ver]; !ok { + ctx.llpkgMod[ver] = fetchLLPkg(ctx, ver, verbose) + } + } var altPkg *packages.Cached var ssaPkg = createSSAPkg(prog, p, verbose) if llruntime.HasAltPkg(pkgPath) { diff --git a/compiler/internal/installer/config.go b/compiler/internal/installer/config.go new file mode 100644 index 0000000000..e7c1308a6a --- /dev/null +++ b/compiler/internal/installer/config.go @@ -0,0 +1,27 @@ +package installer + +// LLPkgConfig represents the configuration structure parsed from llpkg.cfg files. +type LLPkgConfig struct { + Upstream UpstreamConfig `json:"upstream"` +} + +// UpstreamConfig defines the upstream configuration containing installer settings and package metadata. +type UpstreamConfig struct { + Installer InstallerConfig `json:"installer"` + Package Package `json:"package"` +} + +// InstallerConfig specifies the installer type and its configuration options. +// "name" field must match supported installers (e.g., "conan"). +// "config" holds installer-specific parameters (optional). +type InstallerConfig struct { + Name string `json:"name"` + Config map[string]string `json:"config,omitempty"` +} + +// Package defines the metadata required to identify and install a software library. +// The Name and Version fields provide precise identification of the library. +type Package struct { + Name string + Version string +} diff --git a/compiler/internal/installer/config/parse.go b/compiler/internal/installer/config/parse.go new file mode 100644 index 0000000000..0a549ff439 --- /dev/null +++ b/compiler/internal/installer/config/parse.go @@ -0,0 +1,33 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/goplus/llgo/compiler/internal/installer" +) + +// ParseLLPkgConfig reads and parses the llpkg.cfg configuration file +// Performs the following operations: +// +// 1. Opens and reads the configuration file. +// 2. Deserializes JSON content into LLPkgConfig struct. +// 3. Applies default values for missing parameters. +// 4. Returns parsed config or I/O/decoding errors. +func ParseLLPkgConfig(configPath string) (installer.LLPkgConfig, error) { + var config installer.LLPkgConfig + file, err := os.Open(configPath) + if err != nil { + return config, fmt.Errorf("failed to open config file: %w", err) + } + defer file.Close() + + decoder := json.NewDecoder(file) + err = decoder.Decode(&config) + if err != nil { + return config, fmt.Errorf("failed to decode config file: %w", err) + } + + return config, nil +} diff --git a/compiler/internal/installer/githubreleases/githubreleases.go b/compiler/internal/installer/githubreleases/githubreleases.go new file mode 100644 index 0000000000..735a860d60 --- /dev/null +++ b/compiler/internal/installer/githubreleases/githubreleases.go @@ -0,0 +1,272 @@ +package githubreleases + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/goplus/llgo/compiler/internal/installer" +) + +var ErrPackageNotFound = errors.New("package not found") + +// ghReleasesInstaller implements the installer.Installer interface by downloading +// the corresponding package from a GitHub release. +type ghReleasesInstaller struct { + config map[string]string +} + +// NewGHReleasesInstaller creates a new GitHub Release installer with the specified configuration. +// The config map is the info of release repo, for example: +// "owner": `goplus`, +// "repo": `llpkg`, +// "platform": runtime.GOOS, +// "arch": runtime.GOARCH, +func NewGHReleasesInstaller(config map[string]string) installer.Installer { + return &ghReleasesInstaller{ + config: config, + } +} + +func (c *ghReleasesInstaller) Name() string { + return "ghreleases" +} + +func (c *ghReleasesInstaller) Config() map[string]string { + return c.config +} + +// Install downloads the package from the GitHub Release and extracts it to the output directory. +// Unlike conaninstaller which is used for GitHub Action to obtain binary files +// this installer is used for `llgo get` to install binary files. +// The first return value is an empty string, as the pkgConfigName is not necessary for this GitHub Release installer. +func (c *ghReleasesInstaller) Install(pkg installer.Package, outputDir string) ([]string, error) { + compressPath, err := c.download(c.assertUrl(pkg), outputDir) + if err != nil { + return nil, err + } + if strings.HasSuffix(compressPath, ".tar.gz") { + err = c.untargz(outputDir, compressPath) + if err != nil { + return nil, err + } + } else if strings.HasSuffix(compressPath, ".zip") { + err = c.unzip(outputDir, compressPath) + if err != nil { + return nil, err + } + } else { + return nil, errors.New("unsupported compressed file format") + } + err = os.Remove(compressPath) + if err != nil { + return nil, fmt.Errorf("cannot delete compressed file: %w", err) + } + err = c.setPrefix(outputDir) + if err != nil { + return nil, fmt.Errorf("fail to reset .pc prefix: %w", err) + } + return nil, nil +} + +// Warning: not implemented +// Search is unnecessary for this installer +func (c *ghReleasesInstaller) Search(pkg installer.Package) ([]string, error) { + return nil, errors.New("unimplemented") +} + +// assertUrl returns the URL for the specified package. +// The URL is constructed based on the package name, version, and the installer configuration. +func (c *ghReleasesInstaller) assertUrl(pkg installer.Package) string { + releaseName := fmt.Sprintf("%s/%s", pkg.Name, pkg.Version) + fileName := fmt.Sprintf("%s_%s.zip", pkg.Name, c.config["platform"]+"_"+c.config["arch"]) + return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.config["owner"], c.config["repo"], releaseName, fileName) +} + +// download fetches the package from the specified URL and saves it to the output directory. +func (c *ghReleasesInstaller) download(url string, outputDir string) (string, error) { + client := &http.Client{} + resp, err := client.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return "", ErrPackageNotFound + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + parts := strings.Split(url, "/") + filename := parts[len(parts)-1] + + err = os.MkdirAll(outputDir, 0755) + if err != nil { + return "", err + } + + outputPath := filepath.Join(outputDir, filename) + outFile, err := os.Create(outputPath) + if err != nil { + return "", err + } + defer outFile.Close() + + _, err = io.Copy(outFile, resp.Body) + if err != nil { + return "", err + } + + return outputPath, nil +} + +// untargz extracts the gzip-compressed tarball to the output directory. +// The gzipPath must be a .tar.gz file. +func (c *ghReleasesInstaller) untargz(outputDir string, gzipPath string) error { + fr, err := os.Open(gzipPath) + if err != nil { + return err + } + defer fr.Close() + + gr, err := gzip.NewReader(fr) + if err != nil { + return err + } + defer gr.Close() + + tr := tar.NewReader(gr) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + path := filepath.Join(outputDir, header.Name) + info := header.FileInfo() + + if info.IsDir() { + if err = os.MkdirAll(path, info.Mode()); err != nil { + return err + } + continue + } + dir := filepath.Dir(path) + if err = os.MkdirAll(dir, 0755); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + if _, err = io.Copy(file, tr); err != nil { + return err + } + file.Close() + } + return nil +} + +// unzip extracts the zip file to the output directory. +// The zipPath must be a .zip file. +func (c *ghReleasesInstaller) unzip(outputDir string, zipPath string) error { + r, err := zip.OpenReader(zipPath) + if err != nil { + return err + } + defer r.Close() + decompress := func(file *zip.File) error { + path := filepath.Join(outputDir, file.Name) + + if file.FileInfo().IsDir() { + if err := os.MkdirAll(path, 0755); err != nil { + return err + } + return nil + } + fs, err := file.Open() + if err != nil { + return err + } + w, err := os.Create(path) + if err != nil { + return err + } + defer fs.Close() + if _, err := io.Copy(w, fs); err != nil { + w.Close() + return err + } + return w.Close() + } + + for _, file := range r.File { + if err = decompress(file); err != nil { + break + } + } + return err +} + +// setPrefix can generate .pc files from .pc.tmpl files +func (c *ghReleasesInstaller) setPrefix(outputDir string) error { + absOutputDir, err := filepath.Abs(outputDir) + if err != nil { + return err + } + + // move to path where .pc files are stored + pkgConfigPath := filepath.Join(outputDir, "lib", "pkgconfig") + + pcTmpls, err := filepath.Glob(filepath.Join(pkgConfigPath, "*.pc.tmpl")) + if err != nil { + return err + } + if len(pcTmpls) == 0 { + return errors.New("no .pc.tmpl files found") + } + for _, pcTmpl := range pcTmpls { + tmplContent, err := os.ReadFile(pcTmpl) + if err != nil { + return err + } + tmplName := filepath.Base(pcTmpl) + tmpl, err := template.New(tmplName).Parse(string(tmplContent)) + if err != nil { + return err + } + + pcFilePath := filepath.Join(pkgConfigPath, strings.TrimSuffix(tmplName, ".tmpl")) + var buf bytes.Buffer + // The Prefix field specifies the absolute path to the output directory, + // which is used to replace placeholders in the .pc template files. + if err := tmpl.Execute(&buf, map[string]any{ + "Prefix": absOutputDir, + }); err != nil { + return err + } + if err := os.WriteFile(pcFilePath, buf.Bytes(), 0644); err != nil { + return err + } + // remove .pc.tmpl file + err = os.Remove(filepath.Join(pkgConfigPath, tmplName)) + if err != nil { + return fmt.Errorf("failed to remove template file: %w", err) + } + } + + return nil +} diff --git a/compiler/internal/installer/githubreleases/githubreleases_test.go b/compiler/internal/installer/githubreleases/githubreleases_test.go new file mode 100644 index 0000000000..86a8f70887 --- /dev/null +++ b/compiler/internal/installer/githubreleases/githubreleases_test.go @@ -0,0 +1,124 @@ +package githubreleases + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/goplus/llgo/compiler/internal/installer" + "github.com/goplus/llgo/compiler/internal/pc" +) + +func TestGHInstaller(t *testing.T) { + ghr := &ghReleasesInstaller{ + config: map[string]string{ + "owner": `goplus`, + "repo": `llpkg`, + "platform": runtime.GOOS, + "arch": runtime.GOARCH, + }, + } + + pkg := installer.Package{ + Version: `v1.0.0`, + Name: `libxml2`, + } + + tempDir, err := os.MkdirTemp("", "llpkg-tool") + if err != nil { + t.Errorf("Unexpected error when creating temp dir: %s", err) + return + } + defer os.RemoveAll(tempDir) + + if _, err = ghr.Install(pkg, tempDir); err != nil { + t.Errorf("Install failed: %s", err) + } + + if err := verify(tempDir); err != nil { + t.Errorf("Verify failed: %s", err) + } +} + +func TestNotExistsReleases(t *testing.T) { + ghr := &ghReleasesInstaller{ + config: map[string]string{ + "owner": `goplus`, + "repo": `llpkg`, + "platform": runtime.GOOS, + "arch": runtime.GOARCH, + }, + } + + pkg := installer.Package{ + Version: `v1.0.0`, + Name: `not-exists-pkg`, + } + + tempDir, err := os.MkdirTemp("", "llpkg-tool") + if err != nil { + t.Errorf("Unexpected error when creating temp dir: %s", err) + return + } + defer os.RemoveAll(tempDir) + + if _, err = ghr.Install(pkg, tempDir); err == nil { + t.Errorf("Expecting error but got nil") + } +} + +func verify(installDir string) error { + // 1. ensure .pc file exists + pcFiles, err := filepath.Glob(filepath.Join(installDir, "lib", "pkgconfig", "*.pc")) + if err != nil { + return err + } + if len(pcFiles) == 0 { + return errors.New("cannot find .pc file") + } + absPath, err := filepath.Abs(installDir) + if err != nil { + return err + } + + for _, pcFile := range pcFiles { + // 2. ensure .pc file's prefix is correctly set + pcContent, err := os.ReadFile(pcFile) + if err != nil { + return err + } + prefixMatch := regexp.MustCompile(`^prefix=(.*)`) + if prefixMatch.FindString(string(pcContent)) != fmt.Sprintf("prefix=%s", absPath) { + return errors.New("prefix is not correctly set") + } + // 3. ensure pkg-config can find .pc file + buildCmd := exec.Command("pkg-config", "--libs", strings.TrimSuffix(filepath.Base(pcFile), ".pc")) + pc.SetPath(buildCmd, absPath) + out, err := buildCmd.CombinedOutput() + if err != nil { + return errors.New("pkg-config failed: " + err.Error() + " with output: " + string(out)) + } + } + + // 4. ensure .so or .dylib file exists + switch runtime.GOOS { + case "linux": + matches, _ := filepath.Glob(filepath.Join(installDir, "lib", "*.so")) + if len(matches) == 0 { + return errors.New("cannot find so file") + } + case "darwin": + matches, _ := filepath.Glob(filepath.Join(installDir, "lib", "*.dylib")) + if len(matches) == 0 { + return errors.New("cannot find dylib file") + } + } + + return nil +} diff --git a/compiler/internal/installer/installer.go b/compiler/internal/installer/installer.go new file mode 100644 index 0000000000..37d159a12f --- /dev/null +++ b/compiler/internal/installer/installer.go @@ -0,0 +1,15 @@ +package installer + +// Installer represents a package installer that can download, install, and locate binaries from a remote repository. +// It provides methods to install packages to specific directories and search for installed package information. +type Installer interface { + Name() string + Config() map[string]string + // Install downloads and installs the specified package. + // The outputDir is where build artifacts (e.g., .pc files, headers) are stored. + // Returns an error if installation fails, all the pkgConfigFiles if success. + Install(pkg Package, outputDir string) (pkgConfigFiles []string, err error) + // Search checks remote repository for the specified package availability. + // Returns the search results text and any encountered errors. + Search(pkg Package) ([]string, error) +} diff --git a/compiler/internal/mod/mod.go b/compiler/internal/mod/mod.go index 7b14a493fd..550c63b19d 100644 --- a/compiler/internal/mod/mod.go +++ b/compiler/internal/mod/mod.go @@ -2,8 +2,12 @@ package mod import ( "fmt" + "path/filepath" "github.com/goplus/llgo/compiler/internal/env" + "github.com/goplus/llgo/compiler/internal/installer" + "github.com/goplus/llgo/compiler/internal/installer/config" + "github.com/goplus/mod/modcache" "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -49,6 +53,44 @@ func NewModuleVersionPair(name, version string) (module.Version, error) { return module.Version{Path: name, Version: version}, nil } +func LLPkgCfgFilePath(mod module.Version) (string, error) { + cachePath, err := modcache.Path(mod) + if err != nil { + return "", err + } + + return filepath.Join(cachePath, LLPkgConfigFileName), nil +} + +func LLPkgCacheDirByModule(mod module.Version) (string, error) { + encPath, err := module.EscapePath(mod.Path) + if err != nil { + return "", err + } + + return filepath.Join(LLPkgCacheDir(), encPath+"@"+mod.Version), nil +} + +func LLPkgCacheDir() string { + return filepath.Join(env.LLGOCACHE(), "llpkg") +} + +func ParseLLPkg(mod module.Version) (installer.Package, error) { + cfgPath, err := LLPkgCfgFilePath(mod) + if err != nil { + return installer.Package{}, err + } + + cfg, err := config.ParseLLPkgConfig(cfgPath) + if err != nil { + return installer.Package{}, err + } + return installer.Package{ + Name: cfg.Upstream.Package.Name, + Version: mod.Version, + }, nil +} + // Returns true if the path is a valid module path, false otherwise func IsModulePath(path string) bool { err := module.CheckPath(path) diff --git a/compiler/internal/pc/env.go b/compiler/internal/pc/env.go new file mode 100644 index 0000000000..27b769b374 --- /dev/null +++ b/compiler/internal/pc/env.go @@ -0,0 +1,19 @@ +package pc + +import ( + "fmt" + "os" + "os/exec" +) + +func appendPCPath(path string) string { + if env, ok := os.LookupEnv("PKG_CONFIG_PATH"); ok { + return path + ":" + env + } + return path +} + +func SetPath(cmd *exec.Cmd, path string) { + pcPath := fmt.Sprintf("PKG_CONFIG_PATH=%s", appendPCPath(path)) + cmd.Env = append(os.Environ(), pcPath) +}