diff --git a/internal/catalog/local.go b/internal/catalog/local.go new file mode 100644 index 0000000..df25945 --- /dev/null +++ b/internal/catalog/local.go @@ -0,0 +1,128 @@ +package catalog + +import ( + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/BurntSushi/toml" + "github.com/anchore/syft/syft/file" +) + +// localPyProject captures only the tables that declare where a dependency comes +// from: uv's [tool.uv.sources] and poetry's [tool.poetry.dependencies]. A +// dependency wired to a local path or workspace member is the repo's own code, +// not a public-registry package. +type localPyProject struct { + Tool struct { + Uv struct { + Sources map[string]any `toml:"sources"` + } `toml:"uv"` + Poetry struct { + Dependencies map[string]any `toml:"dependencies"` + } `toml:"poetry"` + } `toml:"tool"` +} + +// pep503Separators matches the runs of "-", "_" and "." that PEP 503 treats as +// equivalent, so "My_Pkg", "my.pkg" and "my-pkg" canonicalise to one name. The +// manifest keys and the cataloguer output can disagree on case and separator. +var pep503Separators = regexp.MustCompile(`[-_.]+`) + +func canonicalPackageName(s string) string { + return pep503Separators.ReplaceAllString(strings.ToLower(strings.TrimSpace(s)), "-") +} + +// findLocalPackageNames walks every pyproject.toml under root and returns the +// canonicalised names of packages declared as local (uv path/workspace sources +// or poetry path dependencies). Vendored trees are skipped, matching the +// cataloguers. Best-effort: unreadable or malformed manifests are ignored so a +// stray file never fails the scan. +func findLocalPackageNames(resolver file.Resolver, root string) map[string]struct{} { + names := map[string]struct{}{} + locs, err := resolver.FilesByGlob("**/pyproject.toml") + if err != nil { + return names + } + for _, loc := range locs { + if isVendoredPath(loc.RealPath) { + continue + } + data, err := os.ReadFile(filepath.Join(root, loc.RealPath)) + if err != nil { + continue + } + collectLocalNames(data, names) + } + return names +} + +// collectLocalNames parses one pyproject.toml and adds the canonical name of +// every locally-sourced package to out. +func collectLocalNames(data []byte, out map[string]struct{}) { + var pp localPyProject + if err := toml.Unmarshal(data, &pp); err != nil { + return + } + for name, src := range pp.Tool.Uv.Sources { + if isLocalUVSource(src) { + out[canonicalPackageName(name)] = struct{}{} + } + } + for name, spec := range pp.Tool.Poetry.Dependencies { + if strings.EqualFold(name, "python") { + continue + } + if isLocalPoetryDep(spec) { + out[canonicalPackageName(name)] = struct{}{} + } + } +} + +// isLocalUVSource reports whether a [tool.uv.sources] entry points at local +// code: a `path = "..."` (editable path dep) or `workspace = true` (workspace +// member). git/url sources are remote and left alone. uv also allows an array +// of source tables (one per environment marker); any local entry makes the +// dependency local. +func isLocalUVSource(v any) bool { + switch t := v.(type) { + case map[string]any: + return sourceTableIsLocal(t) + case []map[string]any: + for _, e := range t { + if sourceTableIsLocal(e) { + return true + } + } + case []any: + for _, e := range t { + if m, ok := e.(map[string]any); ok && sourceTableIsLocal(m) { + return true + } + } + } + return false +} + +func sourceTableIsLocal(m map[string]any) bool { + if _, ok := m["path"]; ok { + return true + } + if w, ok := m["workspace"].(bool); ok && w { + return true + } + return false +} + +// isLocalPoetryDep reports whether a [tool.poetry.dependencies] entry is a local +// path dependency (`{ path = "..." }`). Bare version strings and git/url tables +// are remote. +func isLocalPoetryDep(v any) bool { + m, ok := v.(map[string]any) + if !ok { + return false + } + _, ok = m["path"] + return ok +} diff --git a/internal/catalog/local_test.go b/internal/catalog/local_test.go new file mode 100644 index 0000000..00ddc09 --- /dev/null +++ b/internal/catalog/local_test.go @@ -0,0 +1,160 @@ +package catalog + +import ( + "testing" +) + +func TestCanonicalPackageName(t *testing.T) { + tests := map[string]string{ + "Common": "common", + " Models ": "models", + "my_pkg": "my-pkg", + "my.pkg": "my-pkg", + "My__Weird..P": "my-weird-p", + "already-dash": "already-dash", + } + for in, want := range tests { + if got := canonicalPackageName(in); got != want { + t.Errorf("canonicalPackageName(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCollectLocalNames_UVSources(t *testing.T) { + // Mirrors the OSS-1389 report: internal packages wired to local paths / + // workspace, alongside a git source that must stay remote. + body := []byte(` +[project] +name = "app" +dependencies = ["common", "models", "shared", "fastapi>=0.127.0", "requests"] + +[tool.uv.sources] +models = { path = "../models", editable = true } +common = { path = "../common", editable = true } +shared = { workspace = true } +somelib = { git = "https://github.com/x/somelib" } +`) + out := map[string]struct{}{} + collectLocalNames(body, out) + + for _, want := range []string{"models", "common", "shared"} { + if _, ok := out[want]; !ok { + t.Errorf("expected %q flagged local; got %v", want, keysOf(out)) + } + } + // git source and normal pypi deps are remote — never flagged. + for _, notWant := range []string{"somelib", "fastapi", "requests"} { + if _, ok := out[notWant]; ok { + t.Errorf("%q is remote and must not be flagged local; got %v", notWant, keysOf(out)) + } + } +} + +func TestCollectLocalNames_UVSourceArray(t *testing.T) { + // uv allows per-marker arrays of source tables; any local entry counts. + body := []byte(` +[tool.uv.sources] +mixed = [ + { path = "../mixed", marker = "sys_platform == 'linux'" }, + { index = "pypi", marker = "sys_platform == 'win32'" }, +] +remote = [ + { git = "https://github.com/x/remote" }, +] +`) + out := map[string]struct{}{} + collectLocalNames(body, out) + + if _, ok := out["mixed"]; !ok { + t.Errorf("array source with a path entry should be local; got %v", keysOf(out)) + } + if _, ok := out["remote"]; ok { + t.Errorf("array source with only remote entries must not be local; got %v", keysOf(out)) + } +} + +func TestCollectLocalNames_PoetryPathDeps(t *testing.T) { + body := []byte(` +[tool.poetry] +name = "app" + +[tool.poetry.dependencies] +python = "^3.11" +internal = { path = "../internal", develop = true } +requests = "^2.31.0" +django = { git = "https://github.com/django/django.git" } +`) + out := map[string]struct{}{} + collectLocalNames(body, out) + + if _, ok := out["internal"]; !ok { + t.Errorf("poetry path dep should be local; got %v", keysOf(out)) + } + // python, a versioned dep, and a git dep are all remote/non-packages. + for _, notWant := range []string{"python", "requests", "django"} { + if _, ok := out[notWant]; ok { + t.Errorf("%q must not be flagged local; got %v", notWant, keysOf(out)) + } + } +} + +func TestCollectLocalNames_NoSources(t *testing.T) { + body := []byte(` +[project] +name = "app" +dependencies = ["requests", "flask"] +`) + out := map[string]struct{}{} + collectLocalNames(body, out) + if len(out) != 0 { + t.Errorf("no local sources declared; got %v", keysOf(out)) + } +} + +func TestCollectLocalNames_MalformedIsIgnored(t *testing.T) { + out := map[string]struct{}{} + collectLocalNames([]byte("not = = valid toml"), out) + if len(out) != 0 { + t.Errorf("malformed manifest should yield nothing; got %v", keysOf(out)) + } +} + +func TestMarkLocalPackages(t *testing.T) { + pkgs := []Package{ + {Name: "common", Type: "pypi"}, + {Name: "Models", Type: "pypi"}, // canonicalised match despite casing + {Name: "requests", Type: "pypi"}, // not local + {Name: "common", Type: "npm"}, // same name, wrong ecosystem — untouched + } + local := map[string]struct{}{"common": {}, "models": {}} + markLocalPackages(pkgs, local) + + if !pkgs[0].Local { + t.Error("common (pypi) should be marked local") + } + if !pkgs[1].Local { + t.Error("Models (pypi) should be marked local via canonical match") + } + if pkgs[2].Local { + t.Error("requests should not be marked local") + } + if pkgs[3].Local { + t.Error("npm common must not be marked local (pypi-only)") + } +} + +func TestMarkLocalPackages_EmptySet(t *testing.T) { + pkgs := []Package{{Name: "common", Type: "pypi"}} + markLocalPackages(pkgs, nil) + if pkgs[0].Local { + t.Error("empty local set should mark nothing") + } +} + +func keysOf(m map[string]struct{}) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/internal/catalog/syft.go b/internal/catalog/syft.go index dbacd3d..56580de 100644 --- a/internal/catalog/syft.go +++ b/internal/catalog/syft.go @@ -19,6 +19,12 @@ type Package struct { Type string Source []string Locations []string + // Local marks a package declared as local code (a uv path/workspace source + // or a poetry path dependency) rather than a public-registry package. The + // platform filters these out before scanning — they are the repo's own code, + // never published, so scanning them only yields spurious NOT_FOUND warnings + // (OSS-1389). + Local bool } // Catalog returns Python + JavaScript packages under path. @@ -114,7 +120,27 @@ func Catalog(ctx context.Context, path string) ([]Package, error) { } } - return mergeVersionless(out), nil + merged := mergeVersionless(out) + markLocalPackages(merged, findLocalPackageNames(resolver, absRoot)) + return merged, nil +} + +// markLocalPackages sets Local=true on every pypi package whose name matches a +// locally-declared package (uv path/workspace source or poetry path dep). Names +// are compared PEP 503-canonically so casing/separator differences between the +// manifest and the cataloguer output don't matter. No-op when local is empty. +func markLocalPackages(pkgs []Package, local map[string]struct{}) { + if len(local) == 0 { + return + } + for i := range pkgs { + if pkgs[i].Type != "pypi" { + continue + } + if _, ok := local[canonicalPackageName(pkgs[i].Name)]; ok { + pkgs[i].Local = true + } + } } // mergeVersionless collapses a package emitted both with and without a version. diff --git a/internal/scan/scan.go b/internal/scan/scan.go index 5d8ddad..46c1e7e 100644 --- a/internal/scan/scan.go +++ b/internal/scan/scan.go @@ -50,13 +50,21 @@ func Run(ctx context.Context, opts Options) (*ossbom.SBOM, error) { sbom.Name = project for _, p := range pkgs { - sbom.AddComponent(ossbom.Component{ + c := ossbom.Component{ Name: p.Name, Version: p.Version, Type: p.Type, Source: p.Source, Location: p.Locations, - }) + } + // Flag locally-defined packages (uv path/workspace sources, poetry path + // deps) so the platform can filter them before scanning (OSS-1389). The + // flag rides in metadata because it round-trips through the OSSBOM the + // platform parses from `--local` output. + if p.Local { + c.Metadata = map[string]any{"local": true} + } + sbom.AddComponent(c) } return sbom, nil diff --git a/internal/scan/scan_test.go b/internal/scan/scan_test.go index f081577..ba9d1b1 100644 --- a/internal/scan/scan_test.go +++ b/internal/scan/scan_test.go @@ -3,6 +3,7 @@ package scan import ( "context" "errors" + "os" "path/filepath" "runtime" "strings" @@ -105,6 +106,48 @@ func TestRun_Fixtures(t *testing.T) { } } +// TestRun_FlagsLocalPackages is the OSS-1389 end-to-end: internal packages +// wired to local paths in [tool.uv.sources] are catalogued as pypi components +// and flagged metadata.local=true, while a real registry dep is not. +func TestRun_FlagsLocalPackages(t *testing.T) { + dir := t.TempDir() + manifest := `[project] +name = "app" +dependencies = ["common", "models", "requests>=2.0"] + +[tool.uv.sources] +common = { path = "../common", editable = true } +models = { path = "../models", editable = true } +` + if err := os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(manifest), 0o644); err != nil { + t.Fatalf("write pyproject: %v", err) + } + + sbom, err := Run(context.Background(), Options{Path: dir}) + if err != nil { + t.Fatalf("Run: %v", err) + } + + isLocal := func(c ossbom.Component) bool { + v, ok := c.Metadata["local"].(bool) + return ok && v + } + local := map[string]bool{} + for _, c := range sbom.Components { + if isLocal(c) { + local[strings.ToLower(c.Name)] = true + } + if strings.ToLower(c.Name) == "requests" && isLocal(c) { + t.Error("requests (registry dep) must not be flagged local") + } + } + for _, want := range []string{"common", "models"} { + if !local[want] { + t.Errorf("expected %q flagged local; components=%v", want, componentNames(sbom)) + } + } +} + func TestRun_MissingPath(t *testing.T) { _, err := Run(context.Background(), Options{Path: "does-not-exist-fixture"}) if err == nil {