Skip to content

Commit

Permalink
all: account for language package overwrites
Browse files Browse the repository at this point in the history
Signed-off-by: RTann <[email protected]>
  • Loading branch information
RTann committed Apr 9, 2024
1 parent 813e7cc commit 6af0edd
Show file tree
Hide file tree
Showing 11 changed files with 207 additions and 219 deletions.
21 changes: 20 additions & 1 deletion gobin/coalescer.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,13 @@ func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts)
Packages: map[string]*claircore.Package{},
Repositories: map[string]*claircore.Repository{},
}
for _, l := range ls {
// Similar to ir.Packages, except instead of mapping
// id -> package, it maps packageDB -> package.
// For langauge packages, it is possible the
// packageDB is overwritten.
packages := make(map[string]*claircore.Package)
for i := len(ls) - 1; i >= 0; i-- {
l := ls[i]

Check warning on line 25 in gobin/coalescer.go

View check run for this annotation

Codecov / codecov/patch

gobin/coalescer.go#L23-L25

Added lines #L23 - L25 were not covered by tests
var rid string
for _, r := range l.Repos {
// Magic strings copied out of the osv package.
Expand All @@ -31,6 +37,19 @@ func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts)
if !strings.HasPrefix(pkg.PackageDB, "go:") {
continue
}
if childPkg, exists := packages[pkg.PackageDB]; exists {

Check warning on line 40 in gobin/coalescer.go

View check run for this annotation

Codecov / codecov/patch

gobin/coalescer.go#L40

Added line #L40 was not covered by tests
// If the package was renamed or has a different version in a high layer,
// then we consider this a different package and ignore the
// original in the lower layer.
if pkg.Name != childPkg.Name || pkg.Version != childPkg.Version {
continue

Check warning on line 45 in gobin/coalescer.go

View check run for this annotation

Codecov / codecov/patch

gobin/coalescer.go#L44-L45

Added lines #L44 - L45 were not covered by tests
}
// The name and version is the same, so delete the entry related to the higher
// layer, as this package was likely introduced in the lower layer.
delete(ir.Packages, childPkg.ID)
delete(ir.Environments, childPkg.ID)

Check warning on line 50 in gobin/coalescer.go

View check run for this annotation

Codecov / codecov/patch

gobin/coalescer.go#L49-L50

Added lines #L49 - L50 were not covered by tests
}
packages[pkg.PackageDB] = pkg

Check warning on line 52 in gobin/coalescer.go

View check run for this annotation

Codecov / codecov/patch

gobin/coalescer.go#L52

Added line #L52 was not covered by tests
ir.Packages[pkg.ID] = pkg
ir.Environments[pkg.ID] = []*claircore.Environment{
{
Expand Down
42 changes: 0 additions & 42 deletions java/coalescer.go

This file was deleted.

7 changes: 3 additions & 4 deletions java/ecosystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ import (
"context"

"github.com/quay/claircore/indexer"
"github.com/quay/claircore/language"
)

// NewEcosystem provides the set of scanners for the java ecosystem.
func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
func NewEcosystem(_ context.Context) *indexer.Ecosystem {

Check warning on line 11 in java/ecosystem.go

View check run for this annotation

Codecov / codecov/patch

java/ecosystem.go#L11

Added line #L11 was not covered by tests
return &indexer.Ecosystem{
PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) {
return []indexer.PackageScanner{&Scanner{}}, nil
Expand All @@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) {
return nil, nil
},
Coalescer: func(_ context.Context) (indexer.Coalescer, error) {
return (*coalescer)(nil), nil
},
Coalescer: language.NewCoalescer,
}
}
28 changes: 23 additions & 5 deletions nodejs/coalescer.go → language/coalescer.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package nodejs
package language

import (
"context"
"github.com/quay/claircore"

"github.com/quay/claircore"
"github.com/quay/claircore/indexer"
)

Expand All @@ -21,9 +21,14 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*
Packages: map[string]*claircore.Package{},
Repositories: map[string]*claircore.Repository{},
}

for _, l := range ls {
// If we didn't find at least one npm repo in this layer
// Similar to ir.Packages, except instead of mapping
// id -> package, it maps packageDB -> package.
// For langauge packages, it is possible the
// packageDB is overwritten.
packages := make(map[string]*claircore.Package)
for i := len(ls) - 1; i >= 0; i-- {
l := ls[i]
// If we didn't find at least one repo in this layer
// no point in searching for packages.
if len(l.Repos) == 0 {
continue
Expand All @@ -34,6 +39,19 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*
ir.Repositories[r.ID] = r
}
for _, pkg := range l.Pkgs {
if childPkg, exists := packages[pkg.PackageDB]; exists {
// If the package was renamed or has a different version in a high layer,
// then we consider this a different package and ignore the
// original in the lower layer.
if pkg.Name != childPkg.Name || pkg.Version != childPkg.Version {
continue
}
// The name and version is the same, so delete the entry related to the higher
// layer, as this package was likely introduced in the lower layer.
delete(ir.Packages, childPkg.ID)
delete(ir.Environments, childPkg.ID)
}
packages[pkg.PackageDB] = pkg
ir.Packages[pkg.ID] = pkg
ir.Environments[pkg.ID] = []*claircore.Environment{
{
Expand Down
155 changes: 155 additions & 0 deletions language/coalescer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
package language

import (
"context"
"strconv"
"testing"

"github.com/quay/zlog"

"github.com/quay/claircore"
"github.com/quay/claircore/indexer"
"github.com/quay/claircore/test"
)

func TestCoalescer(t *testing.T) {
t.Parallel()
ctx := zlog.Test(context.Background(), t)
coalescer := &coalescer{}
pkgs := test.GenUniquePackages(6)
repo := []*claircore.Repository{{
Name: "npm",
URI: "https://www.npmjs.com/",
}}
layerArtifacts := []*indexer.LayerArtifacts{
{
Hash: test.RandomSHA256Digest(t),
Pkgs: pkgs[:1],
},
{
Hash: test.RandomSHA256Digest(t),
Pkgs: pkgs[:2],
},
{
Hash: test.RandomSHA256Digest(t),
Pkgs: pkgs[:3],
Repos: repo,
},
{
Hash: test.RandomSHA256Digest(t),
Pkgs: pkgs[:4],
},
{
Hash: test.RandomSHA256Digest(t),
Pkgs: pkgs[:5],
Repos: repo,
},
{
Hash: test.RandomSHA256Digest(t),
Pkgs: pkgs,
},
}
ir, err := coalescer.Coalesce(ctx, layerArtifacts)
if err != nil {
t.Fatalf("received error from coalesce method: %v", err)
}
// Expect 0-5 to have gotten associated with the repository.
for i := range pkgs {
es, ok := ir.Environments[strconv.Itoa(i)]
if !ok && i == 5 {
// Left out the last package.
continue
}
e := es[0]
if len(e.RepositoryIDs) == 0 {
t.Error("expected some repositories")
}
for _, id := range e.RepositoryIDs {
r := ir.Repositories[id]
if got, want := r.Name, "npm"; got != want {
t.Errorf("got: %q, want: %q", got, want)
}
}
}
}

func TestCoalescer_package_overwrite(t *testing.T) {
t.Parallel()
ctx := zlog.Test(context.Background(), t)
coalescer := &coalescer{}
repo := []*claircore.Repository{{
Name: "npm",
URI: "https://www.npmjs.com/",
}}
hashes := []claircore.Digest{
test.RandomSHA256Digest(t),
test.RandomSHA256Digest(t),
test.RandomSHA256Digest(t),
test.RandomSHA256Digest(t),
}
layerArtifacts := []*indexer.LayerArtifacts{
{
Hash: hashes[0],
Pkgs: []*claircore.Package{
{
ID: "0",
Name: "semver",
Version: "7.3.8",
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
},
},
Repos: repo,
},
{
Hash: hashes[1],
},
{
Hash: hashes[2],
Pkgs: []*claircore.Package{
{
ID: "1",
Name: "semver",
Version: "7.5.2",
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
},
},
Repos: repo,
},
{
Hash: hashes[3],
Pkgs: []*claircore.Package{
{
ID: "2",
Name: "semver",
Version: "7.5.2",
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
},
},
Repos: repo,
},
}
ir, err := coalescer.Coalesce(ctx, layerArtifacts)
if err != nil {
t.Fatalf("received error from coalesce method: %v", err)
}
if len(ir.Packages) != 1 {
t.Fatalf("unexpected number of packages: %d != %d", len(ir.Packages), 1)
}
pkg, exists := ir.Packages["1"]
if !exists {
t.Fatal("expected package does not exist")
}
if pkg.Version != "7.5.2" {
t.Fatalf("unexpected version: %s != %s", pkg.Version, "7.5.2")
}
envs, exists := ir.Environments["1"]
if !exists {
t.Fatal("expected environments do not exist")
}
if len(envs) != 1 {
t.Fatalf("unexpected number of envionments: %d != %d", len(envs), 1)
}
if envs[0].IntroducedIn.String() != hashes[2].String() {
t.Fatalf("unexpected introducedIn: %s != %s", envs[0].IntroducedIn.String(), hashes[2].String())
}
}
71 changes: 0 additions & 71 deletions nodejs/coalescer_test.go

This file was deleted.

3 changes: 2 additions & 1 deletion nodejs/ecosystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

"github.com/quay/claircore/indexer"
"github.com/quay/claircore/language"
)

var scanners = []indexer.PackageScanner{&Scanner{}}
Expand All @@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem {
PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil },
DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil },
RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil },
Coalescer: NewCoalescer,
Coalescer: language.NewCoalescer,
}
}
Loading

0 comments on commit 6af0edd

Please sign in to comment.