From 1c7056f4947f4eb9fe4b5b9354bda7af3de5816a Mon Sep 17 00:00:00 2001 From: crozzy Date: Thu, 30 Nov 2023 09:57:37 -0800 Subject: [PATCH] updater: Support delta updates TODO: message Signed-off-by: crozzy --- datastore/postgres/matcher_store.go | 102 ++++++++ datastore/postgres/matcher_store_test.go | 135 ++++++++++ datastore/updater.go | 2 + go.mod | 3 +- go.sum | 6 +- libvuln/driver/updater.go | 4 + libvuln/jsonblob/jsonblob.go | 5 + libvuln/updates/manager.go | 29 ++- libvuln/updates/manager_test.go | 314 +++++++++++++++++++++++ 9 files changed, 593 insertions(+), 7 deletions(-) create mode 100644 datastore/postgres/matcher_store_test.go create mode 100644 libvuln/updates/manager_test.go diff --git a/datastore/postgres/matcher_store.go b/datastore/postgres/matcher_store.go index e716f48d8..f419cc8eb 100644 --- a/datastore/postgres/matcher_store.go +++ b/datastore/postgres/matcher_store.go @@ -3,6 +3,7 @@ package postgres import ( "context" "fmt" + "strconv" "time" "github.com/google/uuid" @@ -11,6 +12,8 @@ import ( "github.com/quay/zlog" "github.com/remind101/migrate" + "github.com/quay/claircore" + "github.com/quay/claircore/datastore" "github.com/quay/claircore/datastore/postgres/migrations" "github.com/quay/claircore/libvuln/driver" @@ -83,3 +86,102 @@ func (s *MatcherStore) RecordUpdaterStatus(ctx context.Context, updaterName stri func (s *MatcherStore) RecordUpdaterSetStatus(ctx context.Context, updaterSet string, updateTime time.Time) error { return recordUpdaterSetStatus(ctx, s.pool, updaterSet, updateTime) } + +func (s *MatcherStore) GetLatestVulnerabilities(ctx context.Context, updater string) ([]*claircore.Vulnerability, error) { + query := ` + SELECT + "vuln"."id", + "name", + "description", + "issued", + "links", + "severity", + "normalized_severity", + "package_name", + "package_version", + "package_module", + "package_arch", + "package_kind", + "dist_id", + "dist_name", + "dist_version", + "dist_version_code_name", + "dist_version_id", + "dist_arch", + "dist_cpe", + "dist_pretty_name", + "arch_operation", + "repo_name", + "repo_key", + "repo_uri", + "fixed_in_version", + "vuln"."updater" + FROM + "vuln" + INNER JOIN "uo_vuln" ON ("vuln"."id" = "uo_vuln"."vuln") + INNER JOIN "latest_update_operations" ON ( + "latest_update_operations"."id" = "uo_vuln"."uo" + ) + WHERE + ( + "latest_update_operations"."kind" = 'vulnerability' + ) + AND + ( + "vuln"."updater" = $1 + ) + ` + results := []*claircore.Vulnerability{} + rows, err := s.pool.Query(ctx, query, updater) + if err != nil { + return nil, err + } + defer rows.Close() + + // unpack all returned rows into claircore.Vulnerability structs + for rows.Next() { + // fully allocate vuln struct + v := &claircore.Vulnerability{ + Package: &claircore.Package{}, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + } + + var id int64 + err := rows.Scan( + &id, + &v.Name, + &v.Description, + &v.Issued, + &v.Links, + &v.Severity, + &v.NormalizedSeverity, + &v.Package.Name, + &v.Package.Version, + &v.Package.Module, + &v.Package.Arch, + &v.Package.Kind, + &v.Dist.DID, + &v.Dist.Name, + &v.Dist.Version, + &v.Dist.VersionCodeName, + &v.Dist.VersionID, + &v.Dist.Arch, + &v.Dist.CPE, + &v.Dist.PrettyName, + &v.ArchOperation, + &v.Repo.Name, + &v.Repo.Key, + &v.Repo.URI, + &v.FixedInVersion, + &v.Updater, + ) + v.ID = strconv.FormatInt(id, 10) + if err != nil { + return nil, fmt.Errorf("failed to scan vulnerability: %v", err) + } + results = append(results, v) + } + + return results, nil +} diff --git a/datastore/postgres/matcher_store_test.go b/datastore/postgres/matcher_store_test.go new file mode 100644 index 000000000..596dd3f7f --- /dev/null +++ b/datastore/postgres/matcher_store_test.go @@ -0,0 +1,135 @@ +package postgres + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/quay/zlog" + + "github.com/quay/claircore" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/test/integration" + pgtest "github.com/quay/claircore/test/postgres" +) + +type latestVulnTestCase struct { + TestName string + Updater string + VulnCount int + FirstOp, SecondOp []*claircore.Vulnerability +} + +func TestGetLatestVulnerabilities(t *testing.T) { + integration.NeedDB(t) + ctx := zlog.Test(context.Background(), t) + + cases := []latestVulnTestCase{ + { + TestName: "test latest returns 3", + Updater: updater, + VulnCount: 3, + FirstOp: []*claircore.Vulnerability{ + { + Updater: updater, + Package: &claircore.Package{ + Name: "vi", + }, + }, + }, + SecondOp: []*claircore.Vulnerability{ + { + Updater: updater, + Package: &claircore.Package{ + Name: "vi", + }, + }, + { + Updater: updater, + Package: &claircore.Package{ + Name: "vim", + }, + }, + { + Updater: updater, + Package: &claircore.Package{ + Name: "nano", + }, + }, + }, + }, + { + TestName: "test latest doesn't return any", + Updater: updater, + VulnCount: 0, + FirstOp: []*claircore.Vulnerability{ + { + Updater: updater, + Package: &claircore.Package{ + Name: "grep", + }, + }, + { + Updater: updater, + Package: &claircore.Package{ + Name: "sed", + }, + }, + }, + SecondOp: []*claircore.Vulnerability{}, + }, + { + TestName: "test wrong updater", + Updater: "bad-updater", + VulnCount: 0, + FirstOp: []*claircore.Vulnerability{}, + SecondOp: []*claircore.Vulnerability{ + { + Updater: updater, + Package: &claircore.Package{ + Name: "python3", + }, + }, + { + Updater: updater, + Package: &claircore.Package{ + Name: "python3-crypto", + }, + }, + { + Updater: updater, + Package: &claircore.Package{ + Name: "python3-urllib3", + }, + }, + }, + }, + } + + // prepare DB + pool := pgtest.TestMatcherDB(ctx, t) + store := NewMatcherStore(pool) + + // run test cases + for _, tc := range cases { + t.Run(tc.TestName, func(t *testing.T) { + _, err := store.UpdateVulnerabilities(ctx, tc.Updater, driver.Fingerprint(uuid.New().String()), tc.FirstOp) + if err != nil { + t.Fatalf("failed to perform update for first op: %v", err) + } + _, err = store.UpdateVulnerabilities(ctx, tc.Updater, driver.Fingerprint(uuid.New().String()), tc.SecondOp) + if err != nil { + t.Fatalf("failed to perform update for second op: %v", err) + } + vulns, err := store.GetLatestVulnerabilities(ctx, tc.Updater) + if err != nil { + t.Fatalf("error getting latest vulns: %v", err) + } + + if l := len(vulns); l != tc.VulnCount { + t.Fatalf("got %d vulns, want %d", l, tc.VulnCount) + } + }) + + } +} diff --git a/datastore/updater.go b/datastore/updater.go index 2ec65f20b..e35f8b90c 100644 --- a/datastore/updater.go +++ b/datastore/updater.go @@ -60,4 +60,6 @@ type Updater interface { RecordUpdaterStatus(ctx context.Context, updaterName string, updateTime time.Time, fingerprint driver.Fingerprint, updaterError error) error // RecordUpdaterSetStatus records that all updaters from an updater set are up to date with vulnerabilities at this time RecordUpdaterSetStatus(ctx context.Context, updaterSet string, updateTime time.Time) error + + GetLatestVulnerabilities(ctx context.Context, updater string) ([]*claircore.Vulnerability, error) } diff --git a/go.mod b/go.mod index 3ba1e3eb3..366d906fc 100644 --- a/go.mod +++ b/go.mod @@ -26,11 +26,12 @@ require ( go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/trace v1.21.0 golang.org/x/crypto v0.15.0 + golang.org/x/exp v0.0.0-20231127185646-65229373498e golang.org/x/sync v0.5.0 golang.org/x/sys v0.14.0 golang.org/x/text v0.14.0 golang.org/x/time v0.4.0 - golang.org/x/tools v0.15.0 + golang.org/x/tools v0.16.0 modernc.org/sqlite v1.27.0 ) diff --git a/go.sum b/go.sum index 58ad1404f..6db1077d2 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -300,8 +302,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= -golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/libvuln/driver/updater.go b/libvuln/driver/updater.go index 77db41e40..8c704dde4 100644 --- a/libvuln/driver/updater.go +++ b/libvuln/driver/updater.go @@ -59,3 +59,7 @@ type ConfigUnmarshaler func(interface{}) error type Configurable interface { Configure(context.Context, ConfigUnmarshaler, *http.Client) error } + +type DeltaUpdater interface { + DeltaParse(context.Context, io.ReadCloser, []*claircore.Vulnerability) ([]*claircore.Vulnerability, error) +} diff --git a/libvuln/jsonblob/jsonblob.go b/libvuln/jsonblob/jsonblob.go index 236947405..c04c5d1e1 100644 --- a/libvuln/jsonblob/jsonblob.go +++ b/libvuln/jsonblob/jsonblob.go @@ -445,6 +445,11 @@ func (s *Store) RecordUpdaterSetStatus(ctx context.Context, updaterSet string, u return nil } +// GetLatestVulnerabilities is unimplemented +func (s *Store) GetLatestVulnerabilities(ctx context.Context, updater string) ([]*claircore.Vulnerability, error) { + return nil, nil +} + var bufPool sync.Pool func getBuf() []byte { diff --git a/libvuln/updates/manager.go b/libvuln/updates/manager.go index ef141d8cb..47a5a9340 100644 --- a/libvuln/updates/manager.go +++ b/libvuln/updates/manager.go @@ -342,6 +342,13 @@ func (m *Manager) driveUpdater(ctx context.Context, u driver.Updater) (err error return } + du, duOK := u.(driver.DeltaUpdater) + if duOK { + zlog.Info(ctx). + Str("updater", u.Name()). + Msg("found DeltaUpdater") + } + var ref uuid.UUID switch { case euOK: @@ -355,10 +362,24 @@ func (m *Manager) driveUpdater(ctx context.Context, u driver.Updater) (err error ref, err = m.store.UpdateEnrichments(ctx, name, newFP, ers) default: var vulns []*claircore.Vulnerability - vulns, err = u.Parse(ctx, vulnDB) - if err != nil { - err = fmt.Errorf("vulnerability database parse failed: %v", err) - return + switch { + case duOK: + oldVulns, getErr := m.store.GetLatestVulnerabilities(ctx, u.Name()) + if getErr != nil { + err = fmt.Errorf("failed to retrieve existing vulnerabilities: %v", getErr) + return + } + vulns, err = du.DeltaParse(ctx, vulnDB, oldVulns) + if err != nil { + err = fmt.Errorf("delta vulnerability database parse failed: %v", err) + return + } + default: + vulns, err = u.Parse(ctx, vulnDB) + if err != nil { + err = fmt.Errorf("vulnerability database parse failed: %v", err) + return + } } ref, err = m.store.UpdateVulnerabilities(ctx, name, newFP, vulns) diff --git a/libvuln/updates/manager_test.go b/libvuln/updates/manager_test.go new file mode 100644 index 000000000..9bb6ef45b --- /dev/null +++ b/libvuln/updates/manager_test.go @@ -0,0 +1,314 @@ +package updates + +import ( + "context" + "io" + "net/http" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/quay/zlog" + + "github.com/quay/claircore" + + "golang.org/x/exp/maps" + + "github.com/quay/claircore/datastore/postgres" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/pkg/ctxlock" + "github.com/quay/claircore/test/integration" + pgtest "github.com/quay/claircore/test/postgres" +) + +func TestDeltaUpdates(t *testing.T) { + integration.NeedDB(t) + ctx := zlog.Test(context.Background(), t) + facs := map[string]driver.UpdaterSetFactory{ + "delta": &Factory{ + vulnGetter: &vulnGetter{ + testVulns: testVulns, + }, + }, + } + + pool := pgtest.TestMatcherDB(ctx, t) + store := postgres.NewMatcherStore(pool) + + locks, err := ctxlock.New(ctx, pool) + if err != nil { + t.Fatalf("%v", err) + } + defer locks.Close(ctx) + + // Using default client here as a non-nil client is an error, + // it's never used. + mgr, err := NewManager(ctx, store, locks, http.DefaultClient, WithFactories(facs)) + if err != nil { + t.Fatalf("%v", err) + } + + for _, tc := range testVulns { + t.Run(tc.testName, func(t *testing.T) { + // force update + if err := mgr.Run(ctx); err != nil { + t.Fatalf("%v", err) + } + + vulns, err := store.GetLatestVulnerabilities(ctx, "test/delta/updater") + if err != nil { + t.Fatalf("%v", err) + } + if len(vulns) != tc.expectedNumber { + t.Fatalf("expecting %d vuln but got %d", tc.expectedNumber, len(vulns)) + } + }) + } + + vulns, err := store.GetLatestVulnerabilities(ctx, "test/delta/updater") + if err != nil { + t.Fatalf("%v", err) + } + + if !cmp.Equal(vulns, finalVulns, + cmpopts.IgnoreFields(claircore.Vulnerability{}, "ID"), // Depends on the DB + cmpopts.SortSlices(func(a, b interface{}) bool { + return a.(*claircore.Vulnerability).Name < b.(*claircore.Vulnerability).Name + })) { + t.Error(cmp.Diff(vulns, finalVulns)) + } +} + +type testUpdater struct { + vulnGetter *vulnGetter +} + +func (tu *testUpdater) Name() string { + return "test/delta/updater" +} + +func (tu *testUpdater) Fetch(context.Context, driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { + // NOOP + return nil, "", nil +} + +func (tu *testUpdater) Parse(context.Context, io.ReadCloser) ([]*claircore.Vulnerability, error) { + // NOOP + return nil, nil +} + +func (tu *testUpdater) DeltaParse(ctx context.Context, vulnUpdates io.ReadCloser, oldVulns []*claircore.Vulnerability) ([]*claircore.Vulnerability, error) { + newVulns := tu.vulnGetter.Get() + oldVulnMap := make(map[string]*claircore.Vulnerability, len(oldVulns)) + for _, v := range oldVulns { + oldVulnMap[v.Name] = v + } + maps.Copy(oldVulnMap, newVulns.vulns) + return maps.Values(oldVulnMap), nil +} + +type Factory struct { + vulnGetter *vulnGetter +} + +func (f *Factory) Configure(ctx context.Context, cf driver.ConfigUnmarshaler, c *http.Client) error { + return nil +} + +func (f *Factory) UpdaterSet(ctx context.Context) (s driver.UpdaterSet, err error) { + s = driver.NewUpdaterSet() + s.Add(&testUpdater{ + vulnGetter: f.vulnGetter, + }) + return s, nil +} + +type vulnGetter struct { + testVulns []*fetchedVulns + idx int +} + +func (vg *vulnGetter) Get() *fetchedVulns { + if vg.idx+1 > len(vg.testVulns) { + return nil + } + defer func() { vg.idx++ }() + return vg.testVulns[vg.idx] +} + +type fetchedVulns struct { + vulns map[string]*claircore.Vulnerability + expectedNumber int + testName string +} + +var testVulns = []*fetchedVulns{ + { + testName: "initial vuln", + vulns: map[string]*claircore.Vulnerability{ + "CVE-2023:123": { + Updater: "test/delta/updater", + Name: "CVE-2023:123", + Description: "bad things", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:123 https://moreprobs.io/CVE-2023:123", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 1, + }, + { + testName: "same vuln desc updated", + vulns: map[string]*claircore.Vulnerability{ + "CVE-2023:123": { + Updater: "test/delta/updater", + Name: "CVE-2023:123", + Description: "worse things", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:123 https://moreprobs.io/CVE-2023:123", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 1, + }, + { + testName: "two new vulns", + vulns: map[string]*claircore.Vulnerability{ + "CVE-2023:456": { + Updater: "test/delta/updater", + Name: "CVE-2023:456", + Description: "problems", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:456 https://moreprobs.io/CVE-2023:456", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + "CVE-2023:789": { + Updater: "test/delta/updater", + Name: "CVE-2023:789", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:789 https://moreprobs.io/CVE-2023:789", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 3, + }, + { + testName: "two updated one new", + vulns: map[string]*claircore.Vulnerability{ + "CVE-2023:456": { + Updater: "test/delta/updater", + Name: "CVE-2023:456", + Description: "problems 2", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:456 https://moreprobs.io/CVE-2023:456", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + "CVE-2023:789": { + Updater: "test/delta/updater", + Name: "CVE-2023:789", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:789 https://moreprobs.io/CVE-2023:789", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + "CVE-2023:101112": { + Updater: "test/delta/updater", + Name: "CVE-2023:101112", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:101112 https://moreprobs.io/CVE-2023:101112", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 4, + }, +} + +var finalVulns = []*claircore.Vulnerability{ + { + Updater: "test/delta/updater", + Name: "CVE-2023:123", + Description: "worse things", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:123 https://moreprobs.io/CVE-2023:123", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:456", + Description: "problems 2", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:456 https://moreprobs.io/CVE-2023:456", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:789", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:789 https://moreprobs.io/CVE-2023:789", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:101112", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:101112 https://moreprobs.io/CVE-2023:101112", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, +}