diff --git a/datastore/postgres/updatevulnerabilities.go b/datastore/postgres/updatevulnerabilities.go index 5fa09d1ef..45d4b4638 100644 --- a/datastore/postgres/updatevulnerabilities.go +++ b/datastore/postgres/updatevulnerabilities.go @@ -32,7 +32,7 @@ var ( Name: "updatevulnerabilities_total", Help: "Total number of database queries issued in the updateVulnerabilities method.", }, - []string{"query"}, + []string{"query", "is_delta"}, ) updateVulnerabilitiesDuration = promauto.NewHistogramVec( prometheus.HistogramOpts{ @@ -41,7 +41,7 @@ var ( Name: "updatevulnerabilities_duration_seconds", Help: "The duration of all queries issued in the updateVulnerabilities method", }, - []string{"query"}, + []string{"query", "is_delta"}, ) ) @@ -51,9 +51,51 @@ var ( // provided vulnerabilities and computes a diff comprising the removed // and added vulnerabilities for this UpdateOperation. func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string, fingerprint driver.Fingerprint, vulns []*claircore.Vulnerability) (uuid.UUID, error) { + ctx = zlog.ContextWithValues(ctx, "component", "datastore/postgres/MatcherStore.UpdateVulnerabilities") + return s.updateVulnerabilities(ctx, updater, fingerprint, vulns, nil, false) +} + +// DeltaUpdateVulnerabilities implements vulnstore.Updater. +// +// It is similar to UpdateVulnerabilities but support processing of +// partial data as opposed to needing an entire vulnerability database +// Order of operations: +// - Create a new UpdateOperation +// - Query existing vulnerabilities for the updater +// - Discount and vulnerabilities with newer updates and deleted vulnerabilities +// - Update the associated updateOperation for the remaining existing vulnerabilities +// - Insert the new vulnerabilities +// - Associate new vulnerabilities with new updateOperation +func (s *MatcherStore) DeltaUpdateVulnerabilities(ctx context.Context, updater string, fingerprint driver.Fingerprint, vulns []*claircore.Vulnerability, deletedVulns []string) (uuid.UUID, error) { + ctx = zlog.ContextWithValues(ctx, "component", "datastore/postgres/MatcherStore.DeltaUpdateVulnerabilities") + return s.updateVulnerabilities(ctx, updater, fingerprint, vulns, deletedVulns, true) +} + +func (s *MatcherStore) updateVulnerabilities(ctx context.Context, updater string, fingerprint driver.Fingerprint, vulns []*claircore.Vulnerability, deletedVulns []string, delta bool) (uuid.UUID, error) { const ( // Create makes a new update operation and returns the reference and ID. create = `INSERT INTO update_operation (updater, fingerprint, kind) VALUES ($1, $2, 'vulnerability') RETURNING id, ref;` + // Select existing vulnerabilities that are associated with the latest_update_operation. + selectExisting = ` + SELECT + "name", + "vuln"."id" + FROM + "vuln" + INNER JOIN "uo_vuln" ON ("vuln"."id" = "uo_vuln"."vuln") + INNER JOIN "latest_update_operations" ON ( + "latest_update_operations"."id" = "uo_vuln"."uo" + ) + WHERE + ( + "latest_update_operations"."kind" = 'vulnerability' + ) + AND + ( + "vuln"."updater" = $1 + )` + // assocExisting associates existing vulnerabilities with new update operations + assocExisting = `INSERT INTO uo_vuln (uo, vuln) VALUES ($1, $2) ON CONFLICT DO NOTHING;` // Insert attempts to create a new vulnerability. It fails silently. insert = ` INSERT INTO vuln ( @@ -82,20 +124,19 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string undo = `DELETE FROM update_operation WHERE id = $1;` refreshView = `REFRESH MATERIALIZED VIEW CONCURRENTLY latest_update_operations;` ) - ctx = zlog.ContextWithValues(ctx, "component", "internal/vulnstore/postgres/updateVulnerabilities") - var id uint64 + var uoID uint64 var ref uuid.UUID start := time.Now() - if err := s.pool.QueryRow(ctx, create, updater, string(fingerprint)).Scan(&id, &ref); err != nil { + if err := s.pool.QueryRow(ctx, create, updater, string(fingerprint)).Scan(&uoID, &ref); err != nil { return uuid.Nil, fmt.Errorf("failed to create update_operation: %w", err) } var success bool defer func() { if !success { - if _, err := s.pool.Exec(ctx, undo, id); err != nil { + if _, err := s.pool.Exec(ctx, undo, uoID); err != nil { zlog.Error(ctx). Err(err). Stringer("ref", ref). @@ -104,8 +145,8 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string } }() - updateVulnerabilitiesCounter.WithLabelValues("create").Add(1) - updateVulnerabilitiesDuration.WithLabelValues("create").Observe(time.Since(start).Seconds()) + updateVulnerabilitiesCounter.WithLabelValues("create", strconv.FormatBool(delta)).Add(1) + updateVulnerabilitiesDuration.WithLabelValues("create", strconv.FormatBool(delta)).Observe(time.Since(start).Seconds()) tx, err := s.pool.Begin(ctx) if err != nil { @@ -117,6 +158,69 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string Str("ref", ref.String()). Msg("update_operation created") + if delta { + ctx = zlog.ContextWithValues(ctx, "mode", "delta") + // Get existing vulns + // The reason this still works even though the new update_operation + // is already created is because the latest_update_operation view isn't updated until + // the end of this function. + start = time.Now() + rows, err := s.pool.Query(ctx, selectExisting, updater) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to get existing vulns: %w", err) + } + defer rows.Close() + updateVulnerabilitiesCounter.WithLabelValues("selectExisting", strconv.FormatBool(delta)).Add(1) + updateVulnerabilitiesDuration.WithLabelValues("selectExisting", strconv.FormatBool(delta)).Observe(time.Since(start).Seconds()) + + oldVulns := make(map[string][]string) + for rows.Next() { + var tmpID int64 + var ID, name string + err := rows.Scan( + &name, + &tmpID, + ) + + ID = strconv.FormatInt(tmpID, 10) + if err != nil { + return uuid.Nil, fmt.Errorf("failed to scan vulnerability: %w", err) + } + oldVulns[name] = append(oldVulns[name], ID) + } + if err := rows.Err(); err != nil { + return uuid.Nil, fmt.Errorf("error reading existing vulnerabilities: %w", err) + } + + if len(oldVulns) > 0 { + for _, v := range vulns { + // If we have an existing vuln in the new batch + // delete it from the oldVulns map so it doesn't + // get associated with the new update_operation. + delete(oldVulns, v.Name) + } + for _, delName := range deletedVulns { + // If we have an existing vuln that has been signaled + // as deleted by the updater then delete it so it doesn't + // get associated with the new update_operation. + delete(oldVulns, delName) + } + } + start = time.Now() + // Associate already existing vulnerabilities with new update_operation. + for _, vs := range oldVulns { + for _, vID := range vs { + _, err := tx.Exec(ctx, assocExisting, uoID, vID) + if err != nil { + return uuid.Nil, fmt.Errorf("could not update old vulnerability with new UO: %w", err) + } + } + } + updateVulnerabilitiesCounter.WithLabelValues("assocExisting", strconv.FormatBool(delta)).Add(float64(len(oldVulns))) + updateVulnerabilitiesDuration.WithLabelValues("assocExisting", strconv.FormatBool(delta)).Observe(time.Since(start).Seconds()) + + } + // batch insert vulnerabilities skipCt := 0 @@ -153,7 +257,7 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string return uuid.Nil, fmt.Errorf("failed to queue vulnerability: %w", err) } - if err := mBatcher.Queue(ctx, assoc, hashKind, hash, id); err != nil { + if err := mBatcher.Queue(ctx, assoc, hashKind, hash, uoID); err != nil { return uuid.Nil, fmt.Errorf("failed to queue association: %w", err) } } @@ -161,8 +265,8 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string return uuid.Nil, fmt.Errorf("failed to finish batch vulnerability insert: %w", err) } - updateVulnerabilitiesCounter.WithLabelValues("insert_batch").Add(1) - updateVulnerabilitiesDuration.WithLabelValues("insert_batch").Observe(time.Since(start).Seconds()) + updateVulnerabilitiesCounter.WithLabelValues("insert_batch", strconv.FormatBool(delta)).Add(1) + updateVulnerabilitiesDuration.WithLabelValues("insert_batch", strconv.FormatBool(delta)).Observe(time.Since(start).Seconds()) if err := tx.Commit(ctx); err != nil { return uuid.Nil, fmt.Errorf("failed to commit transaction: %w", err) diff --git a/datastore/postgres/updatevulnerabilities_test.go b/datastore/postgres/updatevulnerabilities_test.go new file mode 100644 index 000000000..69f218b90 --- /dev/null +++ b/datastore/postgres/updatevulnerabilities_test.go @@ -0,0 +1,323 @@ +package postgres + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/quay/zlog" + + "github.com/quay/claircore" + "github.com/quay/claircore/datastore" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/test/integration" + pgtest "github.com/quay/claircore/test/postgres" +) + +type latestVulnTestCase struct { + TestName string + Updater string + VulnCount int + FirstOp, SecondOp *op + Records []*claircore.IndexRecord +} + +type op struct { + vulns []*claircore.Vulnerability + deletedVulns []string +} + +func TestGetLatestVulnerabilities(t *testing.T) { + integration.NeedDB(t) + ctx := zlog.Test(context.Background(), t) + + cases := []latestVulnTestCase{ + { + TestName: "test initial op vuln still relevant", + Updater: updater, + VulnCount: 1, + FirstOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "vi", + }, + }, + }, + }, + SecondOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-456", + Package: &claircore.Package{ + Name: "vim", + }, + }, + { + Updater: updater, + Name: "CVE-789", + Package: &claircore.Package{ + Name: "nano", + }, + }, + }, + }, + Records: []*claircore.IndexRecord{ + { + Package: &claircore.Package{ + Name: "vi", + Source: &claircore.Package{ + Name: "vi", + Version: "v1.0.0", + }, + }, + }, + }, + }, + { + TestName: "test vuln is overwritten not duped", + Updater: updater, + VulnCount: 1, + FirstOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "grep", + }, + Severity: "BAD", + }, + { + Updater: updater, + Name: "CVE-456", + Package: &claircore.Package{ + Name: "sed", + }, + }, + }, + }, + SecondOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "grep", + }, + Severity: "NOT AS BAD AS WE THOUGHT", + }, + }, + }, + Records: []*claircore.IndexRecord{ + { + Package: &claircore.Package{ + Name: "grep", + Source: &claircore.Package{ + Name: "grep", + Version: "v1.0.0", + }, + }, + }, + }, + }, + { + TestName: "test multiple vulns from same CVE", + Updater: updater, + VulnCount: 1, + FirstOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "grep", + }, + Severity: "BAD", + }, + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "sed", + }, + Severity: "REALLY BAD", + }, + }, + }, + SecondOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "grep", + }, + Severity: "NOT AS BAD AS WE THOUGHT", + }, + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "sed", + }, + Severity: "FINE", + }, + }, + }, + Records: []*claircore.IndexRecord{ + { + Package: &claircore.Package{ + Name: "grep", + Source: &claircore.Package{ + Name: "grep", + Version: "v1.0.0", + }, + }, + }, + }, + }, + + { + TestName: "test two vulns same package different uo", + Updater: updater, + VulnCount: 2, + FirstOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-000", + Package: &claircore.Package{ + Name: "python3", + }, + }, + }, + }, + SecondOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-123", + Package: &claircore.Package{ + Name: "python3", + }, + }, + { + Updater: updater, + Name: "CVE-456", + Package: &claircore.Package{ + Name: "python3-crypto", + }, + }, + { + Updater: updater, + Name: "CVE-789", + Package: &claircore.Package{ + Name: "python3-urllib3", + }, + }, + }, + }, + Records: []*claircore.IndexRecord{ + { + Package: &claircore.Package{ + Name: "python3", + Source: &claircore.Package{ + Name: "python3", + Version: "v1.0.0", + }, + }, + }, + }, + }, + { + TestName: "test deleting vuln", + Updater: updater, + VulnCount: 0, + FirstOp: &op{ + deletedVulns: []string{}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-000", + Package: &claircore.Package{ + Name: "jq", + }, + }, + }, + }, + SecondOp: &op{ + deletedVulns: []string{"CVE-000"}, + vulns: []*claircore.Vulnerability{ + { + Updater: updater, + Name: "CVE-456", + Package: &claircore.Package{ + Name: "jq-libs", + }, + }, + { + Updater: updater, + Name: "CVE-789", + Package: &claircore.Package{ + Name: "jq-docs", + }, + }, + }, + }, + Records: []*claircore.IndexRecord{ + { + Package: &claircore.Package{ + Name: "jq", + Source: &claircore.Package{ + Name: "jq", + Version: "v1.0.0", + }, + }, + }, + }, + }, + } + + // prepare DB + pool := pgtest.TestMatcherDB(ctx, t) + store := NewMatcherStore(pool) + + // run test cases + for _, tc := range cases { + t.Run(tc.TestName, func(t *testing.T) { + _, err := store.DeltaUpdateVulnerabilities(ctx, tc.Updater, driver.Fingerprint(uuid.New().String()), tc.FirstOp.vulns, tc.FirstOp.deletedVulns) + if err != nil { + t.Fatalf("failed to perform update for first op: %v", err) + } + _, err = store.DeltaUpdateVulnerabilities(ctx, tc.Updater, driver.Fingerprint(uuid.New().String()), tc.SecondOp.vulns, tc.SecondOp.deletedVulns) + if err != nil { + t.Fatalf("failed to perform update for second op: %v", err) + } + + res, err := store.Get(ctx, tc.Records, datastore.GetOpts{}) + if err != nil { + t.Fatalf("failed to get vulns: %v", err) + } + ct := 0 + for _, vs := range res { + ct = ct + len(vs) + } + + if ct != tc.VulnCount { + t.Fatalf("got %d vulns, want %d", ct, tc.VulnCount) + } + }) + } +} diff --git a/datastore/updater.go b/datastore/updater.go index 2ec65f20b..705286427 100644 --- a/datastore/updater.go +++ b/datastore/updater.go @@ -19,6 +19,10 @@ type Updater interface { // vulnerabilities, and ensures vulnerabilities from previous updates are // not queried by clients. UpdateVulnerabilities(ctx context.Context, updater string, fingerprint driver.Fingerprint, vulns []*claircore.Vulnerability) (uuid.UUID, error) + // DeltaUpdateVulnerabilities creates a new UpdateOperation consisting of existing + // vulnerabilities and new vulnerabilities. It also takes an array of deleted + // vulnerability names which should no longer be available to query. + DeltaUpdateVulnerabilities(ctx context.Context, updater string, fingerprint driver.Fingerprint, vulns []*claircore.Vulnerability, deletedVulns []string) (uuid.UUID, error) // GetUpdateOperations returns a list of UpdateOperations in date descending // order for the given updaters. // diff --git a/libvuln/driver/updater.go b/libvuln/driver/updater.go index 77db41e40..4a06bdabb 100644 --- a/libvuln/driver/updater.go +++ b/libvuln/driver/updater.go @@ -59,3 +59,11 @@ type ConfigUnmarshaler func(interface{}) error type Configurable interface { Configure(context.Context, ConfigUnmarshaler, *http.Client) error } + +// DeltaUpdater is an interface that Updaters can implement to force the manager to call +// DeltaParse() in lieu of Parse() with the understanding that the resulting vulnerabilities +// represent a delta and not the entire vulnerability database. DeltaParse can also return +// a slice of strings that represent the names of deleted vulnerabilities. +type DeltaUpdater interface { + DeltaParse(ctx context.Context, contents io.ReadCloser) ([]*claircore.Vulnerability, []string, error) +} diff --git a/libvuln/jsonblob/jsonblob.go b/libvuln/jsonblob/jsonblob.go index 371966202..8fde6d313 100644 --- a/libvuln/jsonblob/jsonblob.go +++ b/libvuln/jsonblob/jsonblob.go @@ -452,6 +452,11 @@ func (s *Store) RecordUpdaterSetStatus(ctx context.Context, updaterSet string, u return nil } +// DeltaUpdateVulnerabilities is a noop +func (s *Store) DeltaUpdateVulnerabilities(ctx context.Context, updater string, fingerprint driver.Fingerprint, vulns []*claircore.Vulnerability, deleted []string) (uuid.UUID, error) { + return uuid.Nil, nil +} + var bufPool sync.Pool func getBuf() []byte { diff --git a/libvuln/updates/manager.go b/libvuln/updates/manager.go index ef141d8cb..28252d9d1 100644 --- a/libvuln/updates/manager.go +++ b/libvuln/updates/manager.go @@ -305,12 +305,18 @@ func (m *Manager) driveUpdater(ctx context.Context, u driver.Updater) (err error defer zlog.Info(ctx).Msg("finished update") uoKind := driver.VulnerabilityKind + // Do some assertions eu, euOK := u.(driver.EnrichmentUpdater) if euOK { zlog.Info(ctx). Msg("found EnrichmentUpdater") uoKind = driver.EnrichmentKind } + du, duOK := u.(driver.DeltaUpdater) + if duOK { + zlog.Info(ctx). + Msg("found DeltaUpdater") + } var prevFP driver.Fingerprint opmap, err := m.store.GetUpdateOperations(ctx, uoKind, name) @@ -355,13 +361,26 @@ func (m *Manager) driveUpdater(ctx context.Context, u driver.Updater) (err error ref, err = m.store.UpdateEnrichments(ctx, name, newFP, ers) default: var vulns []*claircore.Vulnerability - vulns, err = u.Parse(ctx, vulnDB) - if err != nil { - err = fmt.Errorf("vulnerability database parse failed: %v", err) - return - } + switch { + case duOK: + var deletedVulns []string + vulns, deletedVulns, err = du.DeltaParse(ctx, vulnDB) + if err != nil { + err = fmt.Errorf("vulnerability database delta parse failed: %v", err) + return + } + + ref, err = m.store.DeltaUpdateVulnerabilities(ctx, name, newFP, vulns, deletedVulns) - ref, err = m.store.UpdateVulnerabilities(ctx, name, newFP, vulns) + default: + vulns, err = u.Parse(ctx, vulnDB) + if err != nil { + err = fmt.Errorf("vulnerability database parse failed: %v", err) + return + } + + ref, err = m.store.UpdateVulnerabilities(ctx, name, newFP, vulns) + } } if err != nil { err = fmt.Errorf("failed to update: %v", err) diff --git a/libvuln/updates/manager_test.go b/libvuln/updates/manager_test.go new file mode 100644 index 000000000..c6371e8ec --- /dev/null +++ b/libvuln/updates/manager_test.go @@ -0,0 +1,413 @@ +package updates + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/quay/zlog" + + "github.com/quay/claircore" + + "github.com/quay/claircore/datastore/postgres" + "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/pkg/ctxlock" + "github.com/quay/claircore/test/integration" + pgtest "github.com/quay/claircore/test/postgres" +) + +func TestDeltaUpdates(t *testing.T) { + integration.NeedDB(t) + ctx := zlog.Test(context.Background(), t) + facs := map[string]driver.UpdaterSetFactory{ + "delta": &Factory{ + vulnGetter: &vulnGetter{ + testVulns: testVulns, + }, + }, + } + + pool := pgtest.TestMatcherDB(ctx, t) + store := postgres.NewMatcherStore(pool) + + locks, err := ctxlock.New(ctx, pool) + if err != nil { + t.Fatalf("%v", err) + } + defer locks.Close(ctx) + + // Using default client here as a non-nil client is an error, + // it's never used. + mgr, err := NewManager(ctx, store, locks, http.DefaultClient, WithFactories(facs)) + if err != nil { + t.Fatalf("%v", err) + } + + for _, tc := range testVulns { + t.Run(tc.testName, func(t *testing.T) { + // force update + if err := mgr.Run(ctx); err != nil { + t.Fatalf("%v", err) + } + + vulns, err := getLatestVulnerabilities(ctx, pool, "test/delta/updater") + if err != nil { + t.Fatalf("%v", err) + } + if len(vulns) != tc.expectedNumber { + t.Fatalf("expecting %d vuln but got %d", tc.expectedNumber, len(vulns)) + } + }) + } + + vulns, err := getLatestVulnerabilities(ctx, pool, "test/delta/updater") + if err != nil { + t.Fatalf("%v", err) + } + + if !cmp.Equal(vulns, finalVulns, + cmpopts.IgnoreFields(claircore.Vulnerability{}, "ID"), // Depends on the DB + cmpopts.SortSlices(func(a, b interface{}) bool { + return a.(*claircore.Vulnerability).Name < b.(*claircore.Vulnerability).Name + })) { + t.Error(cmp.Diff(vulns, finalVulns)) + } +} + +var _ driver.DeltaUpdater = (*testUpdater)(nil) +var _ driver.Updater = (*testUpdater)(nil) + +type testUpdater struct { + vulnGetter *vulnGetter +} + +func (tu *testUpdater) Name() string { + return "test/delta/updater" +} + +// DeltaFetch signals to the manager that we want to use DeltaFetch and store.DeltaUpdateVulnerabilities. +func (tu *testUpdater) Fetch(context.Context, driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) { + // NOOP + return nil, "", nil +} + +func (tu *testUpdater) Parse(ctx context.Context, vulnUpdates io.ReadCloser) ([]*claircore.Vulnerability, error) { + // NOOP + return nil, nil +} + +func (tu *testUpdater) DeltaParse(ctx context.Context, vulnUpdates io.ReadCloser) ([]*claircore.Vulnerability, []string, error) { + newVulns := tu.vulnGetter.Get() + return newVulns.vulns, []string{}, nil +} + +type Factory struct { + vulnGetter *vulnGetter +} + +func (f *Factory) Configure(ctx context.Context, cf driver.ConfigUnmarshaler, c *http.Client) error { + return nil +} + +func (f *Factory) UpdaterSet(ctx context.Context) (s driver.UpdaterSet, err error) { + s = driver.NewUpdaterSet() + s.Add(&testUpdater{ + vulnGetter: f.vulnGetter, + }) + return s, nil +} + +type vulnGetter struct { + testVulns []*fetchedVulns + idx int +} + +func (vg *vulnGetter) Get() *fetchedVulns { + if vg.idx+1 > len(vg.testVulns) { + return nil + } + defer func() { vg.idx++ }() + return vg.testVulns[vg.idx] +} + +type fetchedVulns struct { + vulns []*claircore.Vulnerability + expectedNumber int + testName string +} + +var testVulns = []*fetchedVulns{ + { + testName: "initial vuln", + vulns: []*claircore.Vulnerability{ + { + Updater: "test/delta/updater", + Name: "CVE-2023:123", + Description: "bad things", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:123 https://moreprobs.io/CVE-2023:123", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 1, + }, + { + testName: "same vuln desc updated", + vulns: []*claircore.Vulnerability{ + { + Updater: "test/delta/updater", + Name: "CVE-2023:123", + Description: "worse things", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:123 https://moreprobs.io/CVE-2023:123", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 1, + }, + { + testName: "two new vulns", + vulns: []*claircore.Vulnerability{ + { + Updater: "test/delta/updater", + Name: "CVE-2023:456", + Description: "problems", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:456 https://moreprobs.io/CVE-2023:456", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:789", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:789 https://moreprobs.io/CVE-2023:789", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 3, + }, + { + testName: "two updated one new", + vulns: []*claircore.Vulnerability{ + { + Updater: "test/delta/updater", + Name: "CVE-2023:456", + Description: "problems 2", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:456 https://moreprobs.io/CVE-2023:456", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:789", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:789 https://moreprobs.io/CVE-2023:789", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:101112", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:101112 https://moreprobs.io/CVE-2023:101112", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + }, + }, + expectedNumber: 4, + }, +} + +var finalVulns = []*claircore.Vulnerability{ + { + Updater: "test/delta/updater", + Name: "CVE-2023:123", + Description: "worse things", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:123 https://moreprobs.io/CVE-2023:123", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:456", + Description: "problems 2", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:456 https://moreprobs.io/CVE-2023:456", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:789", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:789 https://moreprobs.io/CVE-2023:789", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, + { + Updater: "test/delta/updater", + Name: "CVE-2023:101112", + Description: "problems again", + Issued: time.Time{}, + Links: "https://ohno.com/CVE-2023:101112 https://moreprobs.io/CVE-2023:101112", + Severity: "Very Medium", + NormalizedSeverity: claircore.Medium, + Package: &claircore.Package{ + Name: "blah", + }, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + }, +} + +func getLatestVulnerabilities(ctx context.Context, pool *pgxpool.Pool, updater string) ([]*claircore.Vulnerability, error) { + query := ` + SELECT + "vuln"."id", + "name", + "description", + "issued", + "links", + "severity", + "normalized_severity", + "package_name", + "package_version", + "package_module", + "package_arch", + "package_kind", + "dist_id", + "dist_name", + "dist_version", + "dist_version_code_name", + "dist_version_id", + "dist_arch", + "dist_cpe", + "dist_pretty_name", + "arch_operation", + "repo_name", + "repo_key", + "repo_uri", + "fixed_in_version", + "vuln"."updater" + FROM + "vuln" + INNER JOIN "uo_vuln" ON ("vuln"."id" = "uo_vuln"."vuln") + INNER JOIN "latest_update_operations" ON ( + "latest_update_operations"."id" = "uo_vuln"."uo" + ) + WHERE + ( + "latest_update_operations"."kind" = 'vulnerability' + ) + AND + ( + "vuln"."updater" = $1 + ) + ` + results := []*claircore.Vulnerability{} + rows, err := pool.Query(ctx, query, updater) + if err != nil { + return nil, err + } + defer rows.Close() + + // unpack all returned rows into claircore.Vulnerability structs + for rows.Next() { + // fully allocate vuln struct + v := &claircore.Vulnerability{ + Package: &claircore.Package{}, + Dist: &claircore.Distribution{}, + Repo: &claircore.Repository{}, + } + + var id int64 + err := rows.Scan( + &id, + &v.Name, + &v.Description, + &v.Issued, + &v.Links, + &v.Severity, + &v.NormalizedSeverity, + &v.Package.Name, + &v.Package.Version, + &v.Package.Module, + &v.Package.Arch, + &v.Package.Kind, + &v.Dist.DID, + &v.Dist.Name, + &v.Dist.Version, + &v.Dist.VersionCodeName, + &v.Dist.VersionID, + &v.Dist.Arch, + &v.Dist.CPE, + &v.Dist.PrettyName, + &v.ArchOperation, + &v.Repo.Name, + &v.Repo.Key, + &v.Repo.URI, + &v.FixedInVersion, + &v.Updater, + ) + v.ID = strconv.FormatInt(id, 10) + if err != nil { + return nil, fmt.Errorf("failed to scan vulnerability: %v", err) + } + results = append(results, v) + } + + return results, nil +}