diff --git a/datastore/postgres/enrichment.go b/datastore/postgres/enrichment.go index 0a7e39907..6ebbd7cf8 100644 --- a/datastore/postgres/enrichment.go +++ b/datastore/postgres/enrichment.go @@ -106,6 +106,7 @@ VALUES ON CONFLICT DO NOTHING;` + refreshView = `REFRESH MATERIALIZED VIEW latest_update_operations;` ) ctx = zlog.ContextWithValues(ctx, "component", "datastore/postgres/UpdateEnrichments") @@ -154,6 +155,9 @@ DO if err := tx.Commit(ctx); err != nil { return uuid.Nil, fmt.Errorf("failed to commit transaction: %w", err) } + if _, err = s.pool.Exec(ctx, refreshView); err != nil { + return uuid.Nil, fmt.Errorf("could not refresh latest_update_operations: %w", err) + } zlog.Debug(ctx). Stringer("ref", ref). Int("inserted", len(es)). @@ -178,11 +182,14 @@ WITH latest AS ( SELECT - max(id) AS id + id FROM - update_operation + latest_update_operations WHERE updater = $1 + AND + kind = 'enrichment' + LIMIT 1 ) SELECT e.tags, e.data diff --git a/datastore/postgres/get.go b/datastore/postgres/get.go index e616713d2..90bf66fb9 100644 --- a/datastore/postgres/get.go +++ b/datastore/postgres/get.go @@ -44,7 +44,7 @@ func (s *MatcherStore) Get(ctx context.Context, records []*claircore.IndexRecord return nil, err } defer tx.Rollback(ctx) - latestUpdatesQuery := "SELECT DISTINCT ON (updater) id FROM update_operation ORDER BY updater, id DESC" + latestUpdatesQuery := "SELECT id FROM latest_update_operations WHERE kind = 'vulnerability';" rows, err := tx.Query(ctx, latestUpdatesQuery) if err != nil { return nil, err diff --git a/datastore/postgres/migrations/matcher/11-add-update_operation-mv.sql b/datastore/postgres/migrations/matcher/11-add-update_operation-mv.sql new file mode 100644 index 000000000..a713337ff --- /dev/null +++ b/datastore/postgres/migrations/matcher/11-add-update_operation-mv.sql @@ -0,0 +1,3 @@ +-- Create materialized view that maintains the lastest update_operation id per updater. +CREATE MATERIALIZED VIEW IF NOT exists latest_update_operations AS +SELECT DISTINCT ON (updater) id, kind, updater FROM update_operation ORDER BY updater, id DESC; diff --git a/datastore/postgres/migrations/migrations.go b/datastore/postgres/migrations/migrations.go index 93b40fc9f..a53e8e175 100644 --- a/datastore/postgres/migrations/migrations.go +++ b/datastore/postgres/migrations/migrations.go @@ -100,4 +100,8 @@ var MatcherMigrations = []migrate.Migration{ ID: 10, Up: runFile("matcher/10-delete-osv.sql"), }, + { + ID: 11, + Up: runFile("matcher/11-add-update_operation-mv.sql"), + }, } diff --git a/datastore/postgres/querybuilder.go b/datastore/postgres/querybuilder.go index 8bde73ab8..a741e6d4f 100644 --- a/datastore/postgres/querybuilder.go +++ b/datastore/postgres/querybuilder.go @@ -19,6 +19,11 @@ func buildGetQuery(record *claircore.IndexRecord, opts *datastore.GetOpts, uos [ psql := goqu.Dialect("postgres") exps := []goqu.Expression{} + // Check that uos isn't empty, if it is it will result in a query error + if len(uos) == 0 { + return "", fmt.Errorf("cannot query without update_operation IDs") + } + // Add package name as first condition in query. if record.Package.Name == "" { return "", fmt.Errorf("IndexRecord must provide a Package.Name") diff --git a/datastore/postgres/updatevulnerabilities.go b/datastore/postgres/updatevulnerabilities.go index d152af90c..5644b7b96 100644 --- a/datastore/postgres/updatevulnerabilities.go +++ b/datastore/postgres/updatevulnerabilities.go @@ -79,7 +79,8 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string $3, (SELECT id FROM vuln WHERE hash_kind = $1 AND hash = $2)) ON CONFLICT DO NOTHING;` - undo = `DELETE FROM update_operation WHERE id = $1;` + undo = `DELETE FROM update_operation WHERE id = $1;` + refreshView = `REFRESH MATERIALIZED VIEW latest_update_operations;` ) ctx = zlog.ContextWithValues(ctx, "component", "internal/vulnstore/postgres/updateVulnerabilities") @@ -166,6 +167,10 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string if err := tx.Commit(ctx); err != nil { return uuid.Nil, fmt.Errorf("failed to commit transaction: %w", err) } + if _, err = s.pool.Exec(ctx, refreshView); err != nil { + return uuid.Nil, fmt.Errorf("could not refresh latest_update_operations: %w", err) + } + success = true zlog.Debug(ctx). Str("ref", ref.String()).