matcher: Use materialized view to speed up queries to update_operations

Querying the update_operations table for the latest operations
creates a lot of CPU load on the DB. When the data is pulled into
a materialized view we can take the CPU hit when the update_operation
is being created as opposed to every time we receive a vulnerability
report request.

Signed-off-by: crozzy <[email protected]>

datastore/postgres/enrichment.go

@@ -106,6 +106,7 @@ VALUES
 ON CONFLICT
 DO
 	NOTHING;`
+		refreshView = `REFRESH MATERIALIZED VIEW latest_update_operations;`
 	)
 	ctx = zlog.ContextWithValues(ctx, "component", "datastore/postgres/UpdateEnrichments")
 
@@ -154,6 +155,9 @@ DO
 	if err := tx.Commit(ctx); err != nil {
 		return uuid.Nil, fmt.Errorf("failed to commit transaction: %w", err)
 	}
+	if _, err = s.pool.Exec(ctx, refreshView); err != nil {
+		return uuid.Nil, fmt.Errorf("could not refresh latest_update_operations: %w", err)
+	}
 	zlog.Debug(ctx).
 		Stringer("ref", ref).
 		Int("inserted", len(es)).
@@ -178,11 +182,14 @@ WITH
 	latest
 		AS (
 			SELECT
-				max(id) AS id
+				id
 			FROM
-				update_operation
+				latest_update_operations
 			WHERE
 				updater = $1
+			AND
+				kind = 'enrichment'
+			LIMIT 1
 		)
 SELECT
 	e.tags, e.data

datastore/postgres/get.go

@@ -44,7 +44,7 @@ func (s *MatcherStore) Get(ctx context.Context, records []*claircore.IndexRecord
 		return nil, err
 	}
 	defer tx.Rollback(ctx)
-	latestUpdatesQuery := "SELECT DISTINCT ON (updater) id FROM update_operation ORDER BY updater, id DESC"
+	latestUpdatesQuery := "SELECT id FROM latest_update_operations WHERE kind = 'vulnerability';"
 	rows, err := tx.Query(ctx, latestUpdatesQuery)
 	if err != nil {
 		return nil, err

datastore/postgres/migrations/matcher/11-add-update_operation-mv.sql

@@ -0,0 +1,3 @@
+-- Create materialized view that maintains the lastest update_operation id per updater.
+CREATE MATERIALIZED VIEW IF NOT exists latest_update_operations AS
+SELECT DISTINCT ON (updater) id, kind, updater FROM update_operation ORDER BY updater, id DESC;

datastore/postgres/migrations/migrations.go

@@ -100,4 +100,8 @@ var MatcherMigrations = []migrate.Migration{
 		ID: 10,
 		Up: runFile("matcher/10-delete-osv.sql"),
 	},
+	{
+		ID: 11,
+		Up: runFile("matcher/11-add-update_operation-mv.sql"),
+	},
 }

datastore/postgres/querybuilder.go

@@ -19,6 +19,11 @@ func buildGetQuery(record *claircore.IndexRecord, opts *datastore.GetOpts, uos [
 	psql := goqu.Dialect("postgres")
 	exps := []goqu.Expression{}
 
+	// Check that uos isn't empty, if it is it will result in a query error
+	if len(uos) == 0 {
+		return "", fmt.Errorf("cannot query without update_operation IDs")
+	}
+
 	// Add package name as first condition in query.
 	if record.Package.Name == "" {
 		return "", fmt.Errorf("IndexRecord must provide a Package.Name")

datastore/postgres/updatevulnerabilities.go

@@ -79,7 +79,8 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string
 			$3,
 			(SELECT id FROM vuln WHERE hash_kind = $1 AND hash = $2))
 		ON CONFLICT DO NOTHING;`
-		undo = `DELETE FROM update_operation WHERE id = $1;`
+		undo        = `DELETE FROM update_operation WHERE id = $1;`
+		refreshView = `REFRESH MATERIALIZED VIEW latest_update_operations;`
 	)
 	ctx = zlog.ContextWithValues(ctx, "component", "internal/vulnstore/postgres/updateVulnerabilities")
 
@@ -166,6 +167,10 @@ func (s *MatcherStore) UpdateVulnerabilities(ctx context.Context, updater string
 	if err := tx.Commit(ctx); err != nil {
 		return uuid.Nil, fmt.Errorf("failed to commit transaction: %w", err)
 	}
+	if _, err = s.pool.Exec(ctx, refreshView); err != nil {
+		return uuid.Nil, fmt.Errorf("could not refresh latest_update_operations: %w", err)
+	}
+
 	success = true
 	zlog.Debug(ctx).
 		Str("ref", ref.String()).