updater: Support delta updates

TODO: message

Signed-off-by: crozzy <[email protected]>

datastore/postgres/matcher_store.go

@@ -3,6 +3,7 @@ package postgres
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/google/uuid"
@@ -11,6 +12,8 @@ import (
 	"github.com/quay/zlog"
 	"github.com/remind101/migrate"
 
+	"github.com/quay/claircore"
+
 	"github.com/quay/claircore/datastore"
 	"github.com/quay/claircore/datastore/postgres/migrations"
 	"github.com/quay/claircore/libvuln/driver"
@@ -83,3 +86,102 @@ func (s *MatcherStore) RecordUpdaterStatus(ctx context.Context, updaterName stri
 func (s *MatcherStore) RecordUpdaterSetStatus(ctx context.Context, updaterSet string, updateTime time.Time) error {
 	return recordUpdaterSetStatus(ctx, s.pool, updaterSet, updateTime)
 }
+
+func (s *MatcherStore) GetLatestVulnerabilities(ctx context.Context, updater string) ([]*claircore.Vulnerability, error) {
+	query := `
+		SELECT 
+			"vuln"."id", 
+			"name", 
+			"description", 
+			"issued", 
+			"links", 
+			"severity", 
+			"normalized_severity", 
+			"package_name", 
+			"package_version", 
+			"package_module", 
+			"package_arch", 
+			"package_kind", 
+			"dist_id", 
+			"dist_name", 
+			"dist_version", 
+			"dist_version_code_name", 
+			"dist_version_id", 
+			"dist_arch", 
+			"dist_cpe", 
+			"dist_pretty_name", 
+			"arch_operation", 
+			"repo_name", 
+			"repo_key", 
+			"repo_uri", 
+			"fixed_in_version", 
+			"vuln"."updater" 
+		FROM 
+			"vuln" 
+			INNER JOIN "uo_vuln" ON ("vuln"."id" = "uo_vuln"."vuln") 
+			INNER JOIN "latest_update_operations" ON (
+			"latest_update_operations"."id" = "uo_vuln"."uo"
+			) 
+		WHERE 
+			(
+			"latest_update_operations"."kind" = 'vulnerability'
+			)
+		AND
+			(
+			"vuln"."updater" = $1
+			)
+	`
+	results := []*claircore.Vulnerability{}
+	rows, err := s.pool.Query(ctx, query, updater)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	// unpack all returned rows into claircore.Vulnerability structs
+	for rows.Next() {
+		// fully allocate vuln struct
+		v := &claircore.Vulnerability{
+			Package: &claircore.Package{},
+			Dist:    &claircore.Distribution{},
+			Repo:    &claircore.Repository{},
+		}
+
+		var id int64
+		err := rows.Scan(
+			&id,
+			&v.Name,
+			&v.Description,
+			&v.Issued,
+			&v.Links,
+			&v.Severity,
+			&v.NormalizedSeverity,
+			&v.Package.Name,
+			&v.Package.Version,
+			&v.Package.Module,
+			&v.Package.Arch,
+			&v.Package.Kind,
+			&v.Dist.DID,
+			&v.Dist.Name,
+			&v.Dist.Version,
+			&v.Dist.VersionCodeName,
+			&v.Dist.VersionID,
+			&v.Dist.Arch,
+			&v.Dist.CPE,
+			&v.Dist.PrettyName,
+			&v.ArchOperation,
+			&v.Repo.Name,
+			&v.Repo.Key,
+			&v.Repo.URI,
+			&v.FixedInVersion,
+			&v.Updater,
+		)
+		v.ID = strconv.FormatInt(id, 10)
+		if err != nil {
+			return nil, fmt.Errorf("failed to scan vulnerability: %v", err)
+		}
+		results = append(results, v)
+	}
+
+	return results, nil
+}

datastore/updater.go

@@ -60,4 +60,6 @@ type Updater interface {
 	RecordUpdaterStatus(ctx context.Context, updaterName string, updateTime time.Time, fingerprint driver.Fingerprint, updaterError error) error
 	// RecordUpdaterSetStatus records that all updaters from an updater set are up to date with vulnerabilities at this time
 	RecordUpdaterSetStatus(ctx context.Context, updaterSet string, updateTime time.Time) error
+
+	GetLatestVulnerabilities(ctx context.Context, updater string) ([]*claircore.Vulnerability, error)
 }

go.mod

@@ -26,11 +26,12 @@ require (
 	go.opentelemetry.io/otel v1.21.0
 	go.opentelemetry.io/otel/trace v1.21.0
 	golang.org/x/crypto v0.14.0
+	golang.org/x/exp v0.0.0-20231127185646-65229373498e
 	golang.org/x/sync v0.5.0
 	golang.org/x/sys v0.14.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.4.0
-	golang.org/x/tools v0.15.0
+	golang.org/x/tools v0.16.0
 	modernc.org/sqlite v1.27.0
 )
 

go.sum

@@ -227,6 +227,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
+golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -300,8 +302,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
-golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
+golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
+golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

libvuln/driver/updater.go

@@ -59,3 +59,7 @@ type ConfigUnmarshaler func(interface{}) error
 type Configurable interface {
 	Configure(context.Context, ConfigUnmarshaler, *http.Client) error
 }
+
+type DeltaUpdater interface {
+	DeltaParse(context.Context, io.ReadCloser, []*claircore.Vulnerability) ([]*claircore.Vulnerability, error)
+}

libvuln/jsonblob/jsonblob.go

@@ -298,3 +298,8 @@ func (s *Store) RecordUpdaterStatus(ctx context.Context, updaterName string, upd
 func (s *Store) RecordUpdaterSetStatus(ctx context.Context, updaterSet string, updateTime time.Time) error {
 	return nil
 }
+
+// GetLatestVulnerabilities is unimplemented
+func (s *Store) GetLatestVulnerabilities(ctx context.Context, updater string) ([]*claircore.Vulnerability, error) {
+	return nil, nil
+}

libvuln/updates/manager.go

@@ -342,6 +342,13 @@ func (m *Manager) driveUpdater(ctx context.Context, u driver.Updater) (err error
 		return
 	}
 
+	du, duOK := u.(driver.DeltaUpdater)
+	if duOK {
+		zlog.Info(ctx).
+			Str("updater", u.Name()).
+			Msg("found DeltaUpdater")
+	}
+
 	var ref uuid.UUID
 	switch {
 	case euOK:
@@ -355,10 +362,24 @@ func (m *Manager) driveUpdater(ctx context.Context, u driver.Updater) (err error
 		ref, err = m.store.UpdateEnrichments(ctx, name, newFP, ers)
 	default:
 		var vulns []*claircore.Vulnerability
-		vulns, err = u.Parse(ctx, vulnDB)
-		if err != nil {
-			err = fmt.Errorf("vulnerability database parse failed: %v", err)
-			return
+		switch {
+		case duOK:
+			oldVulns, getErr := m.store.GetLatestVulnerabilities(ctx, u.Name())
+			if getErr != nil {
+				err = fmt.Errorf("failed to retrieve existing vulnerabilities: %v", getErr)
+				return
+			}
+			vulns, err = du.DeltaParse(ctx, vulnDB, oldVulns)
+			if err != nil {
+				err = fmt.Errorf("delta vulnerability database parse failed: %v", err)
+				return
+			}
+		default:
+			vulns, err = u.Parse(ctx, vulnDB)
+			if err != nil {
+				err = fmt.Errorf("vulnerability database parse failed: %v", err)
+				return
+			}
 		}
 
 		ref, err = m.store.UpdateVulnerabilities(ctx, name, newFP, vulns)