Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(scanner): add keppel scanner #151

Merged
merged 8 commits into from
Aug 22, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
641 changes: 641 additions & 0 deletions scanner/keppel/client/generated.go

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions scanner/keppel/client/genqlient.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

# Default genqlient config; for full documentation see:
# https://github.com/Khan/genqlient/blob/main/docs/genqlient.yaml
schema: ../../../internal/api/graphql/graph/schema/*.graphqls
operations:
- ./query/*.graphql
generated: generated.go
package: client
use_struct_references: true
15 changes: 15 additions & 0 deletions scanner/keppel/client/query/addComponentVersionToIssue.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation AddComponentVersionToIssue($issueId: ID!, $componentVersionId: ID!) {
# @genqlient(typename: "Issue")
addComponentVersionToIssue (
issueId: $issueId,
componentVersionId: $componentVersionId
) {
id
primaryName
description
type
}
}
13 changes: 13 additions & 0 deletions scanner/keppel/client/query/createComponent.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation CreateComponent($input: ComponentInput!) {
# @genqlient(typename: "Component")
createComponent (
input: $input
) {
id
name
type
}
}
13 changes: 13 additions & 0 deletions scanner/keppel/client/query/createComponentVersion.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation CreateComponentVersion($input: ComponentVersionInput!) {
# @genqlient(typename: "ComponentVersion")
createComponentVersion (
input: $input
) {
id
version
componentId
}
}
14 changes: 14 additions & 0 deletions scanner/keppel/client/query/createIssue.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

mutation CreateIssue($input: IssueInput!) {
# @genqlient(typename: "Issue")
createIssue (
input: $input
) {
id
primaryName
description
type
}
}
19 changes: 19 additions & 0 deletions scanner/keppel/client/query/listComponentVersions.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

query ListComponentVersions($filter: ComponentVersionFilter, $first: Int) {
# @genqlient(typename: "ComponentVersionConnection")
ComponentVersions (
filter: $filter,
first: $first,
) {
edges {
# @genqlient(typename: "ComponentVersion")
node {
id
version
componentId
}
}
}
}
19 changes: 19 additions & 0 deletions scanner/keppel/client/query/listComponents.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

query ListComponents($filter: ComponentFilter, $first: Int) {
# @genqlient(typename: "ComponentConnection")
Components (
filter: $filter,
first: $first,
) {
edges {
# @genqlient(typename: "Component")
node {
id
name
type
}
}
}
}
20 changes: 20 additions & 0 deletions scanner/keppel/client/query/listIssues.graphql
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
# SPDX-License-Identifier: Apache-2.0

query ListIssues($filter: IssueFilter, $first: Int) {
# @genqlient(typename: "IssueConnection")
Issues (
filter: $filter,
first: $first,
) {
edges {
# @genqlient(typename: "Issue")
node {
id
primaryName
description
type
}
}
}
}
34 changes: 33 additions & 1 deletion scanner/keppel/go.mod
Original file line number Diff line number Diff line change
@@ -1,3 +1,35 @@
module github.com/cloudoperators/heureka/scanners/keppel
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a policy when to use github.com and when to use github.wdf.sap.com (like here: https://github.com/cloudoperators/heureka/blob/scanner_nvd/scanner/nvd/go.mod#L1) ?


go 1.22.6
go 1.22.6

require (
github.com/aquasecurity/trivy v0.54.1
github.com/gophercloud/gophercloud v1.14.0
github.com/gophercloud/gophercloud/v2 v2.1.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/pkg/errors v0.9.1
)

require (
github.com/Khan/genqlient v0.7.0 // indirect
github.com/agnivade/levenshtein v1.1.1 // indirect
github.com/alexflint/go-arg v1.4.2 // indirect
github.com/alexflint/go-scalar v1.0.0 // indirect
github.com/vektah/gqlparser/v2 v2.5.11 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/tools v0.23.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

require (
github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 // indirect
github.com/google/go-containerregistry v0.20.1 // indirect
github.com/machinebox/graphql v0.2.2
github.com/package-url/packageurl-go v0.1.3 // indirect
github.com/samber/lo v1.46.0 // indirect
github.com/sirupsen/logrus v1.9.3
golang.org/x/text v0.16.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
)
65 changes: 65 additions & 0 deletions scanner/keppel/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w=
github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM=
github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
github.com/alexflint/go-arg v1.4.2 h1:lDWZAXxpAnZUq4qwb86p/3rIJJ2Li81EoMbTMujhVa0=
github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM=
github.com/alexflint/go-scalar v1.0.0 h1:NGupf1XV/Xb04wXskDFzS0KWOLH632W/EO4fAFi+A70=
github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw=
github.com/aquasecurity/trivy v0.54.1 h1:/uNCF06PfdC69v5n3Zh4fXVf0xmXBml0c/ergf066SQ=
github.com/aquasecurity/trivy v0.54.1/go.mod h1:i5S54WUtOEN9egFF0AHsxq6XT7QD11n9pSmIXhMJV0g=
github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 h1:6/T8sFdNVG/AwOGoK6X55h7hF7LYqK8bsuPz8iEz8jM=
github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04/go.mod h1:0T6oy2t1Iedt+yi3Ml5cpOYp5FZT4MI1/mx+3p+PIs8=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0=
github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8=
github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
github.com/gophercloud/gophercloud/v2 v2.1.0 h1:91p6c+uMckXyx39nSIYjDirDBnPVFQq0q1njLNPX+NY=
github.com/gophercloud/gophercloud/v2 v2.1.0/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo=
github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA=
github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs=
github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ=
github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/vektah/gqlparser/v2 v2.5.11 h1:JJxLtXIoN7+3x6MBdtIP59TP1RANnY7pXOaDnADQSf8=
github.com/vektah/gqlparser/v2 v2.5.11/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc=
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
143 changes: 141 additions & 2 deletions scanner/keppel/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,148 @@
package main

import (
"fmt"
"os"
"sync"

"github.com/cloudoperators/heureka/scanners/keppel/client"
"github.com/cloudoperators/heureka/scanners/keppel/models"
"github.com/cloudoperators/heureka/scanners/keppel/processor"
"github.com/cloudoperators/heureka/scanners/keppel/scanner"
"github.com/kelseyhightower/envconfig"
log "github.com/sirupsen/logrus"
)

type Config struct {
LogLevel string `envconfig:"LOG_LEVEL" default:"debug" required:"true" json:"-"`
}

func init() {
// Log as JSON instead of the default ASCII formatter.
log.SetFormatter(&log.JSONFormatter{})

// Output to stdout instead of the default stderr
// Can be any io.Writer, see below for File example
log.SetOutput(os.Stdout)

var cfg Config
err := envconfig.Process("heureka", &cfg)
if err != nil {
log.WithError(err).Fatal("Error while reading env config")
}

level, err := log.ParseLevel(cfg.LogLevel)

if err != nil {
log.WithError(err).Fatal("Error while parsing log level")
}

// Only log the warning severity or above.
log.SetLevel(level)
}

func main() {
fmt.Println("This is the Keppel scanner")
var wg sync.WaitGroup
var scannerCfg scanner.Config
err := envconfig.Process("heureka", &scannerCfg)
if err != nil {
log.WithError(err).Fatal("Error while reading env config for scanner")
}

var processorCfg processor.Config
err = envconfig.Process("heureka", &processorCfg)
if err != nil {
log.WithError(err).Fatal("Error while reading env config for processor")
}

keppelScanner := scanner.NewScanner(scannerCfg)
keppelProcessor := processor.NewProcessor(processorCfg)

err = keppelScanner.Setup()
if err != nil {
log.WithError(err).Fatal("Error during scanner setup")
}
accounts, err := keppelScanner.ListAccounts()
if err != nil {
log.WithError(err).Fatal("Error during ListAccounts")
}

wg.Add(len(accounts))

for _, account := range accounts {
go HandleAccount(account, keppelScanner, keppelProcessor, &wg)
}

wg.Wait()
}

func HandleAccount(account models.Account, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, wg *sync.WaitGroup) error {
defer wg.Done()
repositories, err := keppelScanner.ListRepositories(account.Name)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
}).WithError(err).Error("Error during ProcessRepository")
return err
}

for _, repository := range repositories {
HandleRepository(account, repository, keppelScanner, keppelProcessor)
}

return nil
}

func HandleRepository(account models.Account, repository models.Repository, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) {
component, err := keppelProcessor.ProcessRepository(repository)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ProcessRepository")
component, err = keppelProcessor.GetComponent(repository.Name)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetComponent")
}
}

manifests, err := keppelScanner.ListManifests(account.Name, repository.Name)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ListManifests")
return
}
for _, manifest := range manifests {
HandleManifest(account, repository, manifest, component, keppelScanner, keppelProcessor)
}
}

func HandleManifest(account models.Account, repository models.Repository, manifest models.Manifest, component *client.Component, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor) {
componentVersion, err := keppelProcessor.ProcessManifest(manifest, component.Id)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during ProcessManifest")
componentVersion, err = keppelProcessor.GetComponentVersion(manifest.Digest)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetComponentVersion")
}
}
trivyReport, err := keppelScanner.GetTrivyReport(account.Name, repository.Name, manifest.Digest)
if err != nil {
log.WithFields(log.Fields{
"account:": account.Name,
"repository": repository.Name,
}).WithError(err).Error("Error during GetTrivyReport")
return
}
keppelProcessor.ProcessReport(*trivyReport, componentVersion.Id)
}
Loading