Skip to content

Commit

Permalink
Config profile with repo API integration (#797)
Browse files Browse the repository at this point in the history
  • Loading branch information
eranturgeman authored Dec 22, 2024
1 parent 9ac9e3c commit 802ca5b
Show file tree
Hide file tree
Showing 10 changed files with 194 additions and 83 deletions.
1 change: 0 additions & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,6 @@ jobs:
# Generate mocks
- name: Generate mocks
run: go generate ./...
if: ${{ matrix.suite.name != 'Unit' }}

- name: Run Tests
if: ${{ matrix.suite.name != 'GitHub Integration' || matrix.os == 'ubuntu' }}
Expand Down
7 changes: 4 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,16 @@ require (
github.com/jfrog/build-info-go v1.10.7
github.com/jfrog/froggit-go v1.16.2
github.com/jfrog/gofrog v1.7.6
github.com/jfrog/jfrog-cli-core/v2 v2.57.0
github.com/jfrog/jfrog-cli-security v1.13.5
github.com/jfrog/jfrog-client-go v1.48.2
github.com/jfrog/jfrog-cli-core/v2 v2.57.2
github.com/jfrog/jfrog-cli-security v1.13.6
github.com/jfrog/jfrog-client-go v1.48.4
github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible
github.com/owenrumney/go-sarif/v2 v2.3.1
github.com/stretchr/testify v1.10.0
github.com/urfave/cli/v2 v2.27.4
github.com/xeipuuv/gojsonschema v1.2.0
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
)

Expand Down
13 changes: 7 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,12 @@ github.com/jfrog/gofrog v1.7.6 h1:QmfAiRzVyaI7JYGsB7cxfAJePAZTzFz0gRWZSE27c6s=
github.com/jfrog/gofrog v1.7.6/go.mod h1:ntr1txqNOZtHplmaNd7rS4f8jpA5Apx8em70oYEe7+4=
github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
github.com/jfrog/jfrog-cli-core/v2 v2.57.0 h1:3ON0J6Sjc2+4HZrzh4eSbdciXx3sJsJUIJ3TPQXh/5c=
github.com/jfrog/jfrog-cli-core/v2 v2.57.0/go.mod h1:SThaC/fniC96oN8YgCsHjvOxp5rBM7IppuIybn1oxT0=
github.com/jfrog/jfrog-cli-security v1.13.5 h1:CjWyCpURUtSCiu6xLqgvcwWFfy56hi+kFcMMTqEFhOo=
github.com/jfrog/jfrog-cli-security v1.13.5/go.mod h1:gc14ZiaO0vXRrOWBoOlsPKrbRdKDgcl+jRFuCaAH0Vk=
github.com/jfrog/jfrog-client-go v1.48.2 h1:YVAIiNWuBEa4NbWL54I+YzvXHaxoHDk532USDKTvgLU=
github.com/jfrog/jfrog-client-go v1.48.2/go.mod h1:1a7bmQHkRmPEza9wva2+WVrYzrGbosrMymq57kyG5gU=
github.com/jfrog/jfrog-cli-core/v2 v2.57.2 h1:2shy1CRWm/8yf6WWfVyAW3AdmryQiI73Tkhfb62vgPE=
github.com/jfrog/jfrog-cli-core/v2 v2.57.2/go.mod h1:sgi0gw96J00Yzx2cKG5xTG/x9XD0YiJbglJOnXUeaD0=
github.com/jfrog/jfrog-cli-security v1.13.6 h1:gm8TnGTlprMJbRga0cujeoN2xal7Pagd2kDkUfclvrw=
github.com/jfrog/jfrog-cli-security v1.13.6/go.mod h1:NarJyhl8Kh0HL6br74oeStIosLmCjA7atLYxZIFHJc4=
github.com/jfrog/jfrog-client-go v1.48.4 h1:uXvBr2ebFKpBRUhWgC9TSSJe32IbSYGlbDp9tDzBcaY=
github.com/jfrog/jfrog-client-go v1.48.4/go.mod h1:2ySOMva54L3EYYIlCBYBTcTgqfrrQ19gtpA/MWfA/ec=
github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA=
github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
Expand Down Expand Up @@ -413,6 +413,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
25 changes: 15 additions & 10 deletions scanpullrequest/scanpullrequest.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ func auditTargetBranch(repoConfig *utils.Repository, scanDetails *utils.ScanDeta
// Download target branch (if needed)
cleanupTarget := func() error { return nil }
if !repoConfig.IncludeAllVulnerabilities {
if targetBranchWd, cleanupTarget, err = prepareTargetForScan(repoConfig.PullRequestDetails, scanDetails); err != nil {
if targetBranchWd, cleanupTarget, err = prepareTargetForScan(repoConfig.Git, scanDetails); err != nil {
return
}
}
Expand All @@ -232,8 +232,8 @@ func auditTargetBranch(repoConfig *utils.Repository, scanDetails *utils.ScanDeta
return
}

func prepareTargetForScan(pullRequestDetails vcsclient.PullRequestInfo, scanDetails *utils.ScanDetails) (targetBranchWd string, cleanupTarget func() error, err error) {
target := pullRequestDetails.Target
func prepareTargetForScan(gitDetails utils.Git, scanDetails *utils.ScanDetails) (targetBranchWd string, cleanupTarget func() error, err error) {
target := gitDetails.PullRequestDetails.Target
// Download target branch
if targetBranchWd, cleanupTarget, err = utils.DownloadRepoToTempDir(scanDetails.Client(), target.Owner, target.Repository, target.Name); err != nil {
return
Expand All @@ -243,18 +243,23 @@ func prepareTargetForScan(pullRequestDetails vcsclient.PullRequestInfo, scanDeta
}
log.Debug("Using most common ancestor commit as target branch commit")
// Get common parent commit between source and target and use it (checkout) to the target branch commit
if e := tryCheckoutToMostCommonAncestor(scanDetails, pullRequestDetails.Source.Name, target.Name, targetBranchWd); e != nil {
log.Warn(fmt.Sprintf("Failed to get best common ancestor commit between source branch: %s and target branch: %s, defaulting to target branch commit. Error: %s", pullRequestDetails.Source.Name, target.Name, e.Error()))
if e := tryCheckoutToMostCommonAncestor(scanDetails, gitDetails.PullRequestDetails.Source.Name, target.Name, targetBranchWd, gitDetails.RepositoryCloneUrl); e != nil {
log.Warn(fmt.Sprintf("Failed to get best common ancestor commit between source branch: %s and target branch: %s, defaulting to target branch commit. Error: %s", gitDetails.PullRequestDetails.Source.Name, target.Name, e.Error()))
}
return
}

func tryCheckoutToMostCommonAncestor(scanDetails *utils.ScanDetails, baseBranch, headBranch, targetBranchWd string) (err error) {
repositoryInfo, err := scanDetails.Client().GetRepositoryInfo(context.Background(), scanDetails.RepoOwner, scanDetails.RepoName)
if err != nil {
return
func tryCheckoutToMostCommonAncestor(scanDetails *utils.ScanDetails, baseBranch, headBranch, targetBranchWd, cloneRepoUrl string) (err error) {
if cloneRepoUrl != "" {
scanDetails.Git.RepositoryCloneUrl = cloneRepoUrl
} else {
var repositoryInfo vcsclient.RepositoryInfo
repositoryInfo, err = scanDetails.Client().GetRepositoryInfo(context.Background(), scanDetails.RepoOwner, scanDetails.RepoName)
if err != nil {
return
}
scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
}
scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
// Change working directory to the temp target branch directory
cwd, err := os.Getwd()
if err != nil {
Expand Down
15 changes: 11 additions & 4 deletions scanrepository/scanrepository.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ func (cfp *ScanRepositoryCmd) setCommandPrerequisites(repository *utils.Reposito
SetXrayGraphScanParams(repository.Watches, repository.JFrogProjectKey, len(repository.AllowedLicenses) > 0).
SetFailOnInstallationErrors(*repository.FailOnSecurityIssues).
SetFixableOnly(repository.FixableOnly).
SetConfigProfile(repository.ConfigProfile).
SetSkipAutoInstall(repository.SkipAutoInstall).
SetAllowPartialResults(repository.AllowPartialResults).
SetDisableJas(repository.DisableJas)
Expand All @@ -137,11 +138,17 @@ func (cfp *ScanRepositoryCmd) setCommandPrerequisites(repository *utils.Reposito
if cfp.scanDetails, err = cfp.scanDetails.SetMinSeverity(repository.MinSeverity); err != nil {
return
}
repositoryInfo, err := client.GetRepositoryInfo(context.Background(), cfp.scanDetails.RepoOwner, cfp.scanDetails.RepoName)
if err != nil {
return
if repository.Git.RepositoryCloneUrl != "" {
cfp.scanDetails.Git.RepositoryCloneUrl = repository.Git.RepositoryCloneUrl
} else {
var repositoryInfo vcsclient.RepositoryInfo
repositoryInfo, err = client.GetRepositoryInfo(context.Background(), cfp.scanDetails.RepoOwner, cfp.scanDetails.RepoName)
if err != nil {
return
}
cfp.scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP
}
cfp.scanDetails.Git.RepositoryCloneUrl = repositoryInfo.CloneInfo.HTTP

// Set the flag for aggregating fixes to generate a unified pull request for fixing vulnerabilities
cfp.aggregateFixes = repository.Git.AggregateFixes
// Set the outputwriter interface for the relevant vcs git provider
Expand Down
17 changes: 9 additions & 8 deletions utils/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ const (
azurePipelines ciProvider = "azure-pipelines"

// JFrog platform environment variables
JFrogUserEnv = "JF_USER"
JFrogUrlEnv = "JF_URL"
jfrogXrayUrlEnv = "JF_XRAY_URL"
jfrogArtifactoryUrlEnv = "JF_ARTIFACTORY_URL"
jfrogReleasesRepoEnv = "JF_RELEASES_REPO"
JFrogPasswordEnv = "JF_PASSWORD"
JFrogTokenEnv = "JF_ACCESS_TOKEN"
JfrogConfigProfileEnv = "JF_CONFIG_PROFILE"
JFrogUserEnv = "JF_USER"
JFrogUrlEnv = "JF_URL"
jfrogXrayUrlEnv = "JF_XRAY_URL"
jfrogArtifactoryUrlEnv = "JF_ARTIFACTORY_URL"
jfrogReleasesRepoEnv = "JF_RELEASES_REPO"
JFrogPasswordEnv = "JF_PASSWORD"
JFrogTokenEnv = "JF_ACCESS_TOKEN"
JfrogUseConfigProfileEnv = "JF_USE_CONFIG_PROFILE"
JfrogConfigProfileEnv = "JF_CONFIG_PROFILE"

// Git environment variables
GitProvider = "JF_GIT_PROVIDER"
Expand Down
59 changes: 44 additions & 15 deletions utils/params.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"gopkg.in/yaml.v2"
"net/http"
"net/url"
"os"
Expand All @@ -25,7 +26,6 @@ import (
"github.com/jfrog/froggit-go/vcsutils"
coreconfig "github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-client-go/utils/log"
"gopkg.in/yaml.v3"
)

const (
Expand Down Expand Up @@ -426,11 +426,6 @@ func GetFrogbotDetails(commandName string) (frogbotDetails *FrogbotDetails, err
return
}

configProfile, err := getConfigProfileIfExistsAndValid(xrayVersion, xscVersion, jfrogServer)
if err != nil {
return
}

gitParamsFromEnv, err := extractGitParamsFromEnvs(commandName)
if err != nil {
return
Expand Down Expand Up @@ -458,9 +453,16 @@ func GetFrogbotDetails(commandName string) (frogbotDetails *FrogbotDetails, err
return
}

// We apply the configProfile to all received repositories. This loop must be deleted when we will no longer accept multiple repositories in a single scan
configProfile, repoCloneUrl, err := getConfigProfileIfExistsAndValid(xrayVersion, xscVersion, jfrogServer, client, gitParamsFromEnv)
if err != nil {
return
}

// We apply the configProfile to all received repositories. If no config profile was fetched, a nil value is passed
// TODO This loop must be deleted when we will no longer accept multiple repositories in a single scan
for i := range configAggregator {
configAggregator[i].Scan.ConfigProfile = configProfile
configAggregator[i].Git.RepositoryCloneUrl = repoCloneUrl
}

frogbotDetails = &FrogbotDetails{XrayVersion: xrayVersion, XscVersion: xscVersion, Repositories: configAggregator, GitClient: client, ServerDetails: jfrogServer, ReleasesRepo: os.Getenv(jfrogReleasesRepoEnv)}
Expand Down Expand Up @@ -791,28 +793,55 @@ func readConfigFromTarget(client vcsclient.VcsClient, gitParamsFromEnv *Git) (co
return
}

// This function fetches a config profile if JF_CONFIG_PROFILE is provided.
// If so - it verifies there is only a single module with a '.' path from root. If these conditions doesn't hold we return an error.
func getConfigProfileIfExistsAndValid(xrayVersion, xscVersion string, jfrogServer *coreconfig.ServerDetails) (configProfile *services.ConfigProfile, err error) {
// This function attempts to fetch a config profile if JF_USE_CONFIG_PROFILE is set to true.
// If we need to use a profile, we first try to get the profile by name that can be provided through JF_CONFIG_PROFILE. If name is provided but profile doesn't exist we return an error.
// If we need to use a profile, but name is not provided, we check if there is a config profile associated to the repo URL.
// When a profile is found we verify several conditions on it.
// If a profile was requested but not found by url nor by name we return an error.
func getConfigProfileIfExistsAndValid(xrayVersion, xscVersion string, jfrogServer *coreconfig.ServerDetails, gitClient vcsclient.VcsClient, gitParams *Git) (configProfile *services.ConfigProfile, repoCloneUrl string, err error) {
var useConfigProfile bool
if useConfigProfile, err = getBoolEnv(JfrogUseConfigProfileEnv, false); err != nil || !useConfigProfile {
log.Debug(fmt.Sprintf("Configuration Profile usage is disabled. All configurations will be derived from environment variables and files.\nTo enable a Configuration Profile, please set %s to TRUE", JfrogUseConfigProfileEnv))
return
}

// Attempt to get the config profile by profile's name
profileName := getTrimmedEnv(JfrogConfigProfileEnv)
if profileName == "" {
log.Debug(fmt.Sprintf("No %s environment variable was provided. All configurations will be induced from Env vars and files", JfrogConfigProfileEnv))
if profileName != "" {
log.Debug(fmt.Sprintf("Configuration profile was requested. Searching profile by provided name '%s'", profileName))
if configProfile, err = xsc.GetConfigProfileByName(xrayVersion, xscVersion, jfrogServer, profileName); err != nil || configProfile == nil {
return
}
err = verifyConfigProfileValidity(configProfile)
return
}

if configProfile, err = xsc.GetConfigProfile(xrayVersion, xscVersion, jfrogServer, profileName); err != nil {
// Getting repository's url in order to get repository HTTP url
repositoryInfo, err := gitClient.GetRepositoryInfo(context.Background(), gitParams.RepoOwner, gitParams.RepoName)
if err != nil {
return nil, "", err
}
repoCloneUrl = repositoryInfo.CloneInfo.HTTP

// Attempt to get a config profile associated with the repo URL
log.Debug(fmt.Sprintf("Configuration profile was requested. Searching profile associated to repository '%s'", jfrogServer.Url))
if configProfile, err = xsc.GetConfigProfileByUrl(xrayVersion, jfrogServer, repoCloneUrl); err != nil || configProfile == nil {
return
}
err = verifyConfigProfileValidity(configProfile)
return
}

func verifyConfigProfileValidity(configProfile *services.ConfigProfile) (err error) {
// Currently, only a single Module that represents the entire project is supported
if len(configProfile.Modules) != 1 {
err = fmt.Errorf("more than one module was found '%s' profile. Frogbot currently supports only one module per config profile", configProfile.ProfileName)
return
}
if configProfile.Modules[0].PathFromRoot != "." {
err = fmt.Errorf("module '%s' in profile '%s' contains the following path from root: '%s'. Frogbot currently supports only a single module with a '.' path from root", configProfile.Modules[0].ModuleName, profileName, configProfile.Modules[0].PathFromRoot)
err = fmt.Errorf("module '%s' in profile '%s' contains the following path from root: '%s'. Frogbot currently supports only a single module with a '.' path from root", configProfile.Modules[0].ModuleName, configProfile.ProfileName, configProfile.Modules[0].PathFromRoot)
return
}
log.Info(fmt.Sprintf("Using Config profile '%s'. jfrog-apps-config will be ignored if exists", profileName))
log.Info(fmt.Sprintf("Using Config profile '%s'. jfrog-apps-config will be ignored if exists", configProfile.ProfileName))
return
}
Loading

0 comments on commit 802ca5b

Please sign in to comment.