From a8b255a224c32ca2ea75c57ba414e6c6cc6241b3 Mon Sep 17 00:00:00 2001 From: laurentsimon <64505099+laurentsimon@users.noreply.github.com> Date: Thu, 3 Aug 2023 21:52:15 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20[experimental]=20Probe=20support=20?= =?UTF-8?q?for=20security=20policy=20check=20(#3241)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * update Signed-off-by: laurentsimon * fix unit tests Signed-off-by: laurentsimon * comments Signed-off-by: laurentsimon * compilation fix Signed-off-by: laurentsimon * missing file Signed-off-by: laurentsimon * missing file Signed-off-by: laurentsimon * update reason string Signed-off-by: laurentsimon * typo Signed-off-by: laurentsimon * fix unit tests Signed-off-by: laurentsimon * typo Signed-off-by: laurentsimon * unit tests and linnter Signed-off-by: laurentsimon * comments Signed-off-by: laurentsimon * comments Signed-off-by: laurentsimon * missing file Signed-off-by: laurentsimon * unit tests for probes Signed-off-by: laurentsimon * linter Signed-off-by: laurentsimon * revert FileSize change Signed-off-by: laurentsimon --------- Signed-off-by: laurentsimon --- checks/dependency_update_tool.go | 9 +- checks/dependency_update_tool_test.go | 2 - checks/evaluation/dependency_update_tool.go | 16 + .../evaluation/dependency_update_tool_test.go | 55 ++- checks/evaluation/security_policy.go | 167 ++----- checks/evaluation/security_policy_test.go | 268 +++++------- checks/run_probes.go | 13 +- checks/security_policy.go | 14 +- checks/security_policy_test.go | 4 +- e2e/dependency_update_tool_test.go | 4 - finding/finding.go | 19 + probes/entries.go | 13 + probes/internal/utils/error.go | 21 + probes/internal/utils/secpolicy.go | 53 +++ probes/{ => internal}/utils/tools.go | 0 probes/securityPolicyContainsLinks/def.yml | 40 ++ probes/securityPolicyContainsLinks/impl.go | 68 +++ .../securityPolicyContainsLinks/impl_test.go | 308 +++++++++++++ probes/securityPolicyContainsText/def.yml | 40 ++ probes/securityPolicyContainsText/impl.go | 75 ++++ .../securityPolicyContainsText/impl_test.go | 414 ++++++++++++++++++ .../def.yml | 40 ++ .../impl.go | 66 +++ .../impl_test.go | 269 ++++++++++++ probes/securityPolicyPresent/def.yml | 43 ++ probes/securityPolicyPresent/impl.go | 65 +++ probes/securityPolicyPresent/impl_test.go | 138 ++++++ probes/toolDependabotInstalled/def.yml | 4 +- probes/toolDependabotInstalled/impl.go | 10 +- probes/toolDependabotInstalled/impl_test.go | 126 ++++++ probes/toolPyUpInstalled/def.yml | 4 +- probes/toolPyUpInstalled/impl.go | 10 +- probes/toolPyUpInstalled/impl_test.go | 126 ++++++ probes/toolRenovateInstalled/def.yml | 4 +- probes/toolRenovateInstalled/impl.go | 10 +- probes/toolRenovateInstalled/impl_test.go | 126 ++++++ probes/toolSonatypeLiftInstalled/def.yml | 4 +- probes/toolSonatypeLiftInstalled/impl.go | 10 +- probes/toolSonatypeLiftInstalled/impl_test.go | 126 ++++++ 39 files changed, 2466 insertions(+), 318 deletions(-) create mode 100644 probes/internal/utils/error.go create mode 100644 probes/internal/utils/secpolicy.go rename probes/{ => internal}/utils/tools.go (100%) create mode 100644 probes/securityPolicyContainsLinks/def.yml create mode 100644 probes/securityPolicyContainsLinks/impl.go create mode 100644 probes/securityPolicyContainsLinks/impl_test.go create mode 100644 probes/securityPolicyContainsText/def.yml create mode 100644 probes/securityPolicyContainsText/impl.go create mode 100644 probes/securityPolicyContainsText/impl_test.go create mode 100644 probes/securityPolicyContainsVulnerabilityDisclosure/def.yml create mode 100644 probes/securityPolicyContainsVulnerabilityDisclosure/impl.go create mode 100644 probes/securityPolicyContainsVulnerabilityDisclosure/impl_test.go create mode 100644 probes/securityPolicyPresent/def.yml create mode 100644 probes/securityPolicyPresent/impl.go create mode 100644 probes/securityPolicyPresent/impl_test.go create mode 100644 probes/toolDependabotInstalled/impl_test.go create mode 100644 probes/toolPyUpInstalled/impl_test.go create mode 100644 probes/toolRenovateInstalled/impl_test.go create mode 100644 probes/toolSonatypeLiftInstalled/impl_test.go diff --git a/checks/dependency_update_tool.go b/checks/dependency_update_tool.go index 54f1954f9b1..1222c6e0dd2 100644 --- a/checks/dependency_update_tool.go +++ b/checks/dependency_update_tool.go @@ -44,13 +44,12 @@ func DependencyUpdateTool(c *checker.CheckRequest) checker.CheckResult { return checker.CreateRuntimeErrorResult(CheckDependencyUpdateTool, e) } - // Return raw results. - if c.RawResults != nil { - c.RawResults.DependencyUpdateToolResults = rawData - } + // Set the raw results. + pRawResults := getRawResults(c) + pRawResults.DependencyUpdateToolResults = rawData // Evaluate the probes. - findings, err := evaluateProbes(c, CheckDependencyUpdateTool, probes.DependencyToolUpdates) + findings, err := evaluateProbes(c, pRawResults, probes.DependencyToolUpdates) if err != nil { e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) return checker.CreateRuntimeErrorResult(CheckDependencyUpdateTool, e) diff --git a/checks/dependency_update_tool_test.go b/checks/dependency_update_tool_test.go index 173f4e5198c..7ec379667ea 100644 --- a/checks/dependency_update_tool_test.go +++ b/checks/dependency_update_tool_test.go @@ -150,11 +150,9 @@ func TestDependencyUpdateTool(t *testing.T) { mockRepo.EXPECT().ListFiles(gomock.Any()).Return(tt.files, nil) mockRepo.EXPECT().SearchCommits(gomock.Any()).Return(tt.SearchCommits, nil).Times(tt.CallSearchCommits) dl := scut.TestDetailLogger{} - raw := checker.RawResults{} c := &checker.CheckRequest{ RepoClient: mockRepo, Dlogger: &dl, - RawResults: &raw, } res := DependencyUpdateTool(c) diff --git a/checks/evaluation/dependency_update_tool.go b/checks/evaluation/dependency_update_tool.go index 239167a4e3b..a60ab7f26d3 100644 --- a/checks/evaluation/dependency_update_tool.go +++ b/checks/evaluation/dependency_update_tool.go @@ -16,13 +16,29 @@ package evaluation import ( "github.com/ossf/scorecard/v4/checker" + sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/toolDependabotInstalled" + "github.com/ossf/scorecard/v4/probes/toolPyUpInstalled" + "github.com/ossf/scorecard/v4/probes/toolRenovateInstalled" + "github.com/ossf/scorecard/v4/probes/toolSonatypeLiftInstalled" ) // DependencyUpdateTool applies the score policy for the Dependency-Update-Tool check. func DependencyUpdateTool(name string, findings []finding.Finding, ) checker.CheckResult { + expectedProbes := []string{ + toolDependabotInstalled.Probe, + toolPyUpInstalled.Probe, + toolRenovateInstalled.Probe, + toolSonatypeLiftInstalled.Probe, + } + if !finding.UniqueProbesEqual(findings, expectedProbes) { + e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") + return checker.CreateRuntimeErrorResult(name, e) + } + for i := range findings { f := &findings[i] if f.Outcome == finding.OutcomePositive { diff --git a/checks/evaluation/dependency_update_tool_test.go b/checks/evaluation/dependency_update_tool_test.go index 62667d6c3ca..cd477762c6c 100644 --- a/checks/evaluation/dependency_update_tool_test.go +++ b/checks/evaluation/dependency_update_tool_test.go @@ -39,6 +39,18 @@ func TestDependencyUpdateTool(t *testing.T) { Probe: "toolDependabotInstalled", Outcome: finding.OutcomePositive, }, + { + Probe: "toolPyUpInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolRenovateInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolSonatypeLiftInstalled", + Outcome: finding.OutcomeNegative, + }, }, want: checker.CheckResult{ Score: 10, @@ -47,10 +59,22 @@ func TestDependencyUpdateTool(t *testing.T) { { name: "renovate", findings: []finding.Finding{ + { + Probe: "toolDependabotInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolPyUpInstalled", + Outcome: finding.OutcomeNegative, + }, { Probe: "toolRenovateInstalled", Outcome: finding.OutcomePositive, }, + { + Probe: "toolSonatypeLiftInstalled", + Outcome: finding.OutcomeNegative, + }, }, want: checker.CheckResult{ Score: 10, @@ -59,10 +83,22 @@ func TestDependencyUpdateTool(t *testing.T) { { name: "pyup", findings: []finding.Finding{ + { + Probe: "toolDependabotInstalled", + Outcome: finding.OutcomeNegative, + }, { Probe: "toolPyUpInstalled", Outcome: finding.OutcomePositive, }, + { + Probe: "toolRenovateInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolSonatypeLiftInstalled", + Outcome: finding.OutcomeNegative, + }, }, want: checker.CheckResult{ Score: 10, @@ -72,7 +108,19 @@ func TestDependencyUpdateTool(t *testing.T) { name: "sonatype", findings: []finding.Finding{ { - Probe: "toolSonatypeInstalled", + Probe: "toolDependabotInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolPyUpInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolRenovateInstalled", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "toolSonatypeLiftInstalled", Outcome: finding.OutcomePositive, }, }, @@ -96,7 +144,7 @@ func TestDependencyUpdateTool(t *testing.T) { Outcome: finding.OutcomeNegative, }, { - Probe: "toolSonatypeInstalled", + Probe: "toolSonatypeLiftInstalled", Outcome: finding.OutcomeNegative, }, }, @@ -107,8 +155,7 @@ func TestDependencyUpdateTool(t *testing.T) { { name: "empty tool list", want: checker.CheckResult{ - Score: 0, - Error: nil, + Score: -1, }, }, } diff --git a/checks/evaluation/security_policy.go b/checks/evaluation/security_policy.go index 3512ad67e97..736fdf3ade4 100644 --- a/checks/evaluation/security_policy.go +++ b/checks/evaluation/security_policy.go @@ -18,139 +18,62 @@ import ( "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" + "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" + "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" + "github.com/ossf/scorecard/v4/probes/securityPolicyPresent" ) -func scoreSecurityCriteria(f checker.File, - info []checker.SecurityPolicyInformation, - dl checker.DetailLogger, -) int { - var urls, emails, discvuls, linkedContentLen, score int - - emails = countSecInfo(info, checker.SecurityPolicyInformationTypeEmail, true) - urls = countSecInfo(info, checker.SecurityPolicyInformationTypeLink, true) - discvuls = countSecInfo(info, checker.SecurityPolicyInformationTypeText, false) - - for _, i := range findSecInfo(info, checker.SecurityPolicyInformationTypeEmail, true) { - linkedContentLen += len(i.InformationValue.Match) - } - for _, i := range findSecInfo(info, checker.SecurityPolicyInformationTypeLink, true) { - linkedContentLen += len(i.InformationValue.Match) - } - - msg := checker.LogMessage{ - Path: f.Path, - Type: f.Type, - Text: "", - } - - // #1: linked content found (email/http): score += 6 - if (urls + emails) > 0 { - score += 6 - msg.Text = "Found linked content in security policy" - dl.Info(&msg) - } else { - msg.Text = "no email or URL found in security policy" - dl.Warn(&msg) - } - - // #2: more bytes than the sum of the length of all the linked content found: score += 3 - // rationale: there appears to be information and context around those links - // no credit if there is just a link to a site or an email address (those given above) - // the test here is that each piece of linked content will likely contain a space - // before and after the content (hence the two multiplier) - if f.FileSize > 1 && (f.FileSize > uint(linkedContentLen+((urls+emails)*2))) { - score += 3 - msg.Text = "Found text in security policy" - dl.Info(&msg) - } else { - msg.Text = "No text (beyond any linked content) found in security policy" - dl.Warn(&msg) +// SecurityPolicy applies the score policy for the Security-Policy check. +func SecurityPolicy(name string, findings []finding.Finding) checker.CheckResult { + // We have 4 unique probes, each should have a finding. + expectedProbes := []string{ + securityPolicyContainsVulnerabilityDisclosure.Probe, + securityPolicyContainsLinks.Probe, + securityPolicyContainsText.Probe, + securityPolicyPresent.Probe, } - - // #3: found whole number(s) and or match(es) to "Disclos" and or "Vuln": score += 1 - // rationale: works towards the intent of the security policy file - // regarding whom to contact about vuls and disclosures and timing - // e.g., we'll disclose, report a vulnerability, 30 days, etc. - // looking for at least 2 hits - if discvuls > 1 { - score += 1 - msg.Text = "Found disclosure, vulnerability, and/or timelines in security policy" - dl.Info(&msg) - } else { - msg.Text = "One or no descriptive hints of disclosure, vulnerability, and/or timelines in security policy" - dl.Warn(&msg) + if !finding.UniqueProbesEqual(findings, expectedProbes) { + e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") + return checker.CreateRuntimeErrorResult(name, e) } - return score -} - -func countSecInfo(secInfo []checker.SecurityPolicyInformation, - infoType checker.SecurityPolicyInformationType, - unique bool, -) int { - keys := make(map[string]bool) - count := 0 - for _, entry := range secInfo { - if _, present := keys[entry.InformationValue.Match]; !present && entry.InformationType == infoType { - keys[entry.InformationValue.Match] = true - count += 1 - } else if !unique && entry.InformationType == infoType { - count += 1 + score := 0 + m := make(map[string]bool) + for i := range findings { + f := &findings[i] + if f.Outcome == finding.OutcomePositive { + switch f.Probe { + case securityPolicyContainsVulnerabilityDisclosure.Probe: + score += scoreProbeOnce(f.Probe, m, 1) + case securityPolicyContainsLinks.Probe: + score += scoreProbeOnce(f.Probe, m, 6) + case securityPolicyContainsText.Probe: + score += scoreProbeOnce(f.Probe, m, 3) + case securityPolicyPresent.Probe: + m[f.Probe] = true + default: + e := sce.WithMessage(sce.ErrScorecardInternal, "unknown probe results") + return checker.CreateRuntimeErrorResult(name, e) + } } } - return count -} - -func findSecInfo(secInfo []checker.SecurityPolicyInformation, - infoType checker.SecurityPolicyInformationType, - unique bool, -) []checker.SecurityPolicyInformation { - keys := make(map[string]bool) - var secList []checker.SecurityPolicyInformation - for _, entry := range secInfo { - if _, present := keys[entry.InformationValue.Match]; !present && entry.InformationType == infoType { - keys[entry.InformationValue.Match] = true - secList = append(secList, entry) - } else if !unique && entry.InformationType == infoType { - secList = append(secList, entry) + _, defined := m[securityPolicyPresent.Probe] + if !defined { + if score > 0 { + e := sce.WithMessage(sce.ErrScorecardInternal, "score calculation problem") + return checker.CreateRuntimeErrorResult(name, e) } - } - return secList -} - -// SecurityPolicy applies the score policy for the Security-Policy check. -func SecurityPolicy(name string, dl checker.DetailLogger, r *checker.SecurityPolicyData) checker.CheckResult { - if r == nil { - e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data") - return checker.CreateRuntimeErrorResult(name, e) - } - - // Apply the policy evaluation. - if len(r.PolicyFiles) == 0 { - // If the file is unset, directly return as not detected. return checker.CreateMinScoreResult(name, "security policy file not detected") } - // TODO: although this a loop, the raw checks will only return one security policy - // when more than one security policy file can be aggregated into a composite - // score, that logic can be comprehended here. - score := 0 - for _, spd := range r.PolicyFiles { - score = scoreSecurityCriteria(spd.File, - spd.Information, dl) - - msg := checker.LogMessage{ - Path: spd.File.Path, - Type: spd.File.Type, - } - if msg.Type == finding.FileTypeURL { - msg.Text = "security policy detected in org repo" - } else { - msg.Text = "security policy detected in current repo" - } + return checker.CreateResultWithScore(name, "security policy file detected", score) +} - dl.Info(&msg) +func scoreProbeOnce(probeID string, m map[string]bool, bump int) int { + if _, exists := m[probeID]; !exists { + m[probeID] = true + return bump } - - return checker.CreateResultWithScore(name, "security policy file detected", score) + return 0 } diff --git a/checks/evaluation/security_policy_test.go b/checks/evaluation/security_policy_test.go index 2c6e45fe9b2..35a8563f8bb 100644 --- a/checks/evaluation/security_policy_test.go +++ b/checks/evaluation/security_policy_test.go @@ -19,56 +19,83 @@ import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" - scut "github.com/ossf/scorecard/v4/utests" ) func TestSecurityPolicy(t *testing.T) { t.Parallel() //nolint - type args struct { - name string - r *checker.SecurityPolicyData - } - //nolint tests := []struct { - name string - args args - err bool - want checker.CheckResult + name string + findings []finding.Finding + err bool + want checker.CheckResult }{ { - name: "test_security_policy_1", - args: args{ - name: "test_security_policy_1", + name: "missing findings links", + findings: []finding.Finding{ + { + Probe: "securityPolicyContainsVulnerabilityDisclosure", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyContainsText", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyPresent", + Outcome: finding.OutcomeNegative, + }, }, want: checker.CheckResult{ Score: -1, }, }, { - name: "test_security_policy_2", - args: args{ - name: "test_security_policy_2", - r: &checker.SecurityPolicyData{}, + name: "invalid probe name", + findings: []finding.Finding{ + { + Probe: "securityPolicyContainsVulnerabilityDisclosure", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyContainsLinks", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyContainsText", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyPresent", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyInvalidProbeName", + Outcome: finding.OutcomeNegative, + }, }, want: checker.CheckResult{ - Score: 0, + Score: -1, }, }, { - name: "test_security_policy_3", - args: args{ - name: "test_security_policy_3", - r: &checker.SecurityPolicyData{ - PolicyFiles: []checker.SecurityPolicyFile{ - { - File: checker.File{ - Path: "/etc/security/pam_env.conf", - Type: finding.FileTypeURL, - }, - Information: make([]checker.SecurityPolicyInformation, 0), - }, - }, + name: "file found only", + findings: []finding.Finding{ + { + Probe: "securityPolicyContainsVulnerabilityDisclosure", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyContainsLinks", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyContainsText", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "securityPolicyPresent", + Outcome: finding.OutcomePositive, }, }, want: checker.CheckResult{ @@ -76,159 +103,92 @@ func TestSecurityPolicy(t *testing.T) { }, }, { - name: "test_security_policy_4", - args: args{ - name: "test_security_policy_4", - r: &checker.SecurityPolicyData{ - PolicyFiles: []checker.SecurityPolicyFile{ - { - File: checker.File{ - Path: "/etc/security/pam_env.conf", - }, - Information: make([]checker.SecurityPolicyInformation, 0), - }, - }, + name: "file not found with positive probes", + findings: []finding.Finding{ + { + Probe: "securityPolicyContainsVulnerabilityDisclosure", + Outcome: finding.OutcomePositive, + }, + { + Probe: "securityPolicyContainsLinks", + Outcome: finding.OutcomePositive, + }, + { + Probe: "securityPolicyContainsText", + Outcome: finding.OutcomePositive, + }, + { + Probe: "securityPolicyPresent", + Outcome: finding.OutcomeNegative, }, }, want: checker.CheckResult{ - Score: 0, + Score: -1, }, }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - x := checker.CheckRequest{Dlogger: &scut.TestDetailLogger{}} - - got := SecurityPolicy(tt.args.name, x.Dlogger, tt.args.r) - if tt.err { - if got.Score != -1 { - t.Errorf("SecurityPolicy() = %v, want %v", got, tt.want) - } - } - if got.Score != tt.want.Score { - t.Errorf("SecurityPolicy() = %v, want %v for %v", got.Score, tt.want.Score, tt.name) - } - }) - } -} - -func TestScoreSecurityCriteria(t *testing.T) { - t.Parallel() - tests := []struct { //nolint:govet - name string - file checker.File - info []checker.SecurityPolicyInformation - expectedScore int - }{ { - name: "Full score", - file: checker.File{ - Path: "/path/to/security_policy.md", - FileSize: 100, - }, - info: []checker.SecurityPolicyInformation{ + name: "file found with no disclosure and text", + findings: []finding.Finding{ { - InformationType: checker.SecurityPolicyInformationTypeEmail, - InformationValue: checker.SecurityPolicyValueType{ - Match: "security@example.com", - LineNumber: 2, - Offset: 0, - }, + Probe: "securityPolicyContainsVulnerabilityDisclosure", + Outcome: finding.OutcomeNegative, }, { - InformationType: checker.SecurityPolicyInformationTypeLink, - InformationValue: checker.SecurityPolicyValueType{ - Match: "https://example.com/report", - LineNumber: 4, - Offset: 0, - }, + Probe: "securityPolicyContainsLinks", + Outcome: finding.OutcomePositive, }, { - InformationType: checker.SecurityPolicyInformationTypeText, - InformationValue: checker.SecurityPolicyValueType{ - Match: "Disclose vulnerability", - LineNumber: 6, - Offset: 0, - }, + Probe: "securityPolicyContainsText", + Outcome: finding.OutcomeNegative, }, { - InformationType: checker.SecurityPolicyInformationTypeText, - InformationValue: checker.SecurityPolicyValueType{ - Match: "30 days", - LineNumber: 7, - Offset: 0, - }, + Probe: "securityPolicyPresent", + Outcome: finding.OutcomePositive, }, }, - expectedScore: 10, + want: checker.CheckResult{ + Score: 6, + }, }, { - name: "Partial score", - file: checker.File{ - Path: "/path/to/security_policy.md", - FileSize: 50, - }, - info: []checker.SecurityPolicyInformation{ + name: "file found all positive", + findings: []finding.Finding{ { - InformationType: checker.SecurityPolicyInformationTypeLink, - InformationValue: checker.SecurityPolicyValueType{ - Match: "https://example.com/report", - LineNumber: 4, - Offset: 0, - }, + Probe: "securityPolicyContainsVulnerabilityDisclosure", + Outcome: finding.OutcomePositive, }, { - InformationType: checker.SecurityPolicyInformationTypeText, - InformationValue: checker.SecurityPolicyValueType{ - Match: "Disclose vulnerability", - LineNumber: 6, - Offset: 0, - }, + Probe: "securityPolicyContainsLinks", + Outcome: finding.OutcomePositive, }, - }, - expectedScore: 9, - }, - { - name: "Low score", - file: checker.File{ - Path: "/path/to/security_policy.md", - FileSize: 10, - }, - info: []checker.SecurityPolicyInformation{ { - InformationType: checker.SecurityPolicyInformationTypeEmail, - InformationValue: checker.SecurityPolicyValueType{ - Match: "security@example.com", - LineNumber: 2, - Offset: 0, - }, + Probe: "securityPolicyContainsText", + Outcome: finding.OutcomePositive, + }, + { + Probe: "securityPolicyPresent", + Outcome: finding.OutcomePositive, }, }, - expectedScore: 6, - }, - { - name: "Low score", - file: checker.File{ - Path: "/path/to/security_policy.md", - FileSize: 5, + want: checker.CheckResult{ + Score: 10, }, - info: []checker.SecurityPolicyInformation{}, - expectedScore: 3, }, } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { t.Parallel() - mockDetailLogger := &scut.TestDetailLogger{} - score := scoreSecurityCriteria(tc.file, tc.info, mockDetailLogger) - if score != tc.expectedScore { - t.Errorf("scoreSecurityCriteria() mismatch, expected score: %d, got: %d", tc.expectedScore, score) + got := SecurityPolicy("SecurityPolicy", tt.findings) + if tt.err { + if got.Score != -1 { + t.Errorf("SecurityPolicy() = %v, want %v", got, tt.want) + } + } + if got.Score != tt.want.Score { + t.Errorf("SecurityPolicy() = %v, want %v for %v", got.Score, tt.want.Score, tt.name) } }) } diff --git a/checks/run_probes.go b/checks/run_probes.go index 5af28ac79ed..22ff0e58249 100644 --- a/checks/run_probes.go +++ b/checks/run_probes.go @@ -24,11 +24,11 @@ import ( ) // evaluateProbes runs the probes in probesToRun and logs its findings. -func evaluateProbes(c *checker.CheckRequest, checkName string, +func evaluateProbes(c *checker.CheckRequest, rawResults *checker.RawResults, probesToRun []probes.ProbeImpl, ) ([]finding.Finding, error) { // Run the probes. - findings, err := zrunner.Run(c.RawResults, probesToRun) + findings, err := zrunner.Run(rawResults, probesToRun) if err != nil { return nil, fmt.Errorf("zrunner.Run: %w", err) } @@ -39,3 +39,12 @@ func evaluateProbes(c *checker.CheckRequest, checkName string, } return findings, nil } + +// getRawResults returns a pointer to the raw results in the CheckRequest +// if the pointer is not nil. Else, it creates a new raw result. +func getRawResults(c *checker.CheckRequest) *checker.RawResults { + if c.RawResults != nil { + return c.RawResults + } + return &checker.RawResults{} +} diff --git a/checks/security_policy.go b/checks/security_policy.go index 53f3f5091b0..902c70327a3 100644 --- a/checks/security_policy.go +++ b/checks/security_policy.go @@ -19,6 +19,7 @@ import ( "github.com/ossf/scorecard/v4/checks/evaluation" "github.com/ossf/scorecard/v4/checks/raw" sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/probes" ) // CheckSecurityPolicy is the registred name for SecurityPolicy. @@ -44,9 +45,16 @@ func SecurityPolicy(c *checker.CheckRequest) checker.CheckResult { } // Set the raw results. - if c.RawResults != nil { - c.RawResults.SecurityPolicyResults = rawData + pRawResults := getRawResults(c) + pRawResults.SecurityPolicyResults = rawData + + // Evaluate the probes. + findings, err := evaluateProbes(c, pRawResults, probes.SecurityPolicy) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return checker.CreateRuntimeErrorResult(CheckSecurityPolicy, e) } - return evaluation.SecurityPolicy(CheckSecurityPolicy, c.Dlogger, &rawData) + // Return the score evaluation. + return evaluation.SecurityPolicy(CheckSecurityPolicy, findings) } diff --git a/checks/security_policy_test.go b/checks/security_policy_test.go index fb7e7dc610c..cf759d180a1 100644 --- a/checks/security_policy_test.go +++ b/checks/security_policy_test.go @@ -191,12 +191,12 @@ func TestSecurityPolicy(t *testing.T) { }).AnyTimes() dl := scut.TestDetailLogger{} - c := checker.CheckRequest{ + c := &checker.CheckRequest{ RepoClient: mockRepo, Dlogger: &dl, } - res := SecurityPolicy(&c) + res := SecurityPolicy(c) if !scut.ValidateTestReturn(t, tt.name, &tt.want, &res, &dl) { t.Errorf("test failed: log message not present: %+v on %+v", tt.want, res) diff --git a/e2e/dependency_update_tool_test.go b/e2e/dependency_update_tool_test.go index 244c849f82f..836f5c960e0 100644 --- a/e2e/dependency_update_tool_test.go +++ b/e2e/dependency_update_tool_test.go @@ -39,13 +39,11 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) - raw := checker.RawResults{} req := checker.CheckRequest{ Ctx: context.Background(), RepoClient: repoClient, Repo: repo, Dlogger: &dl, - RawResults: &raw, } expected := scut.TestReturn{ Error: nil, @@ -68,13 +66,11 @@ var _ = Describe("E2E TEST:"+checks.CheckDependencyUpdateTool, func() { err = repoClient.InitRepo(repo, clients.HeadSHA, 0) Expect(err).Should(BeNil()) - raw := checker.RawResults{} req := checker.CheckRequest{ Ctx: context.Background(), RepoClient: repoClient, Repo: repo, Dlogger: &dl, - RawResults: &raw, } expected := scut.TestReturn{ Error: nil, diff --git a/finding/finding.go b/finding/finding.go index c2c23f387fd..7dae00268a2 100644 --- a/finding/finding.go +++ b/finding/finding.go @@ -18,6 +18,7 @@ import ( "embed" "errors" "fmt" + "reflect" "strings" "gopkg.in/yaml.v3" @@ -181,6 +182,24 @@ func (f *Finding) WithMessage(text string) *Finding { return f } +// UniqueProbesEqual checks the probe names present in a list of findings +// and compare them against an expected list. +func UniqueProbesEqual(findings []Finding, probes []string) bool { + // Collect unique probes from findings. + fm := make(map[string]bool) + for i := range findings { + f := &findings[i] + fm[f.Probe] = true + } + // Collect probes from list. + pm := make(map[string]bool) + for i := range probes { + p := &probes[i] + pm[*p] = true + } + return reflect.DeepEqual(pm, fm) +} + // WithLocation adds a location to an existing finding. // No copy is made. func (f *Finding) WithLocation(loc *Location) *Finding { diff --git a/probes/entries.go b/probes/entries.go index 84be2c51fee..1864db9215a 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -17,6 +17,10 @@ package probes import ( "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" + "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" + "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" + "github.com/ossf/scorecard/v4/probes/securityPolicyPresent" "github.com/ossf/scorecard/v4/probes/toolDependabotInstalled" "github.com/ossf/scorecard/v4/probes/toolPyUpInstalled" "github.com/ossf/scorecard/v4/probes/toolRenovateInstalled" @@ -29,6 +33,14 @@ type ProbeImpl func(*checker.RawResults) ([]finding.Finding, string, error) var ( // All represents all the probes. All []ProbeImpl + // SecurityPolicy is all the probes for the + // SecurityPolicy check. + SecurityPolicy = []ProbeImpl{ + securityPolicyPresent.Run, + securityPolicyContainsLinks.Run, + securityPolicyContainsVulnerabilityDisclosure.Run, + securityPolicyContainsText.Run, + } // DependencyToolUpdates is all the probes for the // DpendencyUpdateTool check. DependencyToolUpdates = []ProbeImpl{ @@ -43,6 +55,7 @@ var ( func init() { All = concatMultipleProbes([][]ProbeImpl{ DependencyToolUpdates, + SecurityPolicy, }) } diff --git a/probes/internal/utils/error.go b/probes/internal/utils/error.go new file mode 100644 index 00000000000..67725dc5e8a --- /dev/null +++ b/probes/internal/utils/error.go @@ -0,0 +1,21 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "errors" +) + +var ErrNil = errors.New("nil pointer") diff --git a/probes/internal/utils/secpolicy.go b/probes/internal/utils/secpolicy.go new file mode 100644 index 00000000000..8aabe4de6a0 --- /dev/null +++ b/probes/internal/utils/secpolicy.go @@ -0,0 +1,53 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "github.com/ossf/scorecard/v4/checker" +) + +func CountSecInfo(secInfo []checker.SecurityPolicyInformation, + infoType checker.SecurityPolicyInformationType, + unique bool, +) int { + keys := make(map[string]bool) + count := 0 + for _, entry := range secInfo { + if _, present := keys[entry.InformationValue.Match]; !present && entry.InformationType == infoType { + keys[entry.InformationValue.Match] = true + count += 1 + } else if !unique && entry.InformationType == infoType { + count += 1 + } + } + return count +} + +func FindSecInfo(secInfo []checker.SecurityPolicyInformation, + infoType checker.SecurityPolicyInformationType, + unique bool, +) []checker.SecurityPolicyInformation { + keys := make(map[string]bool) + var secList []checker.SecurityPolicyInformation + for _, entry := range secInfo { + if _, present := keys[entry.InformationValue.Match]; !present && entry.InformationType == infoType { + keys[entry.InformationValue.Match] = true + secList = append(secList, entry) + } else if !unique && entry.InformationType == infoType { + secList = append(secList, entry) + } + } + return secList +} diff --git a/probes/utils/tools.go b/probes/internal/utils/tools.go similarity index 100% rename from probes/utils/tools.go rename to probes/internal/utils/tools.go diff --git a/probes/securityPolicyContainsLinks/def.yml b/probes/securityPolicyContainsLinks/def.yml new file mode 100644 index 00000000000..f89bfe5b0c5 --- /dev/null +++ b/probes/securityPolicyContainsLinks/def.yml @@ -0,0 +1,40 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: securityPolicyContainsLinks +short: Check that the security policy contains web or email links. +motivation: > + URLs point users to additional information as well as online disclosure forms. Emails provide a point of contact for vulnerability disclosure. +implementation: > + The implementation looks for strings "http(s)://" to find URLs; and for strings "...@..." for email addresses. +outcome: + - If links are found, one finding with OutcomePositive (1) is returned for each file. + - If no links are found, one finding with OutcomeNegative (0) is returned for each file. + - If no file is found, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Low + text: + - 'On GitHub:' + - Enable private vulnerability disclosure in your repository settings https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to follow the steps in https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities. + - 'On GitLab:' + - Provide a point of contact in your SECURITY.md. + - 'Examples: https://github.com/ossf/scorecard/blob/main/SECURITY.md, https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md, https://github.com/sigstore/.github/blob/main/SECURITY.md.' + markdown: + - 'On GitHub:' + - Enable private vulnerability disclosure in your [repository settings](https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository) + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to [follow these steps](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities). + - 'On GitLab:' + - Provide a point of contact in your SECURITY.md. + - 'Examples: [OpenSSF Scorecard](https://github.com/ossf/scorecard/blob/main/SECURITY.md), [SLSA builders](https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md), [Sigstore](https://github.com/sigstore/.github/blob/main/SECURITY.md).' \ No newline at end of file diff --git a/probes/securityPolicyContainsLinks/impl.go b/probes/securityPolicyContainsLinks/impl.go new file mode 100644 index 00000000000..d2891b80bfd --- /dev/null +++ b/probes/securityPolicyContainsLinks/impl.go @@ -0,0 +1,68 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyContainsLinks + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "securityPolicyContainsLinks" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } + var findings []finding.Finding + policies := raw.SecurityPolicyResults.PolicyFiles + for i := range policies { + policy := &policies[i] + emails := utils.CountSecInfo(policy.Information, checker.SecurityPolicyInformationTypeEmail, true) + urls := utils.CountSecInfo(policy.Information, checker.SecurityPolicyInformationTypeLink, true) + + if (urls + emails) > 0 { + f, err := finding.NewPositive(fs, Probe, + "Found linked content", policy.File.Location()) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } else { + f, err := finding.NewNegative(fs, Probe, + "no linked content found", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + } + + if len(findings) == 0 { + f, err := finding.NewNegative(fs, Probe, "no security file to analyze", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/securityPolicyContainsLinks/impl_test.go b/probes/securityPolicyContainsLinks/impl_test.go new file mode 100644 index 00000000000..63233991e73 --- /dev/null +++ b/probes/securityPolicyContainsLinks/impl_test.go @@ -0,0 +1,308 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyContainsLinks + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "file present on repo no link", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on repo link", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on repo email", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "hey@google.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on org no link", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on org link", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on org email", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "hey@google.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "files present on org and repo", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "files present on org and repo email", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "hey@google.com", + }, + }, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomeNegative, + }, + }, + { + name: "files present on org and repo link", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomeNegative, + }, + }, + { + name: "file not present", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/securityPolicyContainsText/def.yml b/probes/securityPolicyContainsText/def.yml new file mode 100644 index 00000000000..82dccd87adf --- /dev/null +++ b/probes/securityPolicyContainsText/def.yml @@ -0,0 +1,40 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: securityPolicyContainsText +short: Check that the security policy contains enough text and not just links. +motivation: > + Telling security researchers how to privately dislose problems with your project is important. The more details available, the better. +implementation: > + The implementation checks that the content of the SECURITY.md contains more than just a link or an email address. It does this by comparing the length of the content to the lengths of the links and email addresses. +outcome: + - If links are found, one finding with OutcomePositive (1) is returned for each file. + - If no links are found, one finding with OutcomeNegative (0) is returned for each file. + - If no file is found, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Low + text: + - 'On GitHub:' + - Enable private vulnerability disclosure in your repository settings https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to follow the steps in https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities. + - 'On GitLab:' + - Add a section in your SECURITY.md indicating the process to disclose vulnerabilities for your project. + - 'Examples: https://github.com/ossf/scorecard/blob/main/SECURITY.md, https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md, https://github.com/sigstore/.github/blob/main/SECURITY.md.' + markdown: + - 'On GitHub:' + - Enable private vulnerability disclosure in your [repository settings](https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository) + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to [follow these steps](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities). + - 'On GitLab:' + - Add a section in your SECURITY.md indicating the process to disclose vulnerabilities for your project. + - 'Examples: [OpenSSF Scorecard](https://github.com/ossf/scorecard/blob/main/SECURITY.md), [SLSA builders](https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md), [Sigstore](https://github.com/sigstore/.github/blob/main/SECURITY.md).' \ No newline at end of file diff --git a/probes/securityPolicyContainsText/impl.go b/probes/securityPolicyContainsText/impl.go new file mode 100644 index 00000000000..1e11cf373ae --- /dev/null +++ b/probes/securityPolicyContainsText/impl.go @@ -0,0 +1,75 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyContainsText + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "securityPolicyContainsText" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } + var findings []finding.Finding + policies := raw.SecurityPolicyResults.PolicyFiles + for i := range policies { + policy := &policies[i] + linkedContentLen := 0 + emails := utils.CountSecInfo(policy.Information, checker.SecurityPolicyInformationTypeEmail, true) + urls := utils.CountSecInfo(policy.Information, checker.SecurityPolicyInformationTypeLink, true) + for _, i := range utils.FindSecInfo(policy.Information, checker.SecurityPolicyInformationTypeEmail, true) { + linkedContentLen += len(i.InformationValue.Match) + } + for _, i := range utils.FindSecInfo(policy.Information, checker.SecurityPolicyInformationTypeLink, true) { + linkedContentLen += len(i.InformationValue.Match) + } + + if policy.File.FileSize > 1 && (policy.File.FileSize > uint(linkedContentLen+((urls+emails)*2))) { + f, err := finding.NewPositive(fs, Probe, + "Found text in security policy", policy.File.Location()) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } else { + f, err := finding.NewNegative(fs, Probe, + "No text (besides links / emails) found in security policy", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + } + + if len(findings) == 0 { + f, err := finding.NewNegative(fs, Probe, "no security file to analyze", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/securityPolicyContainsText/impl_test.go b/probes/securityPolicyContainsText/impl_test.go new file mode 100644 index 00000000000..7cd63ff6ed4 --- /dev/null +++ b/probes/securityPolicyContainsText/impl_test.go @@ -0,0 +1,414 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyContainsText + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "file present on repo no text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on repo links no text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on repo links with short text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + FileSize: 10, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on repo links with long text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + FileSize: 50, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on repo no text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on org links no text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on org links with short text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + FileSize: 10, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on org links with long text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + FileSize: 50, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "files present on org and repo no text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "files present on org and repo short text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + FileSize: 10, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "files present on org and repo long text", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + FileSize: 50, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeEmail, + InformationValue: checker.SecurityPolicyValueType{ + Match: "myemail@google.com", + }, + }, + { + InformationType: checker.SecurityPolicyInformationTypeLink, + InformationValue: checker.SecurityPolicyValueType{ + Match: "https://www.bla.com", + }, + }, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomeNegative, + }, + }, + { + name: "file not present", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/securityPolicyContainsVulnerabilityDisclosure/def.yml b/probes/securityPolicyContainsVulnerabilityDisclosure/def.yml new file mode 100644 index 00000000000..ff8075ad03c --- /dev/null +++ b/probes/securityPolicyContainsVulnerabilityDisclosure/def.yml @@ -0,0 +1,40 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: securityPolicyContainsVulnerabilityDisclosure +short: Check that the security policy indicates a vulnerability disclosure process. +motivation: > + If someone finds a vulnerability in the project, it is important for them to be able to communicate it to the maintainers. +implementation: > + The implementation looks for strings "Disclos" and "Vuln". +outcome: + - If information about the disclosure process is found in a security policy file, the probe returns one finding with OutcomePositive (1) for each file. + - If no information about the disclosure process is found, the probe returns one finding with OutcomeNegative (0) for each file. + - if no file is present, the probe returns one finding with OutcomeNegative (0). +remediation: + effort: Low + text: + - 'On GitHub:' + - Enable private vulnerability disclosure in your repository settings https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to follow the steps in https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities. + - 'On GitLab:' + - Add a section in your SECURITY.md indicating the process to disclose vulnerabilities for your project. + - 'Examples: https://github.com/ossf/scorecard/blob/main/SECURITY.md, https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md, https://github.com/sigstore/.github/blob/main/SECURITY.md.' + markdown: + - 'On GitHub:' + - Enable private vulnerability disclosure in your [repository settings](https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository) + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to [follow these steps](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities). + - 'On GitLab:' + - Add a section in your SECURITY.md indicating the process to disclose vulnerabilities for your project. + - 'Examples: [OpenSSF Scorecard](https://github.com/ossf/scorecard/blob/main/SECURITY.md), [SLSA builders](https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md), [Sigstore](https://github.com/sigstore/.github/blob/main/SECURITY.md).' \ No newline at end of file diff --git a/probes/securityPolicyContainsVulnerabilityDisclosure/impl.go b/probes/securityPolicyContainsVulnerabilityDisclosure/impl.go new file mode 100644 index 00000000000..7410fbab2ee --- /dev/null +++ b/probes/securityPolicyContainsVulnerabilityDisclosure/impl.go @@ -0,0 +1,66 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyContainsVulnerabilityDisclosure + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "securityPolicyContainsVulnerabilityDisclosure" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } + var findings []finding.Finding + policies := raw.SecurityPolicyResults.PolicyFiles + for i := range policies { + policy := &policies[i] + discvuls := utils.CountSecInfo(policy.Information, checker.SecurityPolicyInformationTypeText, false) + if discvuls > 1 { + f, err := finding.NewPositive(fs, Probe, + "Found disclosure, vulnerability, and/or timelines in security policy", policy.File.Location()) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } else { + f, err := finding.NewNegative(fs, Probe, + "One or no descriptive hints of disclosure, vulnerability, and/or timelines in security policy", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + } + + if len(findings) == 0 { + f, err := finding.NewNegative(fs, Probe, "no security file to analyze", nil) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/securityPolicyContainsVulnerabilityDisclosure/impl_test.go b/probes/securityPolicyContainsVulnerabilityDisclosure/impl_test.go new file mode 100644 index 00000000000..d18b29a7288 --- /dev/null +++ b/probes/securityPolicyContainsVulnerabilityDisclosure/impl_test.go @@ -0,0 +1,269 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyContainsVulnerabilityDisclosure + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "file present on repo no vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on repo 2 vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on repo 1 vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on org no vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "file present on org 2 vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on org 1 vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "files present on org and repo", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "files present on org and repo 2 vuln", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + Information: []checker.SecurityPolicyInformation{ + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + { + InformationType: checker.SecurityPolicyInformationTypeText, + }, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomeNegative, + }, + }, + { + name: "file not present", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/securityPolicyPresent/def.yml b/probes/securityPolicyPresent/def.yml new file mode 100644 index 00000000000..e5cbe4ffa2c --- /dev/null +++ b/probes/securityPolicyPresent/def.yml @@ -0,0 +1,43 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: securityPolicyPresent +short: Check if a security policy is defined in the repository or in the org's .github repository. +motivation: > + A security policy (typically a SECURITY.md file) can give users information about what constitutes a vulnerability and how to report one securely so that information about a bug is not publicly visible. + If you have a large orgnization, having a unified security policy across all your repositories may simplify the vulnerability disclosure response. +implementation: > + The implementation looks for the presence of security policy files in the repository or in '/.github' repository. See https://github.com/ossf/scorecard/blob/main/checks/raw/security_policy.go#L139 for a detailed list of filenames. +outcome: + - If a security policy file is found, one finding with OutcomePositive (1) is returned. + - If no security file is found, one finding with OutcomeNegative (0) is returned. +remediation: + effort: Medium + text: + - 'On GitHub:' + - Enable private vulnerability disclosure in your repository settings https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to follow the steps in https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities. + - 'On GitLab:' + - Add a section in your SECURITY.md indicating the process to disclose vulnerabilities for your project. + - 'Examples: https://github.com/ossf/scorecard/blob/main/SECURITY.md, https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md, https://github.com/sigstore/.github/blob/main/SECURITY.md.' + - For additional information on vulnerability disclosure, see https://github.com/ossf/oss-vulnerability-guide/blob/main/maintainer-guide.md. + markdown: + - Write a short paragraph for your SECURITY.md to explain the process to disclose security vulnerability for your project. + - 'On GitHub:' + - Enable private vulnerability disclosure in your [repository settings](https://docs.github.com/en/code-security/security-advisories/repository-security-advisories/configuring-private-vulnerability-reporting-for-a-repository) + - Add a section in your SECURITY.md indicating you have enabled private reporting, and tell them to [follow these steps](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability to report vulnerabilities). + - 'On GitLab:' + - Provide a point of contact in your SECURITY.md. + - 'Examples: [OpenSSF Scorecard](https://github.com/ossf/scorecard/blob/main/SECURITY.md), [SLSA builders](https://github.com/slsa-framework/slsa-github-generator/blob/main/SECURITY.md), [Sigstore](https://github.com/sigstore/.github/blob/main/SECURITY.md).' + - For additional information on vulnerability disclosure, see [OpenSSF's maintainer's guide](https://github.com/ossf/oss-vulnerability-guide/blob/main/maintainer-guide.md). \ No newline at end of file diff --git a/probes/securityPolicyPresent/impl.go b/probes/securityPolicyPresent/impl.go new file mode 100644 index 00000000000..98dca841771 --- /dev/null +++ b/probes/securityPolicyPresent/impl.go @@ -0,0 +1,65 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyPresent + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "securityPolicyPresent" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } + var files []checker.File + for i := range raw.SecurityPolicyResults.PolicyFiles { + files = append(files, raw.SecurityPolicyResults.PolicyFiles[i].File) + } + + var findings []finding.Finding + for i := range files { + file := &files[i] + f, err := finding.NewWith(fs, Probe, "security policy file detected", + file.Location(), finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f = f.WithRemediationMetadata(raw.Metadata.Metadata) + findings = append(findings, *f) + } + + // No file found. + if len(findings) == 0 { + f, err := finding.NewWith(fs, Probe, "no security policy file detected", + nil, finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f = f.WithRemediationMetadata(raw.Metadata.Metadata) + findings = append(findings, *f) + } + + return findings, Probe, nil +} diff --git a/probes/securityPolicyPresent/impl_test.go b/probes/securityPolicyPresent/impl_test.go new file mode 100644 index 00000000000..15c804ddc5e --- /dev/null +++ b/probes/securityPolicyPresent/impl_test.go @@ -0,0 +1,138 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package securityPolicyPresent + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "file present on repo", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "file present on org", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "files present on org and repo", + raw: &checker.RawResults{ + SecurityPolicyResults: checker.SecurityPolicyData{ + PolicyFiles: []checker.SecurityPolicyFile{ + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeURL, + }, + }, + { + File: checker.File{ + Path: "SECURITY.md", + Type: finding.FileTypeText, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "file not present", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/toolDependabotInstalled/def.yml b/probes/toolDependabotInstalled/def.yml index 52da4bd6c39..5b1935ea76e 100644 --- a/probes/toolDependabotInstalled/def.yml +++ b/probes/toolDependabotInstalled/def.yml @@ -22,8 +22,8 @@ implementation: > the implementation checks whether commits are authored by Dependabot. If none of these succeed, Dependabot is not installed. NOTE: if the configuration files are found, the probe does not ensure that the Dependabot is run or that the Dependabot's pull requests are merged. outcome: - - If dependendabot is installed, the probe returns OutcomePositive (1) - - If dependendabot is not installed, the probe returns OutcomeNegative (0) + - If dependendabot is installed, the probe returns OutcomePositive (1) for each configuration. + - If dependendabot is not installed, the probe returns one OutcomeNegative (0). remediation: effort: Low text: diff --git a/probes/toolDependabotInstalled/impl.go b/probes/toolDependabotInstalled/impl.go index d89ba6c9140..02f26bb7f16 100644 --- a/probes/toolDependabotInstalled/impl.go +++ b/probes/toolDependabotInstalled/impl.go @@ -17,16 +17,17 @@ package toolDependabotInstalled import ( "embed" + "fmt" "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/probes/utils" + "github.com/ossf/scorecard/v4/probes/internal/utils" ) //go:embed *.yml var fs embed.FS -const probe = "toolDependabotInstalled" +const Probe = "toolDependabotInstalled" type dependabot struct{} @@ -39,12 +40,15 @@ func (t dependabot) Matches(tool *checker.Tool) bool { } func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } tools := raw.DependencyUpdateToolResults.Tools var matcher dependabot // Check whether Dependabot tool is installed on the repo, // and create the corresponding findings. //nolint:wrapcheck - return utils.ToolsRun(tools, fs, probe, + return utils.ToolsRun(tools, fs, Probe, // Tool found will generate a positive result. finding.OutcomePositive, // Tool not found will generate a negative result. diff --git a/probes/toolDependabotInstalled/impl_test.go b/probes/toolDependabotInstalled/impl_test.go new file mode 100644 index 00000000000..c564f137804 --- /dev/null +++ b/probes/toolDependabotInstalled/impl_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package toolDependabotInstalled + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "tool present", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "Dependabot", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "multiple correct tools", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "Dependabot", + }, + { + Name: "Dependabot", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "different tool name", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "not-Dependabot", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "empty results", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/toolPyUpInstalled/def.yml b/probes/toolPyUpInstalled/def.yml index 9529194cea3..1e2f1e175d3 100644 --- a/probes/toolPyUpInstalled/def.yml +++ b/probes/toolPyUpInstalled/def.yml @@ -22,8 +22,8 @@ implementation: > If the file is not found, PyUp is not installed. NOTE: the implementation does not ensure that PyUp is run or that PyUp's pull requests are merged. outcome: - - If PyUp is installed, the probe returns OutcomePositive (1) - - If PyUp is not installed, the probe returns OutcomeNegative (0) + - If PyUp is installed, the probe returns OutcomePositive (1) for each configuration. + - If PyUp is not installed, the probe returns OutcomeNegative (0). remediation: effort: Low text: diff --git a/probes/toolPyUpInstalled/impl.go b/probes/toolPyUpInstalled/impl.go index 42adb82685d..dbb1d42d1be 100644 --- a/probes/toolPyUpInstalled/impl.go +++ b/probes/toolPyUpInstalled/impl.go @@ -17,16 +17,17 @@ package toolPyUpInstalled import ( "embed" + "fmt" "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/probes/utils" + "github.com/ossf/scorecard/v4/probes/internal/utils" ) //go:embed *.yml var fs embed.FS -const probe = "toolPyUpInstalled" +const Probe = "toolPyUpInstalled" type pyup struct{} @@ -39,12 +40,15 @@ func (t pyup) Matches(tool *checker.Tool) bool { } func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } tools := raw.DependencyUpdateToolResults.Tools var matcher pyup // Check whether PyUp tool is installed on the repo, // and create the corresponding findings. //nolint:wrapcheck - return utils.ToolsRun(tools, fs, probe, + return utils.ToolsRun(tools, fs, Probe, // Tool found will generate a positive result. finding.OutcomePositive, // Tool not found will generate a negative result. diff --git a/probes/toolPyUpInstalled/impl_test.go b/probes/toolPyUpInstalled/impl_test.go new file mode 100644 index 00000000000..70833d6c349 --- /dev/null +++ b/probes/toolPyUpInstalled/impl_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package toolPyUpInstalled + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "tool present", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "PyUp", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "multiple correct tools", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "PyUp", + }, + { + Name: "PyUp", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "different tool name", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "not-PyUp", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "empty results", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/toolRenovateInstalled/def.yml b/probes/toolRenovateInstalled/def.yml index 72a9f106f25..7fc4f56261b 100644 --- a/probes/toolRenovateInstalled/def.yml +++ b/probes/toolRenovateInstalled/def.yml @@ -22,8 +22,8 @@ implementation: > If none of these files are found, Renovate is not installed. NOTE: the implementation does not ensure that Renovate is run or that Renovate's pull requests are merged. outcome: - - If Renovate is installed, the probe returns OutcomePositive (1) - - If Renovate is not installed, the probe returns OutcomeNegative (0) + - If Renovate is installed, the probe returns OutcomePositive (1) for each configuration. + - If Renovate is not installed, the probe returns OutcomeNegative (0). remediation: effort: Low text: diff --git a/probes/toolRenovateInstalled/impl.go b/probes/toolRenovateInstalled/impl.go index 1c3d0b91161..c03c2708728 100644 --- a/probes/toolRenovateInstalled/impl.go +++ b/probes/toolRenovateInstalled/impl.go @@ -17,16 +17,17 @@ package toolRenovateInstalled import ( "embed" + "fmt" "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/probes/utils" + "github.com/ossf/scorecard/v4/probes/internal/utils" ) //go:embed *.yml var fs embed.FS -const probe = "toolRenovateInstalled" +const Probe = "toolRenovateInstalled" type renovate struct{} @@ -39,12 +40,15 @@ func (t renovate) Matches(tool *checker.Tool) bool { } func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } tools := raw.DependencyUpdateToolResults.Tools var matcher renovate // Check whether Renovate tool is installed on the repo, // and create the corresponding findings. //nolint:wrapcheck - return utils.ToolsRun(tools, fs, probe, + return utils.ToolsRun(tools, fs, Probe, // Tool found will generate a positive result. finding.OutcomePositive, // Tool not found will generate a negative result. diff --git a/probes/toolRenovateInstalled/impl_test.go b/probes/toolRenovateInstalled/impl_test.go new file mode 100644 index 00000000000..7ebf91939ad --- /dev/null +++ b/probes/toolRenovateInstalled/impl_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package toolRenovateInstalled + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "tool present", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "RenovateBot", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "multiple correct tools", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "RenovateBot", + }, + { + Name: "RenovateBot", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "different tool name", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "not-RenovateBot", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "empty results", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/toolSonatypeLiftInstalled/def.yml b/probes/toolSonatypeLiftInstalled/def.yml index e2d38e1c967..c68dd3dffec 100644 --- a/probes/toolSonatypeLiftInstalled/def.yml +++ b/probes/toolSonatypeLiftInstalled/def.yml @@ -22,8 +22,8 @@ implementation: > If none of these files are found, Sonatype Lyft is not installed. NOTE: the implementation does not ensure that Sonatype Lyft is run or that Sonatype Lyft's pull requests are merged. outcome: - - If Sonatype Lyft is installed, the probe returns OutcomePositive (1) - - If Sonatype Lyft is not installed, the probe returns OutcomeNegative (0) + - If Sonatype Lyft is installed, the probe returns OutcomePositive (1) for each configuration. + - If Sonatype Lyft is not installed, the probe returns OutcomeNegative (0). remediation: effort: Low text: diff --git a/probes/toolSonatypeLiftInstalled/impl.go b/probes/toolSonatypeLiftInstalled/impl.go index 98d0363ae72..5f704db71f3 100644 --- a/probes/toolSonatypeLiftInstalled/impl.go +++ b/probes/toolSonatypeLiftInstalled/impl.go @@ -17,16 +17,17 @@ package toolSonatypeLiftInstalled import ( "embed" + "fmt" "github.com/ossf/scorecard/v4/checker" "github.com/ossf/scorecard/v4/finding" - "github.com/ossf/scorecard/v4/probes/utils" + "github.com/ossf/scorecard/v4/probes/internal/utils" ) //go:embed *.yml var fs embed.FS -const probe = "toolSonatypeLiftInstalled" +const Probe = "toolSonatypeLiftInstalled" type sonatypeLyft struct{} @@ -39,12 +40,15 @@ func (t sonatypeLyft) Matches(tool *checker.Tool) bool { } func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", utils.ErrNil) + } tools := raw.DependencyUpdateToolResults.Tools var matcher sonatypeLyft // Check whether Sona Lyft tool is installed on the repo, // and create the corresponding findings. //nolint:wrapcheck - return utils.ToolsRun(tools, fs, probe, + return utils.ToolsRun(tools, fs, Probe, // Tool found will generate a positive result. finding.OutcomePositive, // Tool not found will generate a negative result. diff --git a/probes/toolSonatypeLiftInstalled/impl_test.go b/probes/toolSonatypeLiftInstalled/impl_test.go new file mode 100644 index 00000000000..e24f862e3c0 --- /dev/null +++ b/probes/toolSonatypeLiftInstalled/impl_test.go @@ -0,0 +1,126 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package toolSonatypeLiftInstalled + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "tool present", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "Sonatype Lift", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "multiple correct tools", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "Sonatype Lift", + }, + { + Name: "Sonatype Lift", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "different tool name", + raw: &checker.RawResults{ + DependencyUpdateToolResults: checker.DependencyUpdateToolData{ + Tools: []checker.Tool{ + { + Name: "not-Sonatype Lift", + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "empty results", + raw: &checker.RawResults{}, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil raw", + err: utils.ErrNil, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +}