From 6d0bc0fb9854447e441ef33f09cea3f840ce00f8 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Fri, 11 Aug 2023 18:45:03 +0300 Subject: [PATCH 01/15] added configuration for custom rules --- secrets/custom_rules.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 secrets/custom_rules.go diff --git a/secrets/custom_rules.go b/secrets/custom_rules.go new file mode 100644 index 00000000..ff6d8588 --- /dev/null +++ b/secrets/custom_rules.go @@ -0,0 +1,19 @@ +package secrets + +type CustomRuleConfiguration struct { + Description string + RegexPattern string + RuleID string + Tags []string + SecretGroup int +} + +var customRules = []CustomRuleConfiguration{ + { + Description: "Identify username:password inside URLS", + RuleID: "username-password-secret", + RegexPattern: ":\\/\\/(.+:.+)?@", + Tags: []string{TagPassword}, + SecretGroup: 1, + }, +} From c9245803ce0d0b11d53e3407fbb492e4274d8100 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Fri, 11 Aug 2023 18:45:31 +0300 Subject: [PATCH 02/15] added test to ensure custom rules are included --- secrets/secrets_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 4fd2a451..bb9bb427 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -31,6 +31,24 @@ func TestLoadAllRules_DuplicateRuleID(t *testing.T) { } } +func TestLoadAllRules_CustomRulesLoaded(t *testing.T) { + allRules, err := loadAllRules() + ruleIDMap := make(map[string]bool) + if err != nil { + t.Error(err) + } + + for _, rule := range allRules { + ruleIDMap[rule.Rule.RuleID] = true + } + for _, customRule := range customRules { + _, ok := ruleIDMap[customRule.RuleID] + if !ok { + t.Errorf("custom rule not found: %s", customRule.RuleID) + } + } +} + func TestIsAllFilter_AllFilterNotPresent(t *testing.T) { filters := []string{"token", "key"} From 29b817384258a59288268ec068e967974025accb Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Fri, 11 Aug 2023 18:46:36 +0300 Subject: [PATCH 03/15] appending custom rules to all rules array --- secrets/secrets.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/secrets/secrets.go b/secrets/secrets.go index d98bd3b0..c0976d93 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -128,6 +128,30 @@ func (s *Secrets) AddRegexRules(patterns []string) error { return nil } +func addCustomRules(rules []CustomRuleConfiguration) ([]Rule, error) { + var customRules []Rule + customRules = make([]Rule, len(rules)) + for idx, rule := range rules { + regex, err := regexp.Compile(rule.RegexPattern) + if err != nil { + return nil, fmt.Errorf("failed to compile custom regex rule %s: %w", rule.RuleID, err) + } + customRules[idx] = Rule{ + Rule: config.Rule{ + Description: rule.Description, + RuleID: rule.RuleID, + Regex: regex, + Keywords: []string{}, + }, + Tags: rule.Tags, + } + if rule.SecretGroup != 0 { + customRules[idx].Rule.SecretGroup = rule.SecretGroup + } + } + return customRules, nil +} + func getFindingId(item plugins.Item, finding report.Finding) string { idParts := []string{item.ID, finding.RuleID, finding.Secret} sha := sha1.Sum([]byte(strings.Join(idParts, "-"))) @@ -369,6 +393,12 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) + builtCustomRules, err := addCustomRules(customRules) + if err != nil { + return nil, err + } + allRules = append(allRules, builtCustomRules...) + return allRules, nil } From db49eb95cceb6cbfa5d8bd35846db320f0305b43 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Fri, 11 Aug 2023 18:45:03 +0300 Subject: [PATCH 04/15] added configuration for custom rules --- secrets/custom_rules.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 secrets/custom_rules.go diff --git a/secrets/custom_rules.go b/secrets/custom_rules.go new file mode 100644 index 00000000..ff6d8588 --- /dev/null +++ b/secrets/custom_rules.go @@ -0,0 +1,19 @@ +package secrets + +type CustomRuleConfiguration struct { + Description string + RegexPattern string + RuleID string + Tags []string + SecretGroup int +} + +var customRules = []CustomRuleConfiguration{ + { + Description: "Identify username:password inside URLS", + RuleID: "username-password-secret", + RegexPattern: ":\\/\\/(.+:.+)?@", + Tags: []string{TagPassword}, + SecretGroup: 1, + }, +} From 00f04107da44228414dfb7a32898fd734b4641ae Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Fri, 11 Aug 2023 18:45:31 +0300 Subject: [PATCH 05/15] added test to ensure custom rules are included --- secrets/secrets_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 4fd2a451..bb9bb427 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -31,6 +31,24 @@ func TestLoadAllRules_DuplicateRuleID(t *testing.T) { } } +func TestLoadAllRules_CustomRulesLoaded(t *testing.T) { + allRules, err := loadAllRules() + ruleIDMap := make(map[string]bool) + if err != nil { + t.Error(err) + } + + for _, rule := range allRules { + ruleIDMap[rule.Rule.RuleID] = true + } + for _, customRule := range customRules { + _, ok := ruleIDMap[customRule.RuleID] + if !ok { + t.Errorf("custom rule not found: %s", customRule.RuleID) + } + } +} + func TestIsAllFilter_AllFilterNotPresent(t *testing.T) { filters := []string{"token", "key"} From 92f1fe90437c8587628c70adef6fc41364391f01 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Fri, 11 Aug 2023 18:46:36 +0300 Subject: [PATCH 06/15] appending custom rules to all rules array --- secrets/secrets.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/secrets/secrets.go b/secrets/secrets.go index d98bd3b0..c0976d93 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -128,6 +128,30 @@ func (s *Secrets) AddRegexRules(patterns []string) error { return nil } +func addCustomRules(rules []CustomRuleConfiguration) ([]Rule, error) { + var customRules []Rule + customRules = make([]Rule, len(rules)) + for idx, rule := range rules { + regex, err := regexp.Compile(rule.RegexPattern) + if err != nil { + return nil, fmt.Errorf("failed to compile custom regex rule %s: %w", rule.RuleID, err) + } + customRules[idx] = Rule{ + Rule: config.Rule{ + Description: rule.Description, + RuleID: rule.RuleID, + Regex: regex, + Keywords: []string{}, + }, + Tags: rule.Tags, + } + if rule.SecretGroup != 0 { + customRules[idx].Rule.SecretGroup = rule.SecretGroup + } + } + return customRules, nil +} + func getFindingId(item plugins.Item, finding report.Finding) string { idParts := []string{item.ID, finding.RuleID, finding.Secret} sha := sha1.Sum([]byte(strings.Join(idParts, "-"))) @@ -369,6 +393,12 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) + builtCustomRules, err := addCustomRules(customRules) + if err != nil { + return nil, err + } + allRules = append(allRules, builtCustomRules...) + return allRules, nil } From a5b1949545b2e500720cd5432b6e3633819fd4fa Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Mon, 14 Aug 2023 21:29:45 +0300 Subject: [PATCH 07/15] test to validate rule finds username:password --- secrets/secrets_test.go | 75 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index bb9bb427..b07b4518 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -2,11 +2,86 @@ package secrets import ( "fmt" + "sync" "testing" + "github.com/checkmarx/2ms/plugins" + "github.com/checkmarx/2ms/reporting" "github.com/zricethezav/gitleaks/v8/config" ) +func TestSecrets(t *testing.T) { + secrets := []struct { + Content string + Name string + ShouldFind bool + }{ + { + Content: "", + Name: "empty", + ShouldFind: false, + }, + { + Content: "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "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", + Name: "GitHub Personal Access Token", + ShouldFind: true, + }, + { + Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT", + Name: "JFROG Secret without keyword", + ShouldFind: false, + }, + { + Content: "--set imagePullSecretJfrog.password=AKCp8kqqfQbYifrbyvqusjyk6N3QKprXTv9B8HTitLbJzXT1kW7dDticXTsJpCrbqtizAwK4D \\", + Name: "JFROG Secret with keyword (real example)", + ShouldFind: true, + }, + { + Content: "--docker-password=AKCp8kqX8yeKBTqgm2XExHsp8yVdJn6SAgQmS1nJMfMDmzxEqX74rUGhedaWu7Eovid3VsMwb", + Name: "JFROG Secret as kubectl argument", + ShouldFind: true, + }, + } + + detector, err := Init([]string{}, []string{}) + if err != nil { + t.Fatal(err) + } + + for _, secret := range secrets { + name := secret.Name + if name == "" { + name = secret.Content + } + t.Run(name, func(t *testing.T) { + fmt.Printf("Start test %s", name) + secretsChan := make(chan reporting.Secret, 1) + wg := &sync.WaitGroup{} + wg.Add(1) + detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) + close(secretsChan) + + s := <-secretsChan + if s.Value == "" && secret.ShouldFind { + t.Errorf("secret \"%s\" not found", secret.Name) + } + if s.Value != "" && !secret.ShouldFind { + t.Errorf("should not find") + } + }) + } + +} func TestLoadAllRules(t *testing.T) { rules, _ := loadAllRules() From a12612ee14aba83fa7de1fd37e3e92cad3fa519e Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Mon, 14 Aug 2023 21:31:03 +0300 Subject: [PATCH 08/15] remove non relevant test --- secrets/secrets_test.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index b07b4518..d8618d5f 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -106,24 +106,6 @@ func TestLoadAllRules_DuplicateRuleID(t *testing.T) { } } -func TestLoadAllRules_CustomRulesLoaded(t *testing.T) { - allRules, err := loadAllRules() - ruleIDMap := make(map[string]bool) - if err != nil { - t.Error(err) - } - - for _, rule := range allRules { - ruleIDMap[rule.Rule.RuleID] = true - } - for _, customRule := range customRules { - _, ok := ruleIDMap[customRule.RuleID] - if !ok { - t.Errorf("custom rule not found: %s", customRule.RuleID) - } - } -} - func TestIsAllFilter_AllFilterNotPresent(t *testing.T) { filters := []string{"token", "key"} From 912f603b2ceb36eae871f6bea28d00b4efa1b50a Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Mon, 14 Aug 2023 21:37:53 +0300 Subject: [PATCH 09/15] changing abstraction explicitly adding rule --- secrets/custom_rules.go | 19 ------------------ secrets/rules/authenticated_url.go | 20 +++++++++++++++++++ secrets/secrets.go | 32 ++---------------------------- 3 files changed, 22 insertions(+), 49 deletions(-) delete mode 100644 secrets/custom_rules.go create mode 100644 secrets/rules/authenticated_url.go diff --git a/secrets/custom_rules.go b/secrets/custom_rules.go deleted file mode 100644 index ff6d8588..00000000 --- a/secrets/custom_rules.go +++ /dev/null @@ -1,19 +0,0 @@ -package secrets - -type CustomRuleConfiguration struct { - Description string - RegexPattern string - RuleID string - Tags []string - SecretGroup int -} - -var customRules = []CustomRuleConfiguration{ - { - Description: "Identify username:password inside URLS", - RuleID: "username-password-secret", - RegexPattern: ":\\/\\/(.+:.+)?@", - Tags: []string{TagPassword}, - SecretGroup: 1, - }, -} diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go new file mode 100644 index 00000000..6ce78b9f --- /dev/null +++ b/secrets/rules/authenticated_url.go @@ -0,0 +1,20 @@ +package secrets + +import ( + "regexp" + + "github.com/zricethezav/gitleaks/v8/config" +) + +func AuthenticatedURL() *config.Rule { + regex, _ := regexp.Compile(":\\/\\/(.+:.+)?@") + rule := config.Rule{ + Description: "Identify username:password inside URLS", + RuleID: "username-password-secret", + Regex: regex, + Keywords: []string{}, + SecretGroup: 1, + } + + return &rule +} diff --git a/secrets/secrets.go b/secrets/secrets.go index c0976d93..eaf838e1 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -11,6 +11,7 @@ import ( "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" + secrets "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" @@ -128,30 +129,6 @@ func (s *Secrets) AddRegexRules(patterns []string) error { return nil } -func addCustomRules(rules []CustomRuleConfiguration) ([]Rule, error) { - var customRules []Rule - customRules = make([]Rule, len(rules)) - for idx, rule := range rules { - regex, err := regexp.Compile(rule.RegexPattern) - if err != nil { - return nil, fmt.Errorf("failed to compile custom regex rule %s: %w", rule.RuleID, err) - } - customRules[idx] = Rule{ - Rule: config.Rule{ - Description: rule.Description, - RuleID: rule.RuleID, - Regex: regex, - Keywords: []string{}, - }, - Tags: rule.Tags, - } - if rule.SecretGroup != 0 { - customRules[idx].Rule.SecretGroup = rule.SecretGroup - } - } - return customRules, nil -} - func getFindingId(item plugins.Item, finding report.Finding) string { idParts := []string{item.ID, finding.RuleID, finding.Secret} sha := sha1.Sum([]byte(strings.Join(idParts, "-"))) @@ -392,12 +369,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) - - builtCustomRules, err := addCustomRules(customRules) - if err != nil { - return nil, err - } - allRules = append(allRules, builtCustomRules...) + allRules = append(allRules, Rule{Rule: *secrets.AuthenticatedURL(), Tags: []string{TagPassword}}) return allRules, nil } From 288268c986b92c62918b08730d9ec9d037a4b420 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Mon, 14 Aug 2023 21:43:07 +0300 Subject: [PATCH 10/15] removing unused code --- secrets/custom_rules.go | 19 ------------------- secrets/secrets.go | 30 ------------------------------ secrets/secrets_test.go | 18 ------------------ 3 files changed, 67 deletions(-) delete mode 100644 secrets/custom_rules.go diff --git a/secrets/custom_rules.go b/secrets/custom_rules.go deleted file mode 100644 index ff6d8588..00000000 --- a/secrets/custom_rules.go +++ /dev/null @@ -1,19 +0,0 @@ -package secrets - -type CustomRuleConfiguration struct { - Description string - RegexPattern string - RuleID string - Tags []string - SecretGroup int -} - -var customRules = []CustomRuleConfiguration{ - { - Description: "Identify username:password inside URLS", - RuleID: "username-password-secret", - RegexPattern: ":\\/\\/(.+:.+)?@", - Tags: []string{TagPassword}, - SecretGroup: 1, - }, -} diff --git a/secrets/secrets.go b/secrets/secrets.go index 69379d88..eaf838e1 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -129,30 +129,6 @@ func (s *Secrets) AddRegexRules(patterns []string) error { return nil } -func addCustomRules(rules []CustomRuleConfiguration) ([]Rule, error) { - var customRules []Rule - customRules = make([]Rule, len(rules)) - for idx, rule := range rules { - regex, err := regexp.Compile(rule.RegexPattern) - if err != nil { - return nil, fmt.Errorf("failed to compile custom regex rule %s: %w", rule.RuleID, err) - } - customRules[idx] = Rule{ - Rule: config.Rule{ - Description: rule.Description, - RuleID: rule.RuleID, - Regex: regex, - Keywords: []string{}, - }, - Tags: rule.Tags, - } - if rule.SecretGroup != 0 { - customRules[idx].Rule.SecretGroup = rule.SecretGroup - } - } - return customRules, nil -} - func getFindingId(item plugins.Item, finding report.Finding) string { idParts := []string{item.ID, finding.RuleID, finding.Secret} sha := sha1.Sum([]byte(strings.Join(idParts, "-"))) @@ -395,12 +371,6 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) allRules = append(allRules, Rule{Rule: *secrets.AuthenticatedURL(), Tags: []string{TagPassword}}) - builtCustomRules, err := addCustomRules(customRules) - if err != nil { - return nil, err - } - allRules = append(allRules, builtCustomRules...) - return allRules, nil } diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index b07b4518..d8618d5f 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -106,24 +106,6 @@ func TestLoadAllRules_DuplicateRuleID(t *testing.T) { } } -func TestLoadAllRules_CustomRulesLoaded(t *testing.T) { - allRules, err := loadAllRules() - ruleIDMap := make(map[string]bool) - if err != nil { - t.Error(err) - } - - for _, rule := range allRules { - ruleIDMap[rule.Rule.RuleID] = true - } - for _, customRule := range customRules { - _, ok := ruleIDMap[customRule.RuleID] - if !ok { - t.Errorf("custom rule not found: %s", customRule.RuleID) - } - } -} - func TestIsAllFilter_AllFilterNotPresent(t *testing.T) { filters := []string{"token", "key"} From ded2b27f80c7a40ab1b4737ad3f2b1041b553977 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Sun, 20 Aug 2023 21:05:10 +0300 Subject: [PATCH 11/15] using raw string to avoid having to escape twice --- secrets/rules/authenticated_url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go index 6ce78b9f..c4ce543b 100644 --- a/secrets/rules/authenticated_url.go +++ b/secrets/rules/authenticated_url.go @@ -7,7 +7,7 @@ import ( ) func AuthenticatedURL() *config.Rule { - regex, _ := regexp.Compile(":\\/\\/(.+:.+)?@") + regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`) rule := config.Rule{ Description: "Identify username:password inside URLS", RuleID: "username-password-secret", From 9ba72718e02d1a1060285f5cc85279c23c464cc5 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Sun, 20 Aug 2023 21:06:21 +0300 Subject: [PATCH 12/15] changed import name and tag --- secrets/secrets.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/secrets/secrets.go b/secrets/secrets.go index eaf838e1..25b475e7 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -11,7 +11,7 @@ import ( "github.com/checkmarx/2ms/plugins" "github.com/checkmarx/2ms/reporting" - secrets "github.com/checkmarx/2ms/secrets/rules" + internalRules "github.com/checkmarx/2ms/secrets/rules" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" @@ -369,7 +369,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.YandexAWSAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}}) allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) - allRules = append(allRules, Rule{Rule: *secrets.AuthenticatedURL(), Tags: []string{TagPassword}}) + allRules = append(allRules, Rule{Rule: *internalRules.AuthenticatedURL(), Tags: []string{TagSensitiveUrl}}) return allRules, nil } From 222becb26f19afc94848e2a4e17349d2682f0a99 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Tue, 22 Aug 2023 13:29:06 +0300 Subject: [PATCH 13/15] merging TestSecrets test --- secrets/secrets_test.go | 97 +++++++++++------------------------------ 1 file changed, 25 insertions(+), 72 deletions(-) diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index 5662aecf..7d662320 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -10,78 +10,6 @@ import ( "github.com/zricethezav/gitleaks/v8/config" ) -func TestSecrets(t *testing.T) { - secrets := []struct { - Content string - Name string - ShouldFind bool - }{ - { - Content: "", - Name: "empty", - ShouldFind: false, - }, - { - Content: "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", - Name: "Authenticated URL", - ShouldFind: true, - }, - { - Content: "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", - Name: "Authenticated URL", - ShouldFind: true, - }, - { - Content: "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", - Name: "GitHub Personal Access Token", - ShouldFind: true, - }, - { - Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT", - Name: "JFROG Secret without keyword", - ShouldFind: false, - }, - { - Content: "--set imagePullSecretJfrog.password=AKCp8kqqfQbYifrbyvqusjyk6N3QKprXTv9B8HTitLbJzXT1kW7dDticXTsJpCrbqtizAwK4D \\", - Name: "JFROG Secret with keyword (real example)", - ShouldFind: true, - }, - { - Content: "--docker-password=AKCp8kqX8yeKBTqgm2XExHsp8yVdJn6SAgQmS1nJMfMDmzxEqX74rUGhedaWu7Eovid3VsMwb", - Name: "JFROG Secret as kubectl argument", - ShouldFind: true, - }, - } - - detector, err := Init([]string{}, []string{}) - if err != nil { - t.Fatal(err) - } - - for _, secret := range secrets { - name := secret.Name - if name == "" { - name = secret.Content - } - t.Run(name, func(t *testing.T) { - fmt.Printf("Start test %s", name) - secretsChan := make(chan reporting.Secret, 1) - wg := &sync.WaitGroup{} - wg.Add(1) - detector.Detect(plugins.Item{Content: secret.Content}, secretsChan, wg, nil) - close(secretsChan) - - s := <-secretsChan - if s.Value == "" && secret.ShouldFind { - t.Errorf("secret \"%s\" not found", secret.Name) - } - if s.Value != "" && !secret.ShouldFind { - t.Errorf("should not find") - } - }) - } - -} func TestLoadAllRules(t *testing.T) { rules, _ := loadAllRules() @@ -436,6 +364,31 @@ func TestSecrets(t *testing.T) { Name string ShouldFind bool }{ + { + Content: "", + Name: "empty", + ShouldFind: false, + }, + { + Content: "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + Name: "Authenticated URL", + ShouldFind: true, + }, + { + Content: "https://abc:123@google.com", + Name: "Basic Authenticated URL", + ShouldFind: true, + }, + { + Content: "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", + Name: "GitHub Personal Access Token", + ShouldFind: true, + }, { Content: "AKCp8jRRiQSAbghbuZmHKZcaKGEqbAASGH2SAb3rxXJQsSq9dGga8gFXe6aHpcRmzuHxN6oaT", Name: "JFROG Secret without keyword", From a2492c1ed3ba6dcd4658945224d120d22b404436 Mon Sep 17 00:00:00 2001 From: Hagar Fisher Date: Tue, 22 Aug 2023 20:41:58 +0300 Subject: [PATCH 14/15] change rule id Co-authored-by: Baruch Odem (Rothkoff) --- secrets/rules/authenticated_url.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go index c4ce543b..715197f8 100644 --- a/secrets/rules/authenticated_url.go +++ b/secrets/rules/authenticated_url.go @@ -10,7 +10,7 @@ func AuthenticatedURL() *config.Rule { regex, _ := regexp.Compile(`:\/\/(.+:.+)?@`) rule := config.Rule{ Description: "Identify username:password inside URLS", - RuleID: "username-password-secret", + RuleID: "authenticated-url", Regex: regex, Keywords: []string{}, SecretGroup: 1, From 319a704daa5db45d92cc704d0ecf3f3aa4de9ec0 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 23 Aug 2023 09:31:55 +0300 Subject: [PATCH 15/15] add positives to the rule definition --- secrets/rules/authenticated_url.go | 15 ++++++++++-- secrets/rules/rule.go | 37 ++++++++++++++++++++++++++++++ secrets/rules/rule_test.go | 28 ++++++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 secrets/rules/rule.go create mode 100644 secrets/rules/rule_test.go diff --git a/secrets/rules/authenticated_url.go b/secrets/rules/authenticated_url.go index 715197f8..999533f1 100644 --- a/secrets/rules/authenticated_url.go +++ b/secrets/rules/authenticated_url.go @@ -1,4 +1,4 @@ -package secrets +package rules import ( "regexp" @@ -16,5 +16,16 @@ func AuthenticatedURL() *config.Rule { SecretGroup: 1, } - return &rule + tPositives := []string{ + "mongodb+srv://radar:mytoken@io.dbb.mongodb.net/?retryWrites=true&w=majority", + "--output=https://elastic:bF21iC0bfTVXo3qhpJqTGs78@c22f5bc9787c4c268d3b069ad866bdc2.eu-central-1.aws.cloud.es.io:9243/tfs", + "https://abc:123@google.com", + } + + fPositives := []string{ + "https://google.com", + "https://google.com?user=abc&password=123", + } + + return validate(rule, tPositives, fPositives) } diff --git a/secrets/rules/rule.go b/secrets/rules/rule.go new file mode 100644 index 00000000..59d4b36e --- /dev/null +++ b/secrets/rules/rule.go @@ -0,0 +1,37 @@ +package rules + +import ( + "strings" + + "github.com/rs/zerolog/log" + "github.com/zricethezav/gitleaks/v8/config" + "github.com/zricethezav/gitleaks/v8/detect" +) + +// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go +func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule { + // normalize keywords like in the config package + var keywords []string + for _, k := range r.Keywords { + keywords = append(keywords, strings.ToLower(k)) + } + r.Keywords = keywords + + rules := make(map[string]config.Rule) + rules[r.RuleID] = r + d := detect.NewDetector(config.Config{ + Rules: rules, + Keywords: keywords, + }) + for _, tp := range truePositives { + if len(d.DetectString(tp)) != 1 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], true positive [%s] was not detected by regexp [%s]", r.RuleID, tp, r.Regex) + } + } + for _, fp := range falsePositives { + if len(d.DetectString(fp)) != 0 { + log.Fatal().Msgf("Failed to validate. For rule ID [%s], false positive [%s] was detected by regexp [%s]", r.RuleID, fp, r.Regex) + } + } + return &r +} diff --git a/secrets/rules/rule_test.go b/secrets/rules/rule_test.go new file mode 100644 index 00000000..34e27da3 --- /dev/null +++ b/secrets/rules/rule_test.go @@ -0,0 +1,28 @@ +package rules_test + +import ( + "testing" + + "github.com/checkmarx/2ms/secrets/rules" + "github.com/zricethezav/gitleaks/v8/config" +) + +func Test2msRules(t *testing.T) { + t.Parallel() + + testsRules := []struct { + name string + validate func() *config.Rule + }{ + {name: "AuthenticatedURL", validate: rules.AuthenticatedURL}, + } + + for _, tRule := range testsRules { + testRule := tRule // fix for loop variable being captured by func literal + t.Run(testRule.name, func(t *testing.T) { + t.Parallel() + + testRule.validate() + }) + } +}