From 09a83559efc658af510d513368203f6411cee951 Mon Sep 17 00:00:00 2001 From: "Baruch Odem (Rothkoff)" Date: Sun, 10 Sep 2023 16:40:36 +0300 Subject: [PATCH] feat: change the include and exclude to select and ignore (#177) BREAKING CHANGE: `--include-rule` renamed to `--rule` `--exclude-rule` renamed to `--ignore-rule` Now you can use them together - redundant error return Part of #174 --- {scripts => .ci}/check_new_rules.go | 0 .github/workflows/new-rules.yml | 2 +- README.md | 4 +- cmd/main.go | 15 ++-- secrets/secrets.go | 64 +++++++------- secrets/secrets_test.go | 127 ++++++++++++++++++---------- 6 files changed, 121 insertions(+), 91 deletions(-) rename {scripts => .ci}/check_new_rules.go (100%) diff --git a/scripts/check_new_rules.go b/.ci/check_new_rules.go similarity index 100% rename from scripts/check_new_rules.go rename to .ci/check_new_rules.go diff --git a/.github/workflows/new-rules.yml b/.github/workflows/new-rules.yml index 0fa2213d..cf755d51 100644 --- a/.github/workflows/new-rules.yml +++ b/.github/workflows/new-rules.yml @@ -13,4 +13,4 @@ jobs: with: go-version: "^1.20" - name: Check Gitleaks new rules - run: go run scripts/check_new_rules.go + run: go run .ci/check_new_rules.go diff --git a/README.md b/README.md index 61d3473c..e4db2ad1 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,13 @@ Additional Commands: Flags: --config string config file path - --exclude-rule strings exclude rules by name or tag to apply to the scan (removes from list, starts from all) -h, --help help for 2ms --ignore-result strings ignore specific result by id - --include-rule strings include rules by name or tag to apply to the scan (adds to list, starts from empty) + --ignore-rule strings ignore rules by name or tag --log-level string log level (trace, debug, info, warn, error, fatal) (default "info") --regex stringArray custom regexes to apply to the scan, must be valid Go regex --report-path strings path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif) + --rule strings select rules by name or tag to apply to this scan --stdout-format string stdout output format, available formats are: json, yaml, sarif (default "yaml") -v, --version version for 2ms diff --git a/cmd/main.go b/cmd/main.go index c357a1e9..2a4f8f11 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -33,8 +33,8 @@ const ( reportPathFlagName = "report-path" stdoutFormatFlagName = "stdout-format" customRegexRuleFlagName = "regex" - includeRuleFlagName = "include-rule" - excludeRuleFlagName = "exclude-rule" + ruleFlagName = "rule" + ignoreRuleFlagName = "ignore-rule" ignoreFlagName = "ignore-result" ) @@ -43,8 +43,8 @@ var ( reportPathVar []string stdoutFormatVar string customRegexRuleVar []string - includeRuleVar []string - excludeRuleVar []string + ruleVar []string + ignoreRuleVar []string ignoreVar []string ) @@ -117,9 +117,8 @@ func Execute() { rootCmd.PersistentFlags().StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)") rootCmd.PersistentFlags().StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif") rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex") - rootCmd.PersistentFlags().StringSliceVar(&includeRuleVar, includeRuleFlagName, []string{}, "include rules by name or tag to apply to the scan (adds to list, starts from empty)") - rootCmd.PersistentFlags().StringSliceVar(&excludeRuleVar, excludeRuleFlagName, []string{}, "exclude rules by name or tag to apply to the scan (removes from list, starts from all)") - rootCmd.MarkFlagsMutuallyExclusive(includeRuleFlagName, excludeRuleFlagName) + rootCmd.PersistentFlags().StringSliceVar(&ruleVar, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan") + rootCmd.PersistentFlags().StringSliceVar(&ignoreRuleVar, ignoreRuleFlagName, []string{}, "ignore rules by name or tag") rootCmd.PersistentFlags().StringSliceVar(&ignoreVar, ignoreFlagName, []string{}, "ignore specific result by id") rootCmd.AddCommand(secrets.RulesCommand) @@ -160,7 +159,7 @@ func validateFormat(stdout string, reportPath []string) { func preRun(cmd *cobra.Command, args []string) { validateFormat(stdoutFormatVar, reportPathVar) - secrets, err := secrets.Init(includeRuleVar, excludeRuleVar) + secrets, err := secrets.Init(ruleVar, ignoreRuleVar) if err != nil { log.Fatal().Msg(err.Error()) } diff --git a/secrets/secrets.go b/secrets/secrets.go index 0a13e385..56b9f771 100644 --- a/secrets/secrets.go +++ b/secrets/secrets.go @@ -52,28 +52,29 @@ const TagWebhook = "webhook" const customRegexRuleIdFormat = "custom-regex-%d" -func Init(includeList, excludeList []string) (*Secrets, error) { - if len(includeList) > 0 && len(excludeList) > 0 { - return nil, fmt.Errorf("cannot use both include and exclude flags") +func Init(selectedList, ignoreList []string) (*Secrets, error) { + if len(selectedList) > 0 && len(ignoreList) > 0 { + log.Warn().Msgf("Both 'rule' and 'ignoreRule' flags were provided.") } - allRules, _ := loadAllRules() - rulesToBeApplied := make(map[string]config.Rule) - if len(includeList) > 0 { - rulesToBeApplied = selectRules(allRules, includeList) - } else if len(excludeList) > 0 { - rulesToBeApplied = excludeRules(allRules, excludeList) - } else { - for _, rule := range allRules { - // required to be empty when not running via cli. otherwise rule will be ignored - rule.Rule.Keywords = []string{} - rulesToBeApplied[rule.Rule.RuleID] = rule.Rule - } + selectedRules := loadAllRules() + if len(selectedList) > 0 { + selectedRules = selectRules(selectedRules, selectedList) + } + if len(ignoreList) > 0 { + selectedRules = ignoreRules(selectedRules, ignoreList) } - if len(rulesToBeApplied) == 0 { + if len(selectedRules) == 0 { return nil, fmt.Errorf("no rules were selected") } + rulesToBeApplied := make(map[string]config.Rule) + for _, rule := range selectedRules { + // required to be empty when not running via cli. otherwise rule will be ignored + rule.Rule.Keywords = []string{} + rulesToBeApplied[rule.Rule.RuleID] = rule.Rule + } + config := config.Config{ Rules: rulesToBeApplied, } @@ -144,30 +145,26 @@ func isSecretIgnored(secret *reporting.Secret, ignoredIds *[]string) bool { return false } -func selectRules(allRules []Rule, tags []string) map[string]config.Rule { - rulesToBeApplied := make(map[string]config.Rule) +func selectRules(allRules []Rule, tags []string) []Rule { + selectedRules := []Rule{} for _, rule := range allRules { if isRuleMatch(rule, tags) { - // required to be empty when not running via cli. otherwise rule will be ignored - rule.Rule.Keywords = []string{} - rulesToBeApplied[rule.Rule.RuleID] = rule.Rule + selectedRules = append(selectedRules, rule) } } - return rulesToBeApplied + return selectedRules } -func excludeRules(allRules []Rule, tags []string) map[string]config.Rule { - rulesToBeApplied := make(map[string]config.Rule) +func ignoreRules(allRules []Rule, tags []string) []Rule { + selectedRules := []Rule{} for _, rule := range allRules { if !isRuleMatch(rule, tags) { - // required to be empty when not running via cli. otherwise rule will be ignored - rule.Rule.Keywords = []string{} - rulesToBeApplied[rule.Rule.RuleID] = rule.Rule + selectedRules = append(selectedRules, rule) } } - return rulesToBeApplied + return selectedRules } func isRuleMatch(rule Rule, tags []string) bool { @@ -184,7 +181,7 @@ func isRuleMatch(rule Rule, tags []string) bool { return false } -func loadAllRules() ([]Rule, error) { +func loadAllRules() []Rule { var allRules []Rule allRules = make([]Rule, 0) @@ -346,7 +343,7 @@ func loadAllRules() ([]Rule, error) { allRules = append(allRules, Rule{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}}) allRules = append(allRules, Rule{Rule: *internalRules.AuthenticatedURL(), Tags: []string{TagSensitiveUrl}}) - return allRules, nil + return allRules } var RulesCommand = &cobra.Command{ @@ -355,10 +352,7 @@ var RulesCommand = &cobra.Command{ Long: `List all rules`, RunE: func(cmd *cobra.Command, args []string) error { - rules, err := loadAllRules() - if err != nil { - return err - } + rules := loadAllRules() tab := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0) @@ -367,7 +361,7 @@ var RulesCommand = &cobra.Command{ for _, rule := range rules { fmt.Fprintf(tab, "%s\t%s\t%s\n", rule.Rule.RuleID, rule.Rule.Description, strings.Join(rule.Tags, ",")) } - if err = tab.Flush(); err != nil { + if err := tab.Flush(); err != nil { return err } diff --git a/secrets/secrets_test.go b/secrets/secrets_test.go index a3d8c47a..61b74259 100644 --- a/secrets/secrets_test.go +++ b/secrets/secrets_test.go @@ -11,7 +11,7 @@ import ( ) func TestLoadAllRules(t *testing.T) { - rules, _ := loadAllRules() + rules := loadAllRules() if len(rules) <= 1 { t.Error("no rules were loaded") @@ -20,10 +20,7 @@ func TestLoadAllRules(t *testing.T) { func TestLoadAllRules_DuplicateRuleID(t *testing.T) { ruleIDMap := make(map[string]bool) - allRules, err := loadAllRules() - if err != nil { - t.Error(err) - } + allRules := loadAllRules() for _, rule := range allRules { if _, ok := ruleIDMap[rule.Rule.RuleID]; ok { @@ -34,53 +31,85 @@ func TestLoadAllRules_DuplicateRuleID(t *testing.T) { } } -func TestInit(t *testing.T) { - allRules, err := loadAllRules() - if err != nil { - t.Error(err) - } +func Test_Init_SelectRules(t *testing.T) { + allRules := loadAllRules() rulesCount := len(allRules) tests := []struct { - name string - includeList []string - excludeList []string - expectedErr error - expectedLen int + name string + selectedList []string + ignoreList []string + expectedErr error + expectedLen int }{ { - name: "include and exclude flags used together", - includeList: []string{"tag1"}, - excludeList: []string{"tag2"}, - expectedErr: fmt.Errorf("cannot use both include and exclude flags"), - expectedLen: 0, + name: "selected flag used for one rule", + selectedList: []string{allRules[0].Rule.RuleID}, + ignoreList: []string{}, + expectedErr: nil, + expectedLen: 1, }, { - name: "non existent include flag", - includeList: []string{"non-existent-tag-name"}, - excludeList: []string{}, - expectedErr: fmt.Errorf("no rules were selected"), - expectedLen: 0, + name: "selected flag used for multiple rules", + selectedList: []string{allRules[0].Rule.RuleID, allRules[1].Rule.RuleID}, + ignoreList: []string{}, + expectedErr: nil, + expectedLen: 2, }, { - name: "non existent exclude flag", - includeList: []string{}, - excludeList: []string{"non-existent-tag-name"}, - expectedErr: nil, - expectedLen: rulesCount, + name: "ignore flag used for one rule", + selectedList: []string{}, + ignoreList: []string{allRules[0].Rule.RuleID}, + expectedErr: nil, + expectedLen: rulesCount - 1, }, { - name: "no flags", - includeList: []string{}, - excludeList: []string{}, - expectedErr: nil, - expectedLen: rulesCount, + name: "ignore flag used for multiple rules", + selectedList: []string{}, + ignoreList: []string{allRules[0].Rule.RuleID, allRules[1].Rule.RuleID}, + expectedErr: nil, + expectedLen: rulesCount - 2, + }, + { + name: "selected and ignore flags used together for different rules", + selectedList: []string{allRules[0].Rule.RuleID}, + ignoreList: []string{allRules[1].Rule.RuleID}, + expectedErr: nil, + expectedLen: 1, + }, + { + name: "selected and ignore flags used together for the same rule", + selectedList: []string{allRules[0].Rule.RuleID}, + ignoreList: []string{allRules[0].Rule.RuleID}, + expectedErr: fmt.Errorf("no rules were selected"), + expectedLen: 0, + }, + { + name: "non existent select flag", + selectedList: []string{"non-existent-tag-name"}, + ignoreList: []string{}, + expectedErr: fmt.Errorf("no rules were selected"), + expectedLen: 0, + }, + { + name: "non existent ignore flag", + selectedList: []string{}, + ignoreList: []string{"non-existent-tag-name"}, + expectedErr: nil, + expectedLen: rulesCount, + }, + { + name: "no flags", + selectedList: []string{}, + ignoreList: []string{}, + expectedErr: nil, + expectedLen: rulesCount, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - secrets, err := Init(tt.includeList, tt.excludeList) + secrets, err := Init(tt.selectedList, tt.ignoreList) if err != nil { if tt.expectedErr == nil { @@ -88,7 +117,7 @@ func TestInit(t *testing.T) { } else if err.Error() == tt.expectedErr.Error() { return } else { - t.Errorf("expected error %s, but got %s", tt.expectedErr, err) + t.Errorf("expected error '%s', but got '%s'", tt.expectedErr, err) } } else if tt.expectedErr != nil { t.Errorf("expected error %s, but got none", tt.expectedErr) @@ -101,6 +130,14 @@ func TestInit(t *testing.T) { } } +func rulesToMap(rules []Rule) map[string]config.Rule { + rulesMap := make(map[string]config.Rule) + for _, rule := range rules { + rulesMap[rule.Rule.RuleID] = rule.Rule + } + return rulesMap +} + func TestSelectRules(t *testing.T) { testCases := []struct { name string @@ -158,7 +195,7 @@ func TestSelectRules(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result := selectRules(tc.allRules, tc.tags) + result := rulesToMap(selectRules(tc.allRules, tc.tags)) if len(result) != len(tc.expectedResult) { t.Errorf("Expected %d rules to be applied, but got %d", len(tc.expectedResult), len(result)) @@ -177,7 +214,7 @@ func TestSelectRules(t *testing.T) { } } -func TestExcludeRules(t *testing.T) { +func TestIgnoreRules(t *testing.T) { tests := []struct { name string allRules []Rule @@ -194,7 +231,7 @@ func TestExcludeRules(t *testing.T) { expectedResult: createRules("rule1", "rule2"), }, { - name: "Exclude non-existing tag", + name: "Ignore non-existing tag", allRules: []Rule{ createRule("rule1", "tag1", "tag2"), createRule("rule2", "tag2", "tag3"), @@ -203,7 +240,7 @@ func TestExcludeRules(t *testing.T) { expectedResult: createRules("rule1", "rule2"), }, { - name: "Exclude one rule ID", + name: "Ignore one rule ID", allRules: []Rule{ createRule("rule1", "tag1", "tag2"), createRule("rule2", "tag2", "tag3"), @@ -212,7 +249,7 @@ func TestExcludeRules(t *testing.T) { expectedResult: createRules("rule2"), }, { - name: "Exclude one tag", + name: "Ignore one tag", allRules: []Rule{ createRule("rule1", "tag1", "tag2"), createRule("rule2", "tag2", "tag3"), @@ -221,7 +258,7 @@ func TestExcludeRules(t *testing.T) { expectedResult: map[string]config.Rule{}, }, { - name: "Exclude all tags", + name: "Ignore all tags", allRules: []Rule{ createRule("rule1", "tag1", "tag2"), createRule("rule2", "tag2", "tag3"), @@ -233,7 +270,7 @@ func TestExcludeRules(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotResult := excludeRules(tt.allRules, tt.tags) + gotResult := rulesToMap(ignoreRules(tt.allRules, tt.tags)) if len(gotResult) != len(tt.expectedResult) { t.Errorf("expected %d rules, but got %d", len(tt.expectedResult), len(gotResult)) @@ -246,7 +283,7 @@ func TestExcludeRules(t *testing.T) { } } else { if _, ok := gotResult[rule.Rule.RuleID]; ok { - t.Errorf("expected rule %s to be excluded, but it was not", rule.Rule.RuleID) + t.Errorf("expected rule %s to be ignored, but it was not", rule.Rule.RuleID) } } }