Merge pull request #6611 from Checkmarx/kics-987

fix(query): countLines, IgnoreLines and fileCommands

e2e/fixtures/E2E_CLI_083_RESULT.json

@@ -0,0 +1,28 @@
+{
+	"kics_version": "development",
+	"files_scanned": 3,
+	"lines_scanned": 89,
+	"files_parsed": 3,
+	"lines_parsed": 86,
+	"lines_ignored": 3,
+	"files_failed_to_scan": 0,
+	"queries_total": 1,
+	"queries_failed_to_execute": 0,
+	"queries_failed_to_compute_similarity_id": 0,
+	"scan_id": "console",
+	"severity_counters": {
+		"HIGH": 0,
+		"INFO": 0,
+		"LOW": 0,
+		"MEDIUM": 0,
+		"TRACE": 0
+	},
+	"total_counter": 0,
+	"total_bom_resources": 0,
+	"start": "2024-02-06T12:29:45.3845776Z",
+	"end": "2024-02-06T12:29:49.5261723Z",
+	"paths": [
+		"/path/test/fixtures/helm_ignore"
+	],
+	"queries": []
+}

e2e/fixtures/E2E_CLI_084_RESULT.json

@@ -0,0 +1,28 @@
+{
+	"kics_version": "development",
+	"files_scanned": 3,
+	"lines_scanned": 89,
+	"files_parsed": 3,
+	"lines_parsed": 34,
+	"lines_ignored": 55,
+	"files_failed_to_scan": 0,
+	"queries_total": 1,
+	"queries_failed_to_execute": 0,
+	"queries_failed_to_compute_similarity_id": 0,
+	"scan_id": "console",
+	"severity_counters": {
+		"HIGH": 0,
+		"INFO": 0,
+		"LOW": 0,
+		"MEDIUM": 0,
+		"TRACE": 0
+	},
+	"total_counter": 0,
+	"total_bom_resources": 0,
+	"start": "2024-02-06T12:29:45.3845776Z",
+	"end": "2024-02-06T12:29:49.5261723Z",
+	"paths": [
+    "/path/test/fixtures/helm_ignore_block"
+	],
+	"queries": []
+}

e2e/fixtures/E2E_CLI_085_RESULT.json

@@ -0,0 +1,28 @@
+{
+  "kics_version": "development",
+  "files_scanned": 3,
+  "lines_scanned": 89,
+  "files_parsed": 3,
+  "lines_parsed": 86,
+  "lines_ignored": 3,
+  "files_failed_to_scan": 0,
+  "queries_total": 1,
+  "queries_failed_to_execute": 0,
+  "queries_failed_to_compute_similarity_id": 0,
+  "scan_id": "console",
+  "severity_counters": {
+    "HIGH": 0,
+    "INFO": 0,
+    "LOW": 0,
+    "MEDIUM": 0,
+    "TRACE": 0
+  },
+  "total_counter": 0,
+  "total_bom_resources": 0,
+  "start": "2024-02-06T15:01:20.657455Z",
+  "end": "2024-02-06T15:01:25.1183483Z",
+  "paths": [
+    "/path/test/fixtures/helm_disable_query"
+  ],
+  "queries": []
+}

e2e/testcases/e2e-cli-083_helm_ignore.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-083 - KICS  scan
+// should perform a scan and return zero results ignoring the file
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan and return zero results ignoring the file [E2E-CLI-083]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_083_RESULT",
+					"-p", "\"/path/test/fixtures/helm_ignore\"",
+					"-i", "b7652612-de4e-4466-a0bf-1cd81f0c6063",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_083_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-084_helm_ignore_block.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-084 - KICS  scan
+// should perform a scan and return zero results ignoring the block
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan and return zero results ignoring the block [E2E-CLI-084]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_084_RESULT",
+					"-p", "\"/path/test/fixtures/helm_ignore_block\"",
+					"-i", "b7652612-de4e-4466-a0bf-1cd81f0c6063",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_084_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/testcases/e2e-cli-085_helm_disable_query.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-085 - KICS  scan
+// should perform a scan and return zero results ignoring the query
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan and return zero results ignoring the query [E2E-CLI-085]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_085_RESULT",
+					"-p", "\"/path/test/fixtures/helm_disable_query\"",
+					"-i", "b7652612-de4e-4466-a0bf-1cd81f0c6063",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_085_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{0},
+	}
+
+	Tests = append(Tests, testSample)
+}

internal/tracker/ci.go

@@ -29,6 +29,9 @@ type CITracker struct {
 	ParsedCountLines   int
 	IgnoreCountLines   int
 	Version            model.Version
+	BagOfFilesParse    map[string]int
+	BagOfFilesFound    map[string]int
+	syncFileMutex      sync.Mutex
 }
 
 // NewTracker will create a new instance of a tracker with the number of lines to display in results output
@@ -39,7 +42,9 @@ func NewTracker(previewLines int) (*CITracker, error) {
 			fmt.Errorf("output lines minimum is %v and maximum is %v", constants.MinimumPreviewLines, constants.MaximumPreviewLines)
 	}
 	return &CITracker{
-		lines: previewLines,
+		lines:           previewLines,
+		BagOfFilesParse: make(map[string]int),
+		BagOfFilesFound: make(map[string]int),
 	}, nil
 }
 
@@ -66,13 +71,29 @@ func (c *CITracker) TrackQueryExecution(queryAggregation int) {
 }
 
 // TrackFileFound adds a found file to be scanned
-func (c *CITracker) TrackFileFound() {
-	c.FoundFiles++
+func (c *CITracker) TrackFileFound(path string) {
+	c.syncFileMutex.Lock()
+	defer c.syncFileMutex.Unlock()
+	count, value := c.BagOfFilesFound[path]
+	if !value {
+		c.BagOfFilesFound[path] = 1
+		c.FoundFiles++
+	} else {
+		c.BagOfFilesFound[path] = count + 1
+	}
 }
 
 // TrackFileParse adds a successful parsed file to be scanned
-func (c *CITracker) TrackFileParse() {
-	c.ParsedFiles++
+func (c *CITracker) TrackFileParse(path string) {
+	c.syncFileMutex.Lock()
+	defer c.syncFileMutex.Unlock()
+	count, value := c.BagOfFilesParse[path]
+	if !value {
+		c.BagOfFilesParse[path] = 1
+		c.ParsedFiles++
+	} else {
+		c.BagOfFilesParse[path] = count + 1
+	}
 }
 
 // FailedDetectLine - queries that fail to detect line are counted as failed to execute queries

internal/tracker/ci_test.go

@@ -70,6 +70,8 @@ func TestCITracker(t *testing.T) {
 			ParsedCountLines:   tt.fields.ParsedCountLines,
 			IgnoreCountLines:   tt.fields.IgnoreCountLines,
 			lines:              tt.fields.lines,
+			BagOfFilesParse:    make(map[string]int),
+			BagOfFilesFound:    make(map[string]int),
 		}
 		t.Run(fmt.Sprintf(tt.name+"_LoadedQueries"), func(t *testing.T) {
 			c.TrackQueryLoad(1)
@@ -82,12 +84,12 @@ func TestCITracker(t *testing.T) {
 		})
 
 		t.Run(fmt.Sprintf(tt.name+"_TrackFileFound"), func(t *testing.T) {
-			c.TrackFileFound()
+			c.TrackFileFound(tt.name)
 			require.Equal(t, 1, c.FoundFiles)
 		})
 
 		t.Run(fmt.Sprintf(tt.name+"_TrackFileParse"), func(t *testing.T) {
-			c.TrackFileParse()
+			c.TrackFileParse(tt.name)
 			require.Equal(t, 1, c.ParsedFiles)
 		})
 		t.Run(fmt.Sprintf(tt.name+"_TrackQueryExecuting"), func(t *testing.T) {
@@ -152,7 +154,9 @@ func TestNewTracker(t *testing.T) {
 				outputLines: 3,
 			},
 			want: CITracker{
-				lines: 3,
+				lines:           3,
+				BagOfFilesFound: make(map[string]int),
+				BagOfFilesParse: make(map[string]int),
 			},
 			wantErr: false,
 		},

pkg/detector/helm/helm_detect.go

@@ -39,7 +39,10 @@ const (
 func (d DetectKindLine) DetectLine(file *model.FileMetadata, searchKey string,
 	outputLines int, logWithFields *zerolog.Logger) model.VulnerabilityLines {
 	searchKey = fmt.Sprintf("%s.%s", strings.TrimRight(strings.TrimLeft(file.HelmID, "# "), ":"), searchKey)
-	lines := *file.LinesOriginalData
+
+	lines := make([]string, len(*file.LinesOriginalData))
+	copy(lines, *file.LinesOriginalData)
+
 	curLineRes := detectCurlLine{
 		foundRes: false,
 		lineRes:  0,

pkg/kics/resolver_sink.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"sort"
 
 	sentryReport "github.com/Checkmarx/kics/internal/sentry"
@@ -27,9 +28,7 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 	}
 
 	for _, rfile := range resFiles.File {
-		s.Tracker.TrackFileFound()
-		countLines := bytes.Count(rfile.Content, []byte{'\n'}) + 1
-		s.Tracker.TrackFileFoundCountLines(countLines)
+		s.Tracker.TrackFileFound(rfile.FileName)
 
 		isMinified := minified.IsMinified(rfile.FileName, rfile.Content)
 		documents, err := s.Parser.Parse(rfile.FileName, rfile.Content, openAPIResolveReferences, isMinified)
@@ -40,6 +39,21 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 			log.Err(err).Msgf("failed to parse file content")
 			return []string{}, nil
 		}
+
+		if kind == model.KindHELM {
+			ignoreList, errorIL := s.getOriginalIgnoreLines(rfile.FileName, rfile.OriginalData, openAPIResolveReferences, isMinified)
+			if errorIL == nil {
+				documents.IgnoreLines = ignoreList
+
+				// Need to ignore #KICS_HELM_ID Line
+				documents.CountLines = bytes.Count(rfile.OriginalData, []byte{'\n'})
+			}
+		} else {
+			documents.CountLines = bytes.Count(rfile.OriginalData, []byte{'\n'}) + 1
+		}
+
+		fileCommands := s.Parser.CommentsCommands(rfile.FileName, rfile.OriginalData)
+
 		for _, document := range documents.Docs {
 			_, err = json.Marshal(document)
 			if err != nil {
@@ -67,6 +81,7 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 				FilePath:          rfile.FileName,
 				Content:           string(rfile.Content),
 				HelmID:            rfile.SplitID,
+				Commands:          fileCommands,
 				IDInfo:            rfile.IDInfo,
 				LinesIgnore:       documents.IgnoreLines,
 				ResolvedFiles:     documents.ResolvedFiles,
@@ -75,9 +90,23 @@ func (s *Service) resolverSink(ctx context.Context, filename, scanID string, ope
 			}
 			s.saveToFile(ctx, &file)
 		}
-		s.Tracker.TrackFileParse()
+		s.Tracker.TrackFileParse(rfile.FileName)
+		s.Tracker.TrackFileFoundCountLines(documents.CountLines)
 		s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines))
 		s.Tracker.TrackFileIgnoreCountLines(len(documents.IgnoreLines))
 	}
 	return resFiles.Excluded, nil
 }
+
+func (s *Service) getOriginalIgnoreLines(filename string,
+	originalFile []uint8,
+	openAPIResolveReferences, isMinified bool) (ignoreLines []int, err error) {
+	refactor := regexp.MustCompile(`.*\n?.*KICS\_HELM\_ID.+\n`).ReplaceAll(originalFile, []uint8{})
+	refactor = regexp.MustCompile(`{{-\s*(.*?)\s*}}`).ReplaceAll(refactor, []uint8{})
+
+	documentsOriginal, err := s.Parser.Parse(filename, refactor, openAPIResolveReferences, isMinified)
+	if err == nil {
+		ignoreLines = documentsOriginal.IgnoreLines
+	}
+	return
+}

pkg/kics/service.go

@@ -40,8 +40,8 @@ type Storage interface {
 // TrackFileFound should increment the number of files to be scanned
 // TrackFileParse should increment the number of files parsed successfully to be scanned
 type Tracker interface {
-	TrackFileFound()
-	TrackFileParse()
+	TrackFileFound(path string)
+	TrackFileParse(path string)
 	TrackFileFoundCountLines(countLines int)
 	TrackFileParseCountLines(countLines int)
 	TrackFileIgnoreCountLines(countLines int)

pkg/kics/sink.go

@@ -29,7 +29,7 @@ var (
 func (s *Service) sink(ctx context.Context, filename, scanID string,
 	rc io.Reader, data []byte,
 	openAPIResolveReferences bool) error {
-	s.Tracker.TrackFileFound()
+	s.Tracker.TrackFileFound(filename)
 	log.Debug().Msgf("Starting to process file %s", filename)
 
 	c, err := getContent(rc, data, s.MaxFileSize, filename)
@@ -92,7 +92,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string,
 
 		s.saveToFile(ctx, &file)
 	}
-	s.Tracker.TrackFileParse()
+	s.Tracker.TrackFileParse(filename)
 	log.Debug().Msgf("Finished to process file %s", filename)
 
 	s.Tracker.TrackFileParseCountLines(documents.CountLines - len(documents.IgnoreLines))