feat(TKC-3022): add cone: true flag for content.files (#6095)

* feat(TKC-3022): add `cone: true` flag for content.files
* chore: update testkube-operator to main

api/v1/testkube.yaml

@@ -9307,6 +9307,9 @@ components:
         mountPath:
           type: string
           description: where to mount the fetched repository contents (defaults to "repo" directory in the data volume)
+        cone:
+          type: boolean
+          description: enable cone mode for sparse checkout with paths
         paths:
           type: array
           description: paths to fetch for the sparse checkout

cmd/testworkflow-toolkit/commands/clone.go

@@ -35,6 +35,7 @@ func NewCloneCmd() *cobra.Command {
 		sshKey   string
 		authType string
 		revision string
+		cone     bool
 	)
 
 	cmd := &cobra.Command{
@@ -59,6 +60,10 @@ func NewCloneCmd() *cobra.Command {
 			paths := make([]string, 0)
 			for _, p := range rawPaths {
 				p = filepath.Clean(p)
+				if cone && p != "/" && strings.HasPrefix(p, "/") {
+					// Delete leading '/' for cone
+					p = p[1:]
+				}
 				if p != "" && p != "." {
 					paths = append(paths, p)
 				}
@@ -114,7 +119,11 @@ func NewCloneCmd() *cobra.Command {
 				ui.Debug("sparse checkout")
 				err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "clone", configArgs, authArgs, "--filter=blob:none", "--no-checkout", "--sparse", "--depth", 1, "--verbose", uri.String(), outputPath)
 				ui.ExitOnError("cloning repository", err)
-				err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "-C", outputPath, configArgs, "sparse-checkout", "set", "--no-cone", paths)
+				coneArgs := []string{"--no-cone"}
+				if cone {
+					coneArgs = nil
+				}
+				err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "-C", outputPath, configArgs, "sparse-checkout", "set", coneArgs, paths)
 				ui.ExitOnError("sparse checkout repository", err)
 				if revision != "" {
 					err = RunWithRetry(CloneRetryOnFailureMaxAttempts, CloneRetryOnFailureBaseDelay, "git", "-C", outputPath, configArgs, "fetch", authArgs, "--depth", 1, "origin", revision)
@@ -168,6 +177,7 @@ func NewCloneCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&sshKey, "sshKey", "s", "", "")
 	cmd.Flags().StringVarP(&authType, "authType", "a", "basic", "allowed: basic, header")
 	cmd.Flags().StringVarP(&revision, "revision", "r", "", "commit hash, branch name or tag")
+	cmd.Flags().BoolVar(&cone, "cone", false, "should enable cone mode for sparse checkout")
 
 	return cmd
 }

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/keygen-sh/jsonapi-go v1.2.1
 	github.com/keygen-sh/keygen-go/v3 v3.2.0
 	github.com/kubepug/kubepug v1.7.1
-	github.com/kubeshop/testkube-operator v1.17.55-0.20241219071935-0330347ee160
+	github.com/kubeshop/testkube-operator v1.17.55-0.20241219143413-6a33aef7130f
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/montanaflynn/stats v0.7.1
 	github.com/moogar0880/problems v0.1.1

go.sum

@@ -336,8 +336,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw=
 github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g=
-github.com/kubeshop/testkube-operator v1.17.55-0.20241219071935-0330347ee160 h1:Bocn/Cgx/XLEsqvU3IdK5U/cFDJDFD3tES/1KS2wLkY=
-github.com/kubeshop/testkube-operator v1.17.55-0.20241219071935-0330347ee160/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
+github.com/kubeshop/testkube-operator v1.17.55-0.20241219143413-6a33aef7130f h1:7wDuGy4MohoFcgz2QPH/02JWCsgBqkWdGgefIH2YSEE=
+github.com/kubeshop/testkube-operator v1.17.55-0.20241219143413-6a33aef7130f/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=

pkg/api/v1/testkube/model_test_workflow_content_git.go

@@ -26,6 +26,8 @@ type TestWorkflowContentGit struct {
 	AuthType   *ContentGitAuthType `json:"authType,omitempty"`
 	// where to mount the fetched repository contents (defaults to \"repo\" directory in the data volume)
 	MountPath string `json:"mountPath,omitempty"`
+	// enable cone mode for sparse checkout with paths
+	Cone bool `json:"cone,omitempty"`
 	// paths to fetch for the sparse checkout
 	Paths []string `json:"paths,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go

@@ -36,11 +36,11 @@ type TestWorkflowIndependentServiceSpec struct {
 	Logs           *BoxedString                       `json:"logs,omitempty"`
 	RestartPolicy  string                             `json:"restartPolicy,omitempty"`
 	ReadinessProbe *Probe                             `json:"readinessProbe,omitempty"`
+	Pvcs           map[string]TestWorkflowPvcConfig   `json:"pvcs,omitempty"`
 	Count          *BoxedString                       `json:"count,omitempty"`
 	MaxCount       *BoxedString                       `json:"maxCount,omitempty"`
 	// matrix of parameters to spawn instances
 	Matrix map[string]interface{} `json:"matrix,omitempty"`
 	// parameters that should be distributed across sharded instances
-	Shards map[string]interface{}           `json:"shards,omitempty"`
-	Pvcs   map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"`
+	Shards map[string]interface{} `json:"shards,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_service_spec.go

@@ -36,12 +36,12 @@ type TestWorkflowServiceSpec struct {
 	Logs           *BoxedString                       `json:"logs,omitempty"`
 	RestartPolicy  string                             `json:"restartPolicy,omitempty"`
 	ReadinessProbe *Probe                             `json:"readinessProbe,omitempty"`
+	Pvcs           map[string]TestWorkflowPvcConfig   `json:"pvcs,omitempty"`
 	Use            []TestWorkflowTemplateRef          `json:"use,omitempty"`
 	Count          *BoxedString                       `json:"count,omitempty"`
 	MaxCount       *BoxedString                       `json:"maxCount,omitempty"`
 	// matrix of parameters to spawn instances
 	Matrix map[string]interface{} `json:"matrix,omitempty"`
 	// parameters that should be distributed across sharded instances
-	Shards map[string]interface{}           `json:"shards,omitempty"`
-	Pvcs   map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"`
+	Shards map[string]interface{} `json:"shards,omitempty"`
 }

pkg/mapper/testworkflows/kube_openapi.go

@@ -403,6 +403,7 @@ func MapContentGitKubeToAPI(v testworkflowsv1.ContentGit) testkube.TestWorkflowC
 		SshKeyFrom:   common.MapPtr(v.SshKeyFrom, MapEnvVarSourceKubeToAPI),
 		AuthType:     MapGitAuthTypeKubeToAPI(v.AuthType),
 		MountPath:    v.MountPath,
+		Cone:         v.Cone,
 		Paths:        v.Paths,
 	}
 }

pkg/mapper/testworkflows/openapi_kube.go

@@ -293,6 +293,7 @@ func MapContentGitAPIToKube(v testkube.TestWorkflowContentGit) testworkflowsv1.C
 		SshKeyFrom:   common.MapPtr(v.SshKeyFrom, MapEnvVarSourceAPIToKube),
 		AuthType:     MapGitAuthTypeAPIToKube(v.AuthType),
 		MountPath:    v.MountPath,
+		Cone:         v.Cone,
 		Paths:        v.Paths,
 	}
 }

pkg/testworkflows/testworkflowprocessor/operations.go

@@ -181,10 +181,15 @@ func ProcessContentGit(_ InternalProcessor, layer Intermediate, container stage.
 		SetWorkingDir("/").
 		SetImage(constants.DefaultToolkitImage).
 		SetImagePullPolicy(corev1.PullIfNotPresent).
-		SetCommand("/toolkit", "clone", step.Content.Git.Uri).
+		SetCommand("/toolkit", "clone").
 		EnableToolkit(stage.Ref())
 
-	args := []string{mountPath}
+	args := []string{step.Content.Git.Uri, mountPath}
+
+	// Enable cone mode if expected
+	if step.Content.Git.Cone {
+		args = append([]string{"--cone"}, args...)
+	}
 
 	// Provide Git username
 	if step.Content.Git.UsernameFrom != nil {