diff --git a/CHANGELOG.md b/CHANGELOG.md index ccb666c..519b82a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. ### Added +* [#26](https://github.com/bitgrip/cattlectl/issues/26) bash completion +* [#28](https://github.com/bitgrip/cattlectl/issues/28) multi file includes + * directory includes + * pattern includes + ### Changed * [#15](https://github.com/bitgrip/cattlectl/issues/15) `--values` can be used multiple times. diff --git a/docs/project_descriptor.md b/docs/project_descriptor.md index 3679f9e..383c7b5 100644 --- a/docs/project_descriptor.md +++ b/docs/project_descriptor.md @@ -36,9 +36,11 @@ ProjectDescriptor Structur: #### include -| Field | Description | -|----------|--------------------------------------------| -| __file__ | The file (relative of absolute) to include | +| Field | Description | +|---------------|-----------------------------------------------------------------------| +| __file__ | The file (relative of absolute) to include | +| __files__ | The files pattern (relative of absolute) to include all matching files| +| __directory__ | The directory (relative of absolute) to include all YAML files from | #### namespaces diff --git a/internal/pkg/rancher/project/model/model.go b/internal/pkg/rancher/project/model/model.go index c1e2f09..142e5a6 100644 --- a/internal/pkg/rancher/project/model/model.go +++ b/internal/pkg/rancher/project/model/model.go @@ -70,7 +70,9 @@ type ProjectMetadata struct { // Include is used to merge multiple descriptors into one type Include struct { - File string `yaml:"file"` + File string `yaml:"file,omitempty"` + Files string `yaml:"files,omitempty"` + Directory string `yaml:"directory,omitempty"` } // Namespace is a subsection of a Project and is represented in K8S as namespace diff --git a/internal/pkg/rancher/project/parser.go b/internal/pkg/rancher/project/parser.go index 20e8beb..d7ad033 100644 --- a/internal/pkg/rancher/project/parser.go +++ b/internal/pkg/rancher/project/parser.go @@ -17,7 +17,9 @@ package project import ( "fmt" "io/ioutil" + "os" "path/filepath" + "sort" "github.com/bitgrip/cattlectl/internal/pkg/rancher/descriptor" projectModel "github.com/bitgrip/cattlectl/internal/pkg/rancher/project/model" @@ -80,33 +82,94 @@ func (parser fileParser) Parse(projectData []byte, target interface{}) error { return err } for _, include := range targetProject.Metadata.Includes { - var childProjectFile string - if filepath.IsAbs(include.File) { - childProjectFile = include.File - } else { - childProjectFile = filepath.Clean(fmt.Sprintf("%s/%s", filepath.Dir(parser.projectFile), include.File)) - } - childFileContent, err := ioutil.ReadFile(childProjectFile) + includeFiles, err := parser.readIncludeFiles(include) if err != nil { return err } - childProjectData, err := template.BuildTemplate(childFileContent, parser.values, filepath.Dir(childProjectFile), parser.pretty) - if err != nil { - return err + sort.Strings(includeFiles) + for _, includeFile := range includeFiles { + if err := parser.include(targetProject, includeFile, allProjectFiles); err != nil { + return err + } } - childTarget := projectModel.Project{} - childParser := newProjectParser(childProjectFile, parser.values, parser.pretty, allProjectFiles) - err = childParser.Parse(childProjectData, &childTarget) + } + + return nil +} + +func (parser fileParser) readIncludeFiles(include projectModel.Include) ([]string, error) { + if include.File != "" && include.Files != "" && include.Directory != "" { + return nil, fmt.Errorf("only one of file, files or directory can have a value") + } + if include.File != "" { + return []string{include.File}, nil + } + if include.Files != "" { + var absFiles string + if filepath.IsAbs(include.Files) { + absFiles = include.Files + } else { + var err error + absFiles, err = filepath.Abs(filepath.Clean(fmt.Sprintf("%s/%s", filepath.Dir(parser.projectFile), include.Files))) + if err != nil { + return nil, err + } + } + matches, err := filepath.Glob(absFiles) if err != nil { - return err + return nil, err + } + return matches, nil + } + if include.Directory != "" { + var absDirectory string + if filepath.IsAbs(include.Directory) { + absDirectory = include.Directory + } else { + absDirectory = filepath.Clean(fmt.Sprintf("%s/%s", filepath.Dir(parser.projectFile), include.Directory)) } - err = MergeProject(childTarget, targetProject) + var files []string + err := filepath.Walk(absDirectory, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + if filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" { + file, _ := filepath.Abs(path) + files = append(files, file) + } + return nil + }) if err != nil { - return err + return nil, err } + return files, nil } + return nil, fmt.Errorf("one of file, files or directory must have a value") +} - return nil +func (parser fileParser) include(targetProject *projectModel.Project, file string, allProjectFiles []string) error { + var childProjectFile string + if filepath.IsAbs(file) { + childProjectFile = file + } else { + childProjectFile = filepath.Clean(fmt.Sprintf("%s/%s", filepath.Dir(parser.projectFile), file)) + } + childFileContent, err := ioutil.ReadFile(childProjectFile) + if err != nil { + return err + } + childProjectData, err := template.BuildTemplate(childFileContent, parser.values, filepath.Dir(childProjectFile), parser.pretty) + if err != nil { + return err + } + childTarget := projectModel.Project{} + childParser := newProjectParser(childProjectFile, parser.values, parser.pretty, allProjectFiles) + err = childParser.Parse(childProjectData, &childTarget) + if err != nil { + return err + } + err = MergeProject(childTarget, targetProject) + return err } func isDescriptor(data []byte, kind string, logger *logrus.Entry) (bool, error) { diff --git a/internal/pkg/rancher/project/parser_test.go b/internal/pkg/rancher/project/parser_test.go index 6a9b458..1d200a9 100644 --- a/internal/pkg/rancher/project/parser_test.go +++ b/internal/pkg/rancher/project/parser_test.go @@ -96,6 +96,8 @@ func TestWithGoldenFile(t *testing.T) { tests := []string{ "simple-include", "cycle-include", + "files-include", + "directory-include", } for _, test := range tests { runTestWithGoldenFile(t, test) diff --git a/internal/pkg/rancher/project/testdata/directory-include/child-directory/child.1.yaml b/internal/pkg/rancher/project/testdata/directory-include/child-directory/child.1.yaml new file mode 100644 index 0000000..1d5c4a2 --- /dev/null +++ b/internal/pkg/rancher/project/testdata/directory-include/child-directory/child.1.yaml @@ -0,0 +1,56 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-child1 +namespaces: +- name: child1-namespace +resources: + certificates: + - name: child1-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child1-namespace + config_maps: + - name: child1-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: child1-registry + registries: + - name: child1.private.registry + password: child1-docker-registry-password + username: child1-docker-registry-user + secrets: + - name: child1-secret + data: + abc: def + bca: fed +storage_classes: +- name: child1-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: + - name: child1-persistent-volume + storage_class_name: child1-storage-classe +apps: +- name: child1-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/directory-include/child-directory/child.2.yaml b/internal/pkg/rancher/project/testdata/directory-include/child-directory/child.2.yaml new file mode 100644 index 0000000..f61b946 --- /dev/null +++ b/internal/pkg/rancher/project/testdata/directory-include/child-directory/child.2.yaml @@ -0,0 +1,56 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-child2 +namespaces: +- name: child2-namespace +resources: + certificates: + - name: child2-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child2-namespace + config_maps: + - name: child2-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: child2-registry + registries: + - name: child2.private.registry + password: child2-docker-registry-password + username: child2-docker-registry-user + secrets: + - name: child2-secret + data: + abc: def + bca: fed +storage_classes: +- name: child2-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: + - name: child2-persistent-volume + storage_class_name: child2-storage-classe +apps: +- name: child2-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/directory-include/project.yaml b/internal/pkg/rancher/project/testdata/directory-include/project.yaml new file mode 100644 index 0000000..239f7ac --- /dev/null +++ b/internal/pkg/rancher/project/testdata/directory-include/project.yaml @@ -0,0 +1,57 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-parent + includes: + - directory: child-directory +namespaces: +- name: parent-namespace +resources: + certificates: + - name: parent-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + config_maps: + - name: parent-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: parent-registry + registries: + - name: parent.private.registry + password: parent-docker-registry-password + username: parent-docker-registry-user + secrets: + - name: parent-secret + data: + abc: def + bca: fed +storage_classes: +- name: parent-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: + - name: parent-persistent-volume + storage_class_name: parent-storage-classe +apps: +- name: parent-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/files-include/child.1.yaml b/internal/pkg/rancher/project/testdata/files-include/child.1.yaml new file mode 100644 index 0000000..1d5c4a2 --- /dev/null +++ b/internal/pkg/rancher/project/testdata/files-include/child.1.yaml @@ -0,0 +1,56 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-child1 +namespaces: +- name: child1-namespace +resources: + certificates: + - name: child1-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child1-namespace + config_maps: + - name: child1-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: child1-registry + registries: + - name: child1.private.registry + password: child1-docker-registry-password + username: child1-docker-registry-user + secrets: + - name: child1-secret + data: + abc: def + bca: fed +storage_classes: +- name: child1-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: + - name: child1-persistent-volume + storage_class_name: child1-storage-classe +apps: +- name: child1-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/files-include/child.2.yaml b/internal/pkg/rancher/project/testdata/files-include/child.2.yaml new file mode 100644 index 0000000..f61b946 --- /dev/null +++ b/internal/pkg/rancher/project/testdata/files-include/child.2.yaml @@ -0,0 +1,56 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-child2 +namespaces: +- name: child2-namespace +resources: + certificates: + - name: child2-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child2-namespace + config_maps: + - name: child2-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: child2-registry + registries: + - name: child2.private.registry + password: child2-docker-registry-password + username: child2-docker-registry-user + secrets: + - name: child2-secret + data: + abc: def + bca: fed +storage_classes: +- name: child2-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: + - name: child2-persistent-volume + storage_class_name: child2-storage-classe +apps: +- name: child2-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/files-include/project.yaml b/internal/pkg/rancher/project/testdata/files-include/project.yaml new file mode 100644 index 0000000..7bff96e --- /dev/null +++ b/internal/pkg/rancher/project/testdata/files-include/project.yaml @@ -0,0 +1,57 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-parent + includes: + - files: child.*.yaml +namespaces: +- name: parent-namespace +resources: + certificates: + - name: parent-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + config_maps: + - name: parent-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: parent-registry + registries: + - name: parent.private.registry + password: parent-docker-registry-password + username: parent-docker-registry-user + secrets: + - name: parent-secret + data: + abc: def + bca: fed +storage_classes: +- name: parent-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: + - name: parent-persistent-volume + storage_class_name: parent-storage-classe +apps: +- name: parent-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/golden-files/directory-include.golden b/internal/pkg/rancher/project/testdata/golden-files/directory-include.golden new file mode 100644 index 0000000..3942713 --- /dev/null +++ b/internal/pkg/rancher/project/testdata/golden-files/directory-include.golden @@ -0,0 +1,161 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-parent + includes: + - directory: child-directory +namespaces: +- name: parent-namespace +- name: child1-namespace +- name: child2-namespace +resources: + certificates: + - name: parent-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + - name: child1-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child1-namespace + - name: child2-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child2-namespace + config_maps: + - name: parent-config-map + data: + abc: def + bca: fed + - name: child1-config-map + data: + abc: def + bca: fed + - name: child2-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: parent-registry + registries: + - name: parent.private.registry + password: parent-docker-registry-password + username: parent-docker-registry-user + - name: child1-registry + registries: + - name: child1.private.registry + password: child1-docker-registry-password + username: child1-docker-registry-user + - name: child2-registry + registries: + - name: child2.private.registry + password: child2-docker-registry-password + username: child2-docker-registry-user + secrets: + - name: parent-secret + data: + abc: def + bca: fed + - name: child1-secret + data: + abc: def + bca: fed + - name: child2-secret + data: + abc: def + bca: fed +storage_classes: +- name: parent-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +- name: child1-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +- name: child2-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: +- name: parent-persistent-volume + type: "" + path: "" + node: "" + storage_class_name: parent-storage-classe + access_modes: [] + capacity: "" + init_script: "" +- name: child1-persistent-volume + type: "" + path: "" + node: "" + storage_class_name: child1-storage-classe + access_modes: [] + capacity: "" + init_script: "" +- name: child2-persistent-volume + type: "" + path: "" + node: "" + storage_class_name: child2-storage-classe + access_modes: [] + capacity: "" + init_script: "" +apps: +- name: parent-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" +- name: child1-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" +- name: child2-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" diff --git a/internal/pkg/rancher/project/testdata/golden-files/files-include.golden b/internal/pkg/rancher/project/testdata/golden-files/files-include.golden new file mode 100644 index 0000000..91e11ec --- /dev/null +++ b/internal/pkg/rancher/project/testdata/golden-files/files-include.golden @@ -0,0 +1,161 @@ +api_version: v1.0 +kind: Project +metadata: + name: include-parent + includes: + - files: child.*.yaml +namespaces: +- name: parent-namespace +- name: child1-namespace +- name: child2-namespace +resources: + certificates: + - name: parent-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + - name: child1-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child1-namespace + - name: child2-cert + key: | + -----BEGIN PRIVATE KEY----- + ... + ... + -----END PRIVATE KEY----- + certs: | + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + ... + ... + -----END CERTIFICATE----- + namespace: child2-namespace + config_maps: + - name: parent-config-map + data: + abc: def + bca: fed + - name: child1-config-map + data: + abc: def + bca: fed + - name: child2-config-map + data: + abc: def + bca: fed + docker_credentials: + - name: parent-registry + registries: + - name: parent.private.registry + password: parent-docker-registry-password + username: parent-docker-registry-user + - name: child1-registry + registries: + - name: child1.private.registry + password: child1-docker-registry-password + username: child1-docker-registry-user + - name: child2-registry + registries: + - name: child2.private.registry + password: child2-docker-registry-password + username: child2-docker-registry-user + secrets: + - name: parent-secret + data: + abc: def + bca: fed + - name: child1-secret + data: + abc: def + bca: fed + - name: child2-secret + data: + abc: def + bca: fed +storage_classes: +- name: parent-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +- name: child1-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +- name: child2-storage-classe + provisioner: kubernetes.io/no-provisioner + reclaim_policy: Delete + volume_bind_mode: WaitForFirstConsumer +persistent_volumes: +- name: parent-persistent-volume + type: "" + path: "" + node: "" + storage_class_name: parent-storage-classe + access_modes: [] + capacity: "" + init_script: "" +- name: child1-persistent-volume + type: "" + path: "" + node: "" + storage_class_name: child1-storage-classe + access_modes: [] + capacity: "" + init_script: "" +- name: child2-persistent-volume + type: "" + path: "" + node: "" + storage_class_name: child2-storage-classe + access_modes: [] + capacity: "" + init_script: "" +apps: +- name: parent-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" +- name: child1-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false" +- name: child2-app + catalog: library + chart: wordpress + version: 2.1.10 + namespace: parent-namespace + answers: + ingress.enabled: "false"