From ef817ca6f708d513505a81f9a32fadb41ef14ba0 Mon Sep 17 00:00:00 2001 From: Nicolas Degory Date: Thu, 14 May 2020 08:48:25 -0700 Subject: [PATCH] post-renderer option Signed-off-by: Nicolas Degory --- Makefile | 2 +- docs/desired_state_specification.md | 41 ++++++++--------- internal/app/plan.go | 4 +- internal/app/release.go | 69 ++++++++++++++++++----------- internal/app/release_test.go | 35 +++++++++++++++ tests/overlay.sample.yaml | 13 ++++++ tests/post-renderer.sh | 3 ++ 7 files changed, 118 insertions(+), 49 deletions(-) create mode 100644 tests/overlay.sample.yaml create mode 100755 tests/post-renderer.sh diff --git a/Makefile b/Makefile index bc42ca57..0ab5e68b 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ generate: .PHONY: generate repo: - @helm repo add stable https://kubernetes-charts.storage.googleapis.com + @helm repo list | grep -q "^stable " || helm repo add stable https://kubernetes-charts.storage.googleapis.com .PHONY: repo test: deps vet repo ## Run unit tests diff --git a/docs/desired_state_specification.md b/docs/desired_state_specification.md index 15483e81..10bf2bce 100644 --- a/docs/desired_state_specification.md +++ b/docs/desired_state_specification.md @@ -347,34 +347,35 @@ Releases must have unique names which are defined under `apps`. Example: in `[ap Options: **Required** -- **namespace** : the namespace where the release should be deployed. The namespace should map to one of the ones defined in [namespaces](#namespaces). -- **enabled** : describes the required state of the release (true for enabled, false for disabled). Once a release is deployed, you can change it to false if you want to delete this release [default is false]. -- **chart** : the chart name. It should contain the repo name as well. Example: repoName/chartName. Changing the chart name means delete and reinstall this release using the new Chart. -- **version** : the chart version. +- **namespace** : the namespace where the release should be deployed. The namespace should map to one of the ones defined in [namespaces](#namespaces). +- **enabled** : describes the required state of the release (true for enabled, false for disabled). Once a release is deployed, you can change it to false if you want to delete this release [default is false]. +- **chart** : the chart name. It should contain the repo name as well. Example: repoName/chartName. Changing the chart name means delete and reinstall this release using the new Chart. +- **version** : the chart version. **Optional** -- **group** : group name this apps belongs to. It has no effect until Helmsman's flag `-group` is passed. Check this [doc](how_to/misc/limit-deployment-to-specific-group-of-apps.md) for more details. -- **description** : a release metadata for human readers. -- **valuesFile** : a valid path (URL, cloud bucket, local absolute/relative file path) to custom Helm values.yaml file. File extension must be `yaml`. Cannot be used with valuesFiles together. Leaving it empty uses the default chart values. -- **valuesFiles** : array of valid paths (URL, cloud bucket, local absolute/relative file path) to custom Helm values.yaml file. File extension must be `yaml`. Cannot be used with valuesFile together. Leaving it empty uses the default chart values. +- **group** : group name this apps belongs to. It has no effect until Helmsman's flag `-group` is passed. Check this [doc](how_to/misc/limit-deployment-to-specific-group-of-apps.md) for more details. +- **description** : a release metadata for human readers. +- **valuesFile** : a valid path (URL, cloud bucket, local absolute/relative file path) to custom Helm values.yaml file. File extension must be `yaml`. Cannot be used with valuesFiles together. Leaving it empty uses the default chart values. +- **valuesFiles** : array of valid paths (URL, cloud bucket, local absolute/relative file path) to custom Helm values.yaml file. File extension must be `yaml`. Cannot be used with valuesFile together. Leaving it empty uses the default chart values. > The values file(s) path is resolved when the DSF yaml/toml file is loaded, relative to the path that the dsf was loaded from. - **secretsFile** : a valid path (URL, cloud bucket, local absolute/relative file path) to custom Helm secrets.yaml file. File extension must be `yaml`. Cannot be used with secretsFiles together. Leaving it empty uses the default chart secrets. - **secretsFiles** : array of valid paths (URL, cloud bucket, local absolute/relative file path) to custom Helm secrets.yaml file. File extension must be `yaml`. Cannot be used with secretsFile together. Leaving it empty uses the default chart secrets. > The secrets file(s) path is resolved when the DSF yaml/toml file is loaded, relative to the path that the dsf was loaded from. > To use the secrets files you must have the helm-secrets plugin -- **test** : defines whether to run the chart tests whenever the release is installed. Default is false. -- **protected** : defines if the release should be protected against changes. Namespace-level protection has higher priority than this flag. Check the [protection guide](how_to/misc/protect_namespaces_and_releases.md) for more details. Default is false. -- **wait** : defines whether Helmsman should block execution until all k8s resources are in a ready state. Default is false. -- **timeout** : helm timeout in seconds. Default 300 seconds. -- **noHooks** : helm noHooks option. If true, it will disable pre/post upgrade hooks. Default is false. -- **priority** : defines the priority of applying operations on this release. Only negative values allowed and the lower the value, the higher the priority. Default priority is 0. Apps with equal priorities will be applied in the order they were added in your state file (DSF). -- **set** : is used to override certain values from values.yaml with values from environment variables (or ,starting from v1.3.0-rc, directly provided in the Desired State File). This is particularly useful for passing secrets to charts. If the an environment variable with the same name as the provided value exists, the environment variable value will be used, otherwise, the provided value will be used as is. The TOML stanza for this is `[apps..set]` -- **setString** : is used to override String values from values.yaml or chart's defaults. This uses the `--set-string` flag in helm which is available only in helm >v2.9.0. This option is useful for image tags and the like. The TOML stanza for this is `[apps..setString]` -- **setFile** : is used to override values from values.yaml or chart's defaults from provided file. This uses the `--set-file` flag in helm. This option is useful for embedding file contents in the values. The TOML stanza for this is `[apps..setFile]` -> set, setString and setFile can't take nested elements. If you need to provide nested values, you can combine them in one line with dots e.g. `TOML: "image.tag"=some_value` `YAML: "image.tag": some_value` -- **helmFlags** : array of `helm` upgrade flags, is used to pass flags to helm install/upgrade commands. **These flags are not passed to helm diff**. For setting values, use **set**, **setString** or **setFile** instead. +- **test** : defines whether to run the chart tests whenever the release is installed. Default is false. +- **protected** : defines if the release should be protected against changes. Namespace-level protection has higher priority than this flag. Check the [protection guide](how_to/misc/protect_namespaces_and_releases.md) for more details. Default is false. +- **wait** : defines whether Helmsman should block execution until all k8s resources are in a ready state. Default is false. +- **timeout** : helm timeout in seconds. Default 300 seconds. +- **noHooks** : helm noHooks option. If true, it will disable pre/post upgrade hooks. Default is false. +- **priority** : defines the priority of applying operations on this release. Only negative values allowed and the lower the value, the higher the priority. Default priority is 0. Apps with equal priorities will be applied in the order they were added in your state file (DSF). +- **set** : is used to override certain values from values.yaml with values from environment variables (or ,starting from v1.3.0-rc, directly provided in the Desired State File). This is particularly useful for passing secrets to charts. If the an environment variable with the same name as the provided value exists, the environment variable value will be used, otherwise, the provided value will be used as is. The TOML stanza for this is `[apps..set]` +- **setString** : is used to override String values from values.yaml or chart's defaults. This uses the `--set-string` flag in helm which is available only in helm >v2.9.0. This option is useful for image tags and the like. The TOML stanza for this is `[apps..setString]` +- **setFile** : is used to override values from values.yaml or chart's defaults from provided file. This uses the `--set-file` flag in helm. This option is useful for embedding file contents in the values. The TOML stanza for this is `[apps..setFile]` +> set, setString and setFile can't take nested elements. If you need to provide nested values, you can combine them in one line with dots e.g. `TOML: "image.tag"=some\_value` `YAML: "image.tag": some\_value` +- **helmFlags** : array of `helm` upgrade flags, is used to pass flags to helm install/upgrade commands. **These flags are not passed to helm diff**. For setting values, use **set**, **setString** or **setFile** instead. - **hooks** : defines global lifecycle hooks to apply yaml manifest before and/or after different helmsman operations. Check [here](how_to/apps/lifecycle_hooks.md) for more details. Unset hooks for a release are inherited from `globalHooks` in the [settings](#Settings) stanza. -- **maxHistory** : defines the maximum number of helm revisions state (secrets/configmap) to keep. If unset, it will inherit the value of `settings.globalMaxHistory`, if that's also unset, it defaults to 10. +- **maxHistory** : defines the maximum number of helm revisions state (secrets/configmap) to keep. If unset, it will inherit the value of `settings.globalMaxHistory`, if that's also unset, it defaults to 10. +- **postRenderer** : the path to an executable to be used for post rendering (requires Helm 3.1+ and helm-diff v3.1.2+) Example: diff --git a/internal/app/plan.go b/internal/app/plan.go index 481b3ccc..b3f4a57f 100644 --- a/internal/app/plan.go +++ b/internal/app/plan.go @@ -140,7 +140,7 @@ func releaseWithHooks(cmd orderedCommand, wg *sync.WaitGroup, sem chan struct{}, return } } - if err:= execOne(cmd.Command, cmd.targetRelease); err != nil { + if err := execOne(cmd.Command, cmd.targetRelease); err != nil { errors <- err log.Verbose(err.Error()) return @@ -171,7 +171,7 @@ func execOne(cmd command, targetRelease *release) error { targetRelease.Name, result.code, strings.TrimSpace(errorMsg)) } else { return fmt.Errorf("%s returned [ %d ] exit code and error message [ %s ]", - cmd.Description, result.code, strings.TrimSpace(errorMsg)) + cmd.Description, result.code, strings.TrimSpace(errorMsg)) } } else { diff --git a/internal/app/release.go b/internal/app/release.go index 8256f51b..0385f612 100644 --- a/internal/app/release.go +++ b/internal/app/release.go @@ -25,6 +25,7 @@ type release struct { ValuesFiles []string `yaml:"valuesFiles"` SecretsFile string `yaml:"secretsFile"` SecretsFiles []string `yaml:"secretsFiles"` + PostRenderer string `yaml:"postRenderer"` Test bool `yaml:"test"` Protected bool `yaml:"protected"` Wait bool `yaml:"wait"` @@ -114,6 +115,12 @@ func (r *release) validate(appLabel string, names map[string]map[string]bool, s } } + if r.PostRenderer != "" { + if _, err := os.Stat(r.PostRenderer); err != nil { + return fmt.Errorf(r.PostRenderer + " must be valid relative (from dsf file) file path.") + } + } + if r.Priority != 0 && r.Priority > 0 { return errors.New("priority can only be 0 or negative value, positive values are not allowed") } @@ -252,7 +259,7 @@ func validateChart(apps, chart, version string, c chan string) { // If chart is local, returns the given release version func getChartVersion(chart, version string) (string, string) { if isLocalChart(chart) { - log.Info("Chart [ " + chart + "] with version [ " + version + " ] was found locally.") + log.Info("Chart [ " + chart + " ] with version [ " + version + " ] was found locally.") return version, "" } @@ -554,6 +561,15 @@ func (r *release) getHelmFlags() []string { return concat(r.getNoHooks(), r.getWait(), r.getTimeout(), r.getMaxHistory(), flags.getDryRunFlags(), []string{force}, flgs) } +// getPostRenderer returns the post-renderer Helm flag +func (r *release) getPostRenderer() []string { + result := []string{} + if r.PostRenderer != "" { + result = append(result, "--post-renderer", r.PostRenderer) + } + return result +} + // getHelmArgsFor returns helm arguments for a specific helm operation func (r *release) getHelmArgsFor(action string, optionalNamespaceOverride ...string) []string { ns := r.Namespace @@ -562,9 +578,9 @@ func (r *release) getHelmArgsFor(action string, optionalNamespaceOverride ...str } switch action { case "install", "upgrade": - return concat([]string{"upgrade", r.Name, r.Chart, "--install", "--version", r.Version, "--namespace", r.Namespace}, r.getValuesFiles(), r.getSetValues(), r.getSetStringValues(), r.getSetFileValues(), r.getHelmFlags()) + return concat([]string{"upgrade", r.Name, r.Chart, "--install", "--version", r.Version, "--namespace", r.Namespace}, r.getValuesFiles(), r.getSetValues(), r.getSetStringValues(), r.getSetFileValues(), r.getHelmFlags(), r.getPostRenderer()) case "diff": - return concat([]string{"upgrade", r.Name, r.Chart, "--version", r.Version, "--namespace", r.Namespace}, r.getValuesFiles(), r.getSetValues(), r.getSetStringValues(), r.getSetFileValues()) + return concat([]string{"upgrade", r.Name, r.Chart, "--version", r.Version, "--namespace", r.Namespace}, r.getValuesFiles(), r.getSetValues(), r.getSetStringValues(), r.getSetFileValues(), r.getPostRenderer()) case "uninstall": return concat([]string{action, "--namespace", ns, r.Name}, flags.getDryRunFlags()) default: @@ -694,29 +710,30 @@ func (r *release) shouldWaitForHook(hookFile string, hookType string, namespace // print prints the details of the release func (r release) print() { fmt.Println("") - fmt.Println("\tname : ", r.Name) - fmt.Println("\tdescription : ", r.Description) - fmt.Println("\tnamespace : ", r.Namespace) - fmt.Println("\tenabled : ", r.Enabled) - fmt.Println("\tchart : ", r.Chart) - fmt.Println("\tversion : ", r.Version) - fmt.Println("\tvaluesFile : ", r.ValuesFile) - fmt.Println("\tvaluesFiles : ", strings.Join(r.ValuesFiles, ",")) - fmt.Println("\ttest : ", r.Test) - fmt.Println("\tprotected : ", r.Protected) - fmt.Println("\twait : ", r.Wait) - fmt.Println("\tpriority : ", r.Priority) - fmt.Println("\tSuccessCondition : ", r.Hooks["successCondition"]) - fmt.Println("\tSuccessTimeout : ", r.Hooks["successTimeout"]) - fmt.Println("\tDeleteOnSuccess : ", r.Hooks["deleteOnSuccess"]) - fmt.Println("\tpreInstall : ", r.Hooks["preInstall"]) - fmt.Println("\tpostInstall : ", r.Hooks["postInstall"]) - fmt.Println("\tpreUpgrade : ", r.Hooks["preUpgrade"]) - fmt.Println("\tpostUpgrade : ", r.Hooks["postUpgrade"]) - fmt.Println("\tpreDelete : ", r.Hooks["preDelete"]) - fmt.Println("\tpostDelete : ", r.Hooks["postDelete"]) - fmt.Println("\tno-hooks : ", r.NoHooks) - fmt.Println("\ttimeout : ", r.Timeout) + fmt.Println("\tname: ", r.Name) + fmt.Println("\tdescription: ", r.Description) + fmt.Println("\tnamespace: ", r.Namespace) + fmt.Println("\tenabled: ", r.Enabled) + fmt.Println("\tchart: ", r.Chart) + fmt.Println("\tversion: ", r.Version) + fmt.Println("\tvaluesFile: ", r.ValuesFile) + fmt.Println("\tvaluesFiles: ", strings.Join(r.ValuesFiles, ",")) + fmt.Println("\tpostRenderer: ", r.PostRenderer) + fmt.Println("\ttest: ", r.Test) + fmt.Println("\tprotected: ", r.Protected) + fmt.Println("\twait: ", r.Wait) + fmt.Println("\tpriority: ", r.Priority) + fmt.Println("\tSuccessCondition: ", r.Hooks["successCondition"]) + fmt.Println("\tSuccessTimeout: ", r.Hooks["successTimeout"]) + fmt.Println("\tDeleteOnSuccess: ", r.Hooks["deleteOnSuccess"]) + fmt.Println("\tpreInstall: ", r.Hooks["preInstall"]) + fmt.Println("\tpostInstall: ", r.Hooks["postInstall"]) + fmt.Println("\tpreUpgrade: ", r.Hooks["preUpgrade"]) + fmt.Println("\tpostUpgrade: ", r.Hooks["postUpgrade"]) + fmt.Println("\tpreDelete: ", r.Hooks["preDelete"]) + fmt.Println("\tpostDelete: ", r.Hooks["postDelete"]) + fmt.Println("\tno-hooks: ", r.NoHooks) + fmt.Println("\ttimeout: ", r.Timeout) fmt.Println("\tvalues to override from env:") printMap(r.Set, 2) fmt.Println("------------------- ") diff --git a/internal/app/release_test.go b/internal/app/release_test.go index d47fddfb..454ac978 100644 --- a/internal/app/release_test.go +++ b/internal/app/release_test.go @@ -362,6 +362,40 @@ func Test_validateRelease(t *testing.T) { s: st, }, want: "PreDelete is an Invalid hook type.", + }, { + name: "test case 21", + args: args{ + r: &release{ + Name: "release21", + Description: "", + Namespace: "namespace", + Enabled: true, + Chart: "repo/chartX", + Version: "1.0", + ValuesFile: "../../tests/values.yaml", + PostRenderer: "../../tests/post-renderer.sh", + Test: true, + }, + s: st, + }, + want: "", + }, { + name: "test case 22", + args: args{ + r: &release{ + Name: "release22", + Description: "", + Namespace: "namespace", + Enabled: true, + Chart: "repo/chartX", + Version: "1.0", + ValuesFile: "../../tests/values.yaml", + PostRenderer: "doesnt-exist.sh", + Test: true, + }, + s: st, + }, + want: "doesnt-exist.sh must be valid relative (from dsf file) file path.", }, } names := make(map[string]map[string]bool) @@ -462,6 +496,7 @@ func createFullReleasePointer(chart, version string) *release { HelmFlags: []string{}, NoHooks: false, Timeout: 0, + PostRenderer: "", } } diff --git a/tests/overlay.sample.yaml b/tests/overlay.sample.yaml new file mode 100644 index 00000000..6f763836 --- /dev/null +++ b/tests/overlay.sample.yaml @@ -0,0 +1,13 @@ +#@ load("@ytt:overlay", "overlay") +#@overlay/remove +#@overlay/match by=overlay.subset({"kind":"ServiceAccount"}),expects="1+" +--- +#@overlay/merge +#@overlay/match by=overlay.subset({"kind":"Deployment"}),expects="1+" +--- +apiVersion: apps/v1 +kind: Deployment +spec: + template: + spec: + serviceAccountName: default diff --git a/tests/post-renderer.sh b/tests/post-renderer.sh new file mode 100755 index 00000000..c2f06558 --- /dev/null +++ b/tests/post-renderer.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +ytt --ignore-unknown-comments -f - -f $(dirname $0)/overlay.sample.yaml