feat: add git chart downloader

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.15 AS khelm
+FROM alpine:3.16 AS khelm
 RUN apk update --no-cache
 RUN mkdir /helm && chown root:nobody /helm && chmod 1777 /helm
 ENV HELM_REPOSITORY_CONFIG=/helm/repository/repositories.yaml

README.md

@@ -24,6 +24,7 @@ Since [kpt](https://github.com/GoogleContainerTools/kpt) is [published](https://
 * Automatically fetches and updates required repository index files when needed
 * Allows to automatically reload dependencies when lock file is out of sync
 * Allows to use any repository without registering it in repositories.yaml
+* Allows to use a Helm chart from a remote git repository
 * Allows to exclude certain resources from the Helm chart output
 * Allows to enforce namespace-scoped resources within the template output
 * Allows to enforce a namespace on all resources
@@ -200,10 +201,17 @@ khelm template cert-manager --version=0.9.x --repo=https://charts.jetstack.io
 _For all available options see the [table](#configuration-options) below._
 
 #### Docker usage example
+
+Generate a manifest from a chart:
 ```sh
 docker run mgoltzsche/khelm:latest template cert-manager --version=0.9.x --repo=https://charts.jetstack.io
 ```
 
+Generate a manifest from a chart within a git repository:
+```sh
+docker run mgoltzsche/khelm:latest template cert-manager --repo=git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2
+```
+
 ### Go API
 
 The khelm Go API `github.com/mgoltzsche/khelm/v2/pkg/helm` provides a simple templating interface on top of the Helm Go API.
@@ -245,8 +253,9 @@ It exposes a `Helm` struct that provides a `Render()` function that returns the
 | `outputPathMapping[].selectors[].kind` |  | Selects resources by kind. |
 | `outputPathMapping[].selectors[].namespace` |  | Selects resources by namespace. |
 | `outputPathMapping[].selectors[].name` |  | Selects resources by name. |
-|  | `--output-replace` | If enabled replace the output directory or file (CLI-only). |
-|  | `--trust-any-repo` | If enabled repositories that are not registered within `repositories.yaml` can be used as well (env var `KHELM_TRUST_ANY_REPO`). Within the kpt function this behaviour can be disabled by mounting `/helm/repository/repositories.yaml` or disabling network access. |
+|  | `--output-replace` | If enabled, replace the output directory or file (CLI-only). |
+|  | `--trust-any-repo` | If enabled, repositories that are not registered within `repositories.yaml` can be used as well (env var `KHELM_TRUST_ANY_REPO`). Within the kpt function this behaviour can be disabled by mounting `/helm/repository/repositories.yaml` or disabling network access. |
+|  | `--enable-git-getter` | If enabled, support helm repository URLs with the git+https scheme (env var `KHELM_ENABLE_GIT_GETTER`). |
 | `debug` | `--debug` | Enables debug log and provides a stack trace on error. |
 
 ### Repository configuration
@@ -257,6 +266,22 @@ When running khelm as kpt function or within a container the `repositories.yaml`
 
 Unlike Helm khelm allows usage of any repository when `repositories.yaml` is not present or `--trust-any-repo` (env var `KHELM_TRUST_ANY_REPO`) is enabled.
 
+#### Git URLs as Helm repositories
+
+Helm charts can be fetched from git repositories by letting the Helm repository URL point to the chart's parent directory using the URL scheme `git+https` or `git+ssh`.
+The path within the git repository URL and the repository part of the URL must be separated by `@`.
+The `ref` parameter can be used to specify the git tag.  
+
+The following example points to an old version of cert-manager using a git URL:
+```
+git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2
+```
+
+To enable this feature, set the `--enable-git-getter` option or the corresponding environment variable: `KHELM_ENABLE_GIT_GETTER=true`.
+
+This feature is meant to be compatible with Helm's [helm-git](https://github.com/aslafy-z/helm-git#usage) plugin (but is reimplemented in Go).
+However currently khelm does not support `sparse` git checkouts (due to [lack of support in go-git](https://github.com/go-git/go-git/issues/90)).
+
 ## Helm support
 
 * Helm 2 is supported by the `v1` module version.

cmd/khelm/root.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"log"
@@ -10,13 +11,16 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mgoltzsche/khelm/v2/pkg/getter/git"
 	"github.com/mgoltzsche/khelm/v2/pkg/helm"
 	"github.com/spf13/cobra"
+	helmgetter "helm.sh/helm/v3/pkg/getter"
 )
 
 const (
 	envKustomizePluginConfig     = "KUSTOMIZE_PLUGIN_CONFIG_STRING"
 	envKustomizePluginConfigRoot = "KUSTOMIZE_PLUGIN_CONFIG_ROOT"
+	envEnableGitGetter           = "KHELM_ENABLE_GIT_GETTER"
 	envTrustAnyRepo              = "KHELM_TRUST_ANY_REPO"
 	envDebug                     = "KHELM_DEBUG"
 	envHelmDebug                 = "HELM_DEBUG"
@@ -36,11 +40,15 @@ func Execute(reader io.Reader, writer io.Writer) error {
 	helmDebug, _ := strconv.ParseBool(os.Getenv(envHelmDebug))
 	h := helm.NewHelm()
 	debug = debug || helmDebug
+	enableGitGetter := false
 	h.Settings.Debug = debug
 	if trustAnyRepo, ok := os.LookupEnv(envTrustAnyRepo); ok {
 		trust, _ := strconv.ParseBool(trustAnyRepo)
 		h.TrustAnyRepository = &trust
 	}
+	if gitSupportStr, ok := os.LookupEnv(envEnableGitGetter); ok {
+		enableGitGetter, _ = strconv.ParseBool(gitSupportStr)
+	}
 
 	// Run as kustomize plugin (if kustomize-specific env var provided)
 	if kustomizeGenCfgYAML, isKustomizePlugin := os.LookupEnv(envKustomizePluginConfig); isKustomizePlugin {
@@ -50,12 +58,16 @@ func Execute(reader io.Reader, writer io.Writer) error {
 		return err
 	}
 
-	logVersionPreRun := func(_ *cobra.Command, _ []string) {
+	preRun := func(_ *cobra.Command, _ []string) {
 		logVersion()
+		if enableGitGetter {
+			addGitGetterSupport(h)
+		}
 	}
 	rootCmd := &cobra.Command{
-		PreRun: logVersionPreRun,
+		PersistentPreRun: preRun,
 	}
+	rootCmd.PersistentFlags().BoolVar(&enableGitGetter, "enable-git-getter", enableGitGetter, fmt.Sprintf("enable git+https helm repository URL scheme support (%s)", envEnableGitGetter))
 	errBuf := bytes.Buffer{}
 
 	if filepath.Base(os.Args[0]) == "khelmfn" {
@@ -65,8 +77,7 @@ func Execute(reader io.Reader, writer io.Writer) error {
 		rootCmd.SetOut(writer)
 		rootCmd.SetErr(&errBuf)
 		rootCmd.PersistentFlags().BoolVar(&debug, "debug", debug, fmt.Sprintf("enable debug log (%s)", envDebug))
-		rootCmd.PreRun = func(_ *cobra.Command, _ []string) {
-			logVersion()
+		rootCmd.PreRun = func(cmd *cobra.Command, args []string) {
 			fmt.Printf("# Reading kpt function input from stdin (use `%s template` to run without kpt)\n", os.Args[0])
 		}
 	}
@@ -88,7 +99,7 @@ In addition to helm's templating capabilities khelm allows to:
 	templateCmd := templateCommand(h, writer)
 	templateCmd.SetOut(writer)
 	templateCmd.SetErr(&errBuf)
-	templateCmd.PreRun = logVersionPreRun
+	templateCmd.PreRun = preRun
 	rootCmd.AddCommand(templateCmd)
 
 	// Run command
@@ -110,3 +121,12 @@ func logVersion() {
 func versionInfo() string {
 	return fmt.Sprintf("%s (helm %s)", khelmVersion, helmVersion)
 }
+
+func addGitGetterSupport(h *helm.Helm) {
+	h.Getters = append(h.Getters, helmgetter.Provider{
+		Schemes: git.Schemes,
+		New: git.New(&h.Settings, h.Repositories, func(ctx context.Context, chartDir, repoDir string) (string, error) {
+			return h.Package(ctx, chartDir, repoDir, chartDir)
+		}),
+	})
+}

cmd/khelm/template_test.go

@@ -93,6 +93,16 @@ func TestTemplateCommand(t *testing.T) {
 			[]string{filepath.Join(exampleDir, "chart-hooks"), "--no-hooks"},
 			1, "myvalue",
 		},
+		{
+			"git-dependency",
+			[]string{filepath.Join(exampleDir, "git-dependency"), "--enable-git-getter", "--trust-any-repo"},
+			24, "ca-sync",
+		},
+		{
+			"local-chart-with-transitive-remote-and-git-dependencies",
+			[]string{filepath.Join(exampleDir, "localrefref-with-git"), "--enable-git-getter", "--trust-any-repo"},
+			33, "admission.certmanager.k8s.io",
+		},
 	} {
 		t.Run(c.name, func(t *testing.T) {
 			var out bytes.Buffer
@@ -128,6 +138,10 @@ func TestTemplateCommandError(t *testing.T) {
 			"reject cluster scoped resources",
 			[]string{"cert-manager", "--repo=https://charts.jetstack.io", "--namespaced-only"},
 		},
+		{
+			"reject git urls by default",
+			[]string{"git-dependency", "--enable-git-getter=false"},
+		},
 	} {
 		t.Run(c.name, func(t *testing.T) {
 			os.Args = append([]string{"testee", "template"}, c.args...)

e2e/cli-tests.bats

@@ -1,6 +1,9 @@
 #!/usr/bin/env bats
 
+bats_require_minimum_version 1.5.0
+
 : ${IMAGE:=mgoltzsche/khelm:latest}
+
 EXAMPLE_DIR="$(pwd)/example"
 OUT_DIR="$(mktemp -d)"
 
@@ -20,6 +23,24 @@ teardown() {
 	grep -q myrelease "$OUT_DIR/subdir/manifest.yaml"
 }
 
+@test "CLI should reject repository when not in repositories.yaml and trust-any disabled" {
+	run -1 docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" -e KHELM_TRUST_ANY_REPO=false "$IMAGE" template cert-manager \
+		--name=myrelease \
+		--version 1.0.4 \
+		--repo https://charts.jetstack.io \
+		--output /out/subdir/manifest.yaml \
+		--debug
+}
+
+@test "CLI should build local chart" {
+	docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" -v "$EXAMPLE_DIR/release-name:/chart" "$IMAGE" template /chart \
+		--version 1.2.3 \
+		--output /out/manifest.yaml \
+		--debug
+	ls -la "$OUT_DIR" "$OUT_DIR/manifest.yaml" >&2
+	cat "$OUT_DIR/manifest.yaml" | tee /dev/stdout /dev/stderr | grep -q 'chartVersion: 1.2.3'
+}
+
 @test "CLI should output kustomization" {
 	docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" -v "$EXAMPLE_DIR/namespace:/chart" "$IMAGE" template /chart \
 		--output /out/kdir/ \
@@ -37,4 +58,46 @@ teardown() {
 		--debug
 	[ -f "$OUT_DIR/manifest.yaml" ]
 	grep -q myreleasex "$OUT_DIR/manifest.yaml"
-}
+}
+
+@test "CLI should accept git url as helm repository" {
+	docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" \
+		-e KHELM_ENABLE_GIT_GETTER=true \
+		"$IMAGE" template cert-manager \
+		--repo git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2 \
+		--output /out/manifest.yaml \
+		--debug
+	[ -f "$OUT_DIR/manifest.yaml" ]
+	grep -q ca-sync "$OUT_DIR/manifest.yaml"
+}
+
+@test "CLI should cache git repository" {
+	mkdir $OUT_DIR/cache
+	docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" -v "$OUT_DIR/cache:/helm/cache" \
+		-e KHELM_ENABLE_GIT_GETTER=true \
+		"$IMAGE" template cert-manager \
+		--repo git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2 \
+		--output /out/manifest.yaml \
+		--debug
+	[ -f "$OUT_DIR/manifest.yaml" ]
+	grep -q ca-sync "$OUT_DIR/manifest.yaml"
+	rm -f "$OUT_DIR/manifest.yaml"
+	docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" -v "$OUT_DIR/cache:/helm/cache" \
+		-e KHELM_ENABLE_GIT_GETTER=true \
+		--network=none "$IMAGE" template cert-manager \
+		--repo git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2 \
+		--output /out/manifest.yaml \
+		--debug
+	[ -f "$OUT_DIR/manifest.yaml" ]
+	grep -q ca-sync "$OUT_DIR/manifest.yaml"
+}
+
+@test "CLI should reject git repository when not in repositories.yaml and trust-any disabled" {
+	run -1 docker run --rm -u $(id -u):$(id -g) -v "$OUT_DIR:/out" \
+		-e KHELM_ENABLE_GIT_GETTER=true \
+		-e KHELM_TRUST_ANY_REPO=false \
+		"$IMAGE" template cert-manager \
+		--repo git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2 \
+		--output /out/manifest.yaml \
+		--debug
+}

example/git-dependency/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: cert-manager
+  repository: git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2
+  version: v0.6.6
+digest: sha256:cd75b404696beff523a4297ba055389e9dd9e8214f47c668cfbf3f200ea41191
+generated: "2022-10-14T00:58:00.186187675+02:00"

example/git-dependency/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+description: example chart using a git url as dependency
+name: git-dependency
+version: 0.1.0
+dependencies:
+- name: cert-manager
+  version: "x.x.x"
+  repository: "git+https://github.com/cert-manager/cert-manager@deploy/charts?ref=v0.6.2"

example/git-dependency/generator.yaml

@@ -0,0 +1,6 @@
+apiVersion: khelm.mgoltzsche.github.com/v2
+kind: ChartRenderer
+metadata:
+  name: cert-manager
+  namespace: cert-manager
+chart: .

example/git-dependency/kustomization.yaml

@@ -0,0 +1,2 @@
+generators:
+- generator.yaml

example/localrefref-with-git/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: intermediate-chart
+  repository: file://../localref/intermediate-chart
+  version: 0.1.1
+- name: git-https-dependency
+  repository: git+https://github.com/mgoltzsche/khelm@example?ref=6d1374f17decb68f7876ae1f592d5bd04c74fee5
+  version: 0.1.0
+digest: sha256:020a51536f70f9fa7c29b0c1573ea40abbde749c89148a35720efddf7709a003
+generated: "2022-10-30T20:22:52.241871102+01:00"

example/localrefref-with-git/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+description: Chart that refers another one that has dependencies itself
+name: myumbrella-chart
+version: 0.2.0
+dependencies:
+- name: intermediate-chart
+  version: "~0.1.0"
+  repository: "file://../localref/intermediate-chart"
+- name: git-https-dependency
+  version: "x.x.x"
+  repository: "git+https://github.com/mgoltzsche/khelm@example?ref=6d1374f17decb68f7876ae1f592d5bd04c74fee5"

example/localrefref-with-git/generator.yaml

@@ -0,0 +1,6 @@
+apiVersion: khelm.mgoltzsche.github.com/v2
+kind: ChartRenderer
+metadata:
+  name: mychart
+  namespace: myotherns
+chart: .

example/release-name/generator.yaml

@@ -4,3 +4,4 @@ metadata:
   name: my-release-name
 chart: .
 kubeVersion: 1.17
+version: 1.9.3

example/release-name/templates/example.yaml

@@ -5,3 +5,4 @@ metadata:
 data:
   key: b
   k8sVersion: {{ .Capabilities.KubeVersion }}
+  chartVersion: {{ .Chart.Version }}