From ff92b7a869462bee1d0e35f845b52301b7145f5e Mon Sep 17 00:00:00 2001 From: Max Goltzsche Date: Sun, 16 Oct 2022 05:18:08 +0200 Subject: [PATCH] feat: support auth for git+https repos Use the credentials configured within repositories.yaml also for git+https urls. To avoid reloading the config, move repository-related code into a separate package. --- cmd/khelm/common.go | 3 +- example/git-getter/Chart.yaml | 2 +- example/localrefref-with-git/Chart.lock | 9 + example/localrefref-with-git/Chart.yaml | 13 + example/localrefref-with-git/generator.yaml | 6 + pkg/getter/git/checkout.go | 103 ++--- pkg/getter/git/gitgetter.go | 28 +- pkg/getter/git/gitgetter_test.go | 10 +- pkg/helm/helm.go | 16 +- pkg/helm/load.go | 41 +- pkg/helm/locate.go | 2 +- pkg/helm/package.go | 5 +- pkg/helm/providers.go | 11 +- pkg/helm/render_test.go | 46 ++- pkg/helm/repositories.go | 414 +++----------------- pkg/repositories/repositories.go | 276 +++++++++++++ pkg/repositories/repositories_test.go | 258 ++++++++++++ 17 files changed, 796 insertions(+), 447 deletions(-) create mode 100644 example/localrefref-with-git/Chart.lock create mode 100644 example/localrefref-with-git/Chart.yaml create mode 100644 example/localrefref-with-git/generator.yaml create mode 100644 pkg/repositories/repositories.go create mode 100644 pkg/repositories/repositories_test.go diff --git a/cmd/khelm/common.go b/cmd/khelm/common.go index 2ab02a2..6bda03f 100644 --- a/cmd/khelm/common.go +++ b/cmd/khelm/common.go @@ -9,6 +9,7 @@ import ( "github.com/mgoltzsche/khelm/v2/pkg/config" "github.com/mgoltzsche/khelm/v2/pkg/helm" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -23,7 +24,7 @@ func render(h *helm.Helm, req *config.ChartConfig) ([]*yaml.RNode, error) { }() rendered, err := h.Render(ctx, req) - if helm.IsUntrustedRepository(err) { + if repositories.IsUntrustedRepository(err) { log.Printf("HINT: access to untrusted repositories can be enabled using env var %s=true or option --%s", envTrustAnyRepo, flagTrustAnyRepo) } return rendered, err diff --git a/example/git-getter/Chart.yaml b/example/git-getter/Chart.yaml index 660fb6d..7cd2f51 100644 --- a/example/git-getter/Chart.yaml +++ b/example/git-getter/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 description: example chart using a git url as dependency -name: git-getter-example-chart +name: git-getter version: 0.1.0 dependencies: - name: cert-manager diff --git a/example/localrefref-with-git/Chart.lock b/example/localrefref-with-git/Chart.lock new file mode 100644 index 0000000..ecc07d0 --- /dev/null +++ b/example/localrefref-with-git/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: intermediate-chart + repository: file://../localref/intermediate-chart + version: 0.1.1 +- name: git-getter + repository: git+https://github.com/mgoltzsche/khelm@example?ref=afdbaa8e7068b7d7dd71fc35dcb022fc1f12ea62 + version: 0.1.0 +digest: sha256:83407caeeb8ff5cf80c07d046efe3ee2961dda5d9dce64ee95d162a124263dfd +generated: "2022-10-17T01:20:17.281584028Z" diff --git a/example/localrefref-with-git/Chart.yaml b/example/localrefref-with-git/Chart.yaml new file mode 100644 index 0000000..794db3e --- /dev/null +++ b/example/localrefref-with-git/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +description: Chart that refers another one that has dependencies itself +name: myumbrella-chart +version: 0.2.0 +dependencies: +- name: intermediate-chart + version: "~0.1.0" + repository: "file://../localref/intermediate-chart" +- name: "git-getter" + version: "x.x.x" + repository: "git+https://github.com/mgoltzsche/khelm@example?ref=afdbaa8e7068b7d7dd71fc35dcb022fc1f12ea62" +#afdbaa8e7068b7d7dd71fc35dcb022fc1f12ea62 +#3857e76b956c4652d3cacc2fd71e7dcff5843302 diff --git a/example/localrefref-with-git/generator.yaml b/example/localrefref-with-git/generator.yaml new file mode 100644 index 0000000..b0e3b6f --- /dev/null +++ b/example/localrefref-with-git/generator.yaml @@ -0,0 +1,6 @@ +apiVersion: khelm.mgoltzsche.github.com/v2 +kind: ChartRenderer +metadata: + name: mychart + namespace: myotherns +chart: . diff --git a/pkg/getter/git/checkout.go b/pkg/getter/git/checkout.go index 59e78b0..d971234 100644 --- a/pkg/getter/git/checkout.go +++ b/pkg/getter/git/checkout.go @@ -2,78 +2,83 @@ package git import ( "context" + "encoding/hex" "fmt" + "log" + "strings" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "helm.sh/helm/v3/pkg/repo" ) -func gitCheckoutImpl(ctx context.Context, repoURL, ref, destDir string) error { - /*err := runCmds(destDir, [][]string{ - {"git", "init", "--quiet"}, - {"git", "remote", "add", "origin", repoURL}, - - {"git", "config", "core.sparseCheckout", "true"}, - {"git", "sparse-checkout", "set", path}, - {"git", "pull", "--quiet", "--depth", "1", "origin", ref}, - - //{"git", "fetch", "--quiet", "--tags", "origin"}, - //{"git", "checkout", "--quiet", ref}, - })*/ - r, err := git.PlainCloneContext(ctx, destDir, false, &git.CloneOptions{ - URL: repoURL, - // TODO: Auth: ... - RemoteName: "origin", - SingleBranch: true, - Depth: 1, - NoCheckout: true, - }) +func gitCheckoutImpl(ctx context.Context, repoURL, ref string, repo *repo.Entry, destDir string) error { + cloneOpts := git.CloneOptions{ + URL: repoURL, + RemoteName: "origin", + NoCheckout: true, + } + isCommitRef := isCommitSHA(ref) + if !isCommitRef { // cannot find commit in other branch when checking out single branch + cloneOpts.SingleBranch = true + cloneOpts.Depth = 1 + } + scheme := strings.SplitN(repoURL, ":", 2)[0] + switch scheme { + case "https": + cloneOpts.Auth = &http.BasicAuth{ + Username: repo.Username, + Password: repo.Password, + } + default: + if repo.Username != "" || repo.Password != "" { + log.Printf("WARNING: ignoring auth config for %s since authentication is not supported for url scheme %q", repoURL, scheme) + } + } + r, err := git.PlainCloneContext(ctx, destDir, false, &cloneOpts) if err != nil { - return err + return fmt.Errorf("git clone: %w", err) } tree, err := r.Worktree() if err != nil { return fmt.Errorf("git worktree: %w", err) } // TODO: support sparse checkout, see https://github.com/go-git/go-git/issues/90 + refType := "without ref" opts := git.CheckoutOptions{} if ref != "" { - opts.Branch = plumbing.ReferenceName("refs/tags/" + ref) + if isCommitRef { + opts.Hash = plumbing.NewHash(ref) + refType = fmt.Sprintf("commit %s", ref) + } else { + opts.Branch = plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", ref)) + refType = fmt.Sprintf("tag %s", ref) + } } err = tree.Checkout(&opts) if err != nil { - return err + return fmt.Errorf("git checkout %s: %w", refType, err) } - return nil -} + /*err := runCmds(destDir, [][]string{ + {"git", "init", "--quiet"}, + {"git", "remote", "add", "origin", repoURL}, -/* -func runCmds(dir string, cmds [][]string) error { - for _, c := range cmds { - err := runCmd(dir, c[0], c[1:]...) - if err != nil { - return err - } - } + {"git", "config", "core.sparseCheckout", "true"}, + {"git", "sparse-checkout", "set", path}, + {"git", "pull", "--quiet", "--depth", "1", "origin", ref}, + + //{"git", "fetch", "--quiet", "--tags", "origin"}, + //{"git", "checkout", "--quiet", ref}, + })*/ return nil } -func runCmd(dir, cmd string, args ...string) error { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - c := exec.CommandContext(ctx, cmd, args...) - var stderr bytes.Buffer - c.Stderr = &stderr - c.Dir = dir - err := c.Run() - if err != nil { - msg := strings.TrimSpace(stderr.String()) - if msg == "" { - msg = err.Error() +func isCommitSHA(s string) bool { + if len(s) == 40 { + if _, err := hex.DecodeString(s); err == nil { + return true } - cmds := append([]string{cmd}, args...) - return fmt.Errorf("%s: %s", strings.Join(cmds, " "), msg) } - return err + return false } -*/ diff --git a/pkg/getter/git/gitgetter.go b/pkg/getter/git/gitgetter.go index 83e9d4f..4f3029b 100644 --- a/pkg/getter/git/gitgetter.go +++ b/pkg/getter/git/gitgetter.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strings" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/cli" helmgetter "helm.sh/helm/v3/pkg/getter" @@ -23,16 +24,24 @@ var gitCheckout = gitCheckoutImpl type HelmPackageFunc func(ctx context.Context, path, repoDir string) (string, error) -func New(settings *cli.EnvSettings, packageFn HelmPackageFunc) helmgetter.Constructor { +type RepositoriesFunc func() (repositories.Interface, error) + +func New(settings *cli.EnvSettings, reposFn RepositoriesFunc, packageFn HelmPackageFunc) helmgetter.Constructor { return func(o ...helmgetter.Option) (helmgetter.Getter, error) { + repos, err := reposFn() + if err != nil { + return nil, err + } return &gitIndexGetter{ settings: settings, + repos: repos, packageFn: packageFn, }, nil } } type gitIndexGetter struct { + repos repositories.Interface settings *cli.EnvSettings Getters helmgetter.Providers packageFn HelmPackageFunc @@ -52,7 +61,7 @@ func (g *gitIndexGetter) Get(location string, options ...helmgetter.Option) (*by if isRepoIndex { // Generate repo index from directory ref = ref.Dir() - repoDir, err := download(ctx, ref, g.settings.RepositoryCache) + repoDir, err := download(ctx, ref, g.settings.RepositoryCache, g.repos) if err != nil { return nil, err } @@ -67,8 +76,9 @@ func (g *gitIndexGetter) Get(location string, options ...helmgetter.Option) (*by } } else { // Build and package chart - chartPath := filepath.FromSlash(strings.TrimSuffix(ref.Path, ".tgz")) - repoDir, err := download(ctx, ref, g.settings.RepositoryCache) + ref.Path = strings.TrimSuffix(ref.Path, ".tgz") + chartPath := filepath.FromSlash(ref.Path) + repoDir, err := download(ctx, ref, g.settings.RepositoryCache, g.repos) if err != nil { return nil, err } @@ -136,7 +146,7 @@ func generateRepoIndex(dir, cacheDir string, u *gitURL) (*repo.IndexFile, error) return idx, nil } -func download(ctx context.Context, ref *gitURL, cacheDir string) (string, error) { +func download(ctx context.Context, ref *gitURL, cacheDir string, repos repositories.Interface) (string, error) { repoRef := *ref repoRef.Path = "" cacheKey := fmt.Sprintf("sha256-%x", sha256.Sum256([]byte(repoRef.String()))) @@ -144,7 +154,11 @@ func download(ctx context.Context, ref *gitURL, cacheDir string) (string, error) destDir := filepath.Join(cacheDir, cacheKey) if _, e := os.Stat(destDir); os.IsNotExist(e) { - err := os.MkdirAll(cacheDir, 0755) + auth, _, err := repos.Get("git+" + ref.String()) + if err != nil { + return "", err + } + err = os.MkdirAll(cacheDir, 0755) if err != nil { return "", err } @@ -155,7 +169,7 @@ func download(ctx context.Context, ref *gitURL, cacheDir string) (string, error) defer os.RemoveAll(tmpDir) tmpRepoDir := tmpDir - err = gitCheckout(ctx, ref.Repo, ref.Ref, tmpRepoDir) + err = gitCheckout(ctx, ref.Repo, ref.Ref, auth, tmpRepoDir) if err != nil { return "", err } diff --git a/pkg/getter/git/gitgetter_test.go b/pkg/getter/git/gitgetter_test.go index 0074b95..ecbba2c 100644 --- a/pkg/getter/git/gitgetter_test.go +++ b/pkg/getter/git/gitgetter_test.go @@ -7,9 +7,11 @@ import ( "strings" "testing" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "github.com/stretchr/testify/require" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" "sigs.k8s.io/yaml" ) @@ -18,7 +20,7 @@ func TestGitGetter(t *testing.T) { tmpDir, err := os.MkdirTemp("", "khelm-git-getter-test-") require.NoError(t, err) defer os.RemoveAll(tmpDir) - gitCheckout = func(_ context.Context, repoURL, ref, destDir string) error { + gitCheckout = func(_ context.Context, repoURL, ref string, auth *repo.Entry, destDir string) error { err := os.MkdirAll(filepath.Join(destDir, "mypath", "fakechart"), 0755) require.NoError(t, err) err = os.WriteFile(filepath.Join(destDir, "mypath", "fakechart", "Chart.yaml"), []byte(` @@ -38,7 +40,11 @@ version: 0.1.0`), 0600) require.NoError(t, err) return file, nil } - testee := New(settings, fakePackageFn) + reposFn := func() (repositories.Interface, error) { + trust := true + return repositories.New(*settings, getter.All(settings), &trust) + } + testee := New(settings, reposFn, fakePackageFn) getter, err := testee() require.NoError(t, err) diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 56a4901..df90cf5 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/helmpath" @@ -14,6 +15,7 @@ type Helm struct { TrustAnyRepository *bool Settings cli.EnvSettings Getters getter.Providers + repos repositories.Interface } // NewHelm creates a new helm environment @@ -24,6 +26,18 @@ func NewHelm() *Helm { settings.RepositoryConfig = filepath.Join(helmHome, "repository", "repositories.yaml") } h := &Helm{Settings: *settings} - h.Getters = getters(settings, &h.TrustAnyRepository) + h.Getters = getters(settings, h.repositories) return h } + +func (h *Helm) repositories() (repositories.Interface, error) { + if h.repos != nil { + return h.repos, nil + } + repos, err := repositories.New(h.Settings, h.Getters, h.TrustAnyRepository) + if err != nil { + return nil, err + } + h.repos = repos + return repos, nil +} diff --git a/pkg/helm/load.go b/pkg/helm/load.go index 2cffe9e..f7880bb 100644 --- a/pkg/helm/load.go +++ b/pkg/helm/load.go @@ -10,6 +10,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/mgoltzsche/khelm/v2/pkg/config" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "github.com/pkg/errors" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -28,7 +29,11 @@ func (h *Helm) loadChart(ctx context.Context, cfg *config.ChartConfig) (*chart.C fileExists := err == nil if cfg.Repository == "" { if fileExists { - return buildAndLoadLocalChart(ctx, cfg, h.TrustAnyRepository, h.Settings, h.Getters) + repos, err := h.repositories() + if err != nil { + return nil, err + } + return buildAndLoadLocalChart(ctx, cfg, repos, h.Settings, h.Getters) } else if l := strings.Split(cfg.Chart, "/"); len(l) == 2 && l[0] != "" && l[1] != "" && l[0] != ".." && l[0] != "." { cfg.Repository = "@" + l[0] cfg.Chart = l[1] @@ -41,7 +46,11 @@ func (h *Helm) loadChart(ctx context.Context, cfg *config.ChartConfig) (*chart.C func (h *Helm) loadRemoteChart(ctx context.Context, cfg *config.ChartConfig) (*chart.Chart, error) { repoURLs := map[string]struct{}{cfg.Repository: {}} - repos, err := reposForURLs(repoURLs, h.TrustAnyRepository, &h.Settings, h.Getters) + repos, err := h.repositories() + if err != nil { + return nil, err + } + reqRepos, err := reposForURLs(repoURLs, repos) if err != nil { return nil, err } @@ -50,18 +59,19 @@ func (h *Helm) loadRemoteChart(ctx context.Context, cfg *config.ChartConfig) (*c return nil, err } if isRange { - if err = repos.UpdateIndex(ctx); err != nil { + // TODO: avoid updating the same repo index multiple times when using multiple charts from the same repo + if err = reqRepos.UpdateIndex(ctx); err != nil { return nil, err } } - chartPath, err := locateChart(ctx, &cfg.LoaderConfig, repos, &h.Settings, h.Getters) + chartPath, err := locateChart(ctx, &cfg.LoaderConfig, reqRepos, &h.Settings, h.Getters) if err != nil { return nil, err } return loader.Load(chartPath) } -func buildAndLoadLocalChart(ctx context.Context, cfg *config.ChartConfig, trustAnyRepo *bool, settings cli.EnvSettings, getters getter.Providers) (*chart.Chart, error) { +func buildAndLoadLocalChart(ctx context.Context, cfg *config.ChartConfig, repos repositories.Interface, settings cli.EnvSettings, getters getter.Providers) (*chart.Chart, error) { chartPath := absPath(cfg.Chart, cfg.BaseDir) chartRequested, err := loader.Load(chartPath) if err != nil { @@ -76,31 +86,32 @@ func buildAndLoadLocalChart(ctx context.Context, cfg *config.ChartConfig, trustA } // Create (temporary) repository configuration that includes all dependencies - repos, err := reposForDependencies(dependencies, trustAnyRepo, &settings, getters) + depRepos, err := reposForDependencies(dependencies, repos) if err != nil { return nil, errors.Wrap(err, "init temp repositories.yaml") } - repos.RequireTempHelmHome(len(localCharts) > 1) - repos, err = repos.Apply() - if err != nil { - return nil, err + if len(localCharts) > 1 { + depRepos, err = depRepos.TempRepositories() + if err != nil { + return nil, err + } } - defer repos.Close() + defer depRepos.Close() tmpSettings := settings - tmpSettings.RepositoryConfig = repos.FilePath() + tmpSettings.RepositoryConfig = depRepos.File() // Download/update repo indices if needsRepoIndexUpdate { - err = repos.UpdateIndex(ctx) + err = depRepos.UpdateIndex(ctx) } else { - err = repos.DownloadIndexFilesIfNotExist(ctx) + err = depRepos.FetchMissingIndexFiles(ctx) } if err != nil { return nil, err } // Build local charts recursively - needsReload, err := buildLocalCharts(ctx, localCharts, &cfg.LoaderConfig, repos, &tmpSettings, getters) + needsReload, err := buildLocalCharts(ctx, localCharts, &cfg.LoaderConfig, depRepos, &tmpSettings, getters) if err != nil { return nil, errors.Wrap(err, "build/fetch dependencies") } diff --git a/pkg/helm/locate.go b/pkg/helm/locate.go index 2beaad3..0033f18 100644 --- a/pkg/helm/locate.go +++ b/pkg/helm/locate.go @@ -27,7 +27,7 @@ func locateChart(ctx context.Context, cfg *config.LoaderConfig, repos repository return name, errors.Errorf("path %q not found", name) } - repoEntry, err := repos.Get(cfg.Repository) + repoEntry, _, err := repos.Get(cfg.Repository) if err != nil { return "", err } diff --git a/pkg/helm/package.go b/pkg/helm/package.go index 32596e0..485b62b 100644 --- a/pkg/helm/package.go +++ b/pkg/helm/package.go @@ -4,14 +4,15 @@ import ( "context" "github.com/mgoltzsche/khelm/v2/pkg/config" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" ) -func packageHelmChart(ctx context.Context, cfg *config.ChartConfig, destDir string, trustAnyRepo *bool, settings cli.EnvSettings, getters getter.Providers) (string, error) { +func packageHelmChart(ctx context.Context, cfg *config.ChartConfig, destDir string, repos repositories.Interface, settings cli.EnvSettings, getters getter.Providers) (string, error) { // TODO: add unit test (there is an e2e/cli test for this though) - _, err := buildAndLoadLocalChart(ctx, cfg, trustAnyRepo, settings, getters) + _, err := buildAndLoadLocalChart(ctx, cfg, repos, settings, getters) if err != nil { return "", err } diff --git a/pkg/helm/providers.go b/pkg/helm/providers.go index 64757b6..49a157c 100644 --- a/pkg/helm/providers.go +++ b/pkg/helm/providers.go @@ -5,21 +5,26 @@ import ( "github.com/mgoltzsche/khelm/v2/pkg/config" "github.com/mgoltzsche/khelm/v2/pkg/getter/git" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "helm.sh/helm/v3/pkg/cli" helmgetter "helm.sh/helm/v3/pkg/getter" ) -func getters(settings *cli.EnvSettings, trustAnyRepo **bool) helmgetter.Providers { +func getters(settings *cli.EnvSettings, reposFn func() (repositories.Interface, error)) helmgetter.Providers { g := helmgetter.All(settings) g = append(g, helmgetter.Provider{ Schemes: []string{"git+https", "git+ssh"}, - New: git.New(settings, func(ctx context.Context, chartDir, repoDir string) (string, error) { + New: git.New(settings, reposFn, func(ctx context.Context, chartDir, repoDir string) (string, error) { + repos, err := reposFn() + if err != nil { + return "", err + } return packageHelmChart(ctx, &config.ChartConfig{ LoaderConfig: config.LoaderConfig{ Chart: chartDir, }, BaseDir: repoDir, - }, chartDir, *trustAnyRepo, *settings, g) + }, chartDir, repos, *settings, g) }), }) return g diff --git a/pkg/helm/render_test.go b/pkg/helm/render_test.go index d28b658..67284d4 100644 --- a/pkg/helm/render_test.go +++ b/pkg/helm/render_test.go @@ -15,10 +15,12 @@ import ( "time" "github.com/mgoltzsche/khelm/v2/pkg/config" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "github.com/stretchr/testify/require" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/helmpath" "helm.sh/helm/v3/pkg/repo" "sigs.k8s.io/kustomize/kyaml/yaml" helmyaml "sigs.k8s.io/yaml" @@ -70,16 +72,17 @@ func TestRender(t *testing.T) { }}, {"chart-hooks-disabled", "example/chart-hooks-disabled/generator.yaml", []string{"default"}, " key: myvalue", []string{"chart-hooks-disabled-myconfig"}}, {"git-getter", "example/git-getter/generator.yaml", []string{"cert-manager", "kube-system"}, "ca-sync", nil}, + {"local-chart-with-transitive-remote-and-git-dependencies", "example/localrefref-with-git/generator.yaml", []string{"kube-system", "myotherns"}, "admission.certmanager.k8s.io", nil}, } { t.Run(c.name, func(t *testing.T) { for _, cached := range []string{"", "cached "} { var rendered bytes.Buffer absFile := filepath.Join(rootDir, c.file) err := renderFile(t, absFile, true, rootDir, &rendered) - require.NoError(t, err, "render %s%s", cached, absFile) + require.NoErrorf(t, err, "render %s%s", cached, absFile) b := rendered.Bytes() l, err := readYaml(b) - require.NoError(t, err, "rendered %syaml:\n%s", cached, b) + require.NoErrorf(t, err, "rendered %syaml:\n%s", cached, b) require.True(t, len(l) > 0, "%s: rendered result of %s is empty", cached, c.file) require.Contains(t, rendered.String(), c.expectedContained, "%syaml", cached) foundResourceNames := []string{} @@ -189,16 +192,24 @@ func TestRenderRebuildsLocalDependencies(t *testing.T) { } func TestRenderUpdateRepositoryIndexIfChartNotFound(t *testing.T) { + dir, err := os.MkdirTemp("", "khelm-test-") + require.NoError(t, err) + defer os.RemoveAll(dir) settings := cli.New() - repoURL := "https://charts.rook.io/stable" + settings.RepositoryCache = dir trust := true - repos, err := reposForURLs(map[string]struct{}{repoURL: {}}, &trust, settings, getter.All(settings)) - require.NoError(t, err, "use repo") - entry, err := repos.Get(repoURL) + repos, err := repositories.New(*settings, getter.All(settings), &trust) + require.NoError(t, err) + repoURL := "https://charts.rook.io/stable" + reqRepos, err := reposForURLs(map[string]struct{}{repoURL: {}}, repos) + require.NoError(t, err, "reposForURLs") + require.NotNil(t, reqRepos) + entry, trusted, err := reqRepos.Get(repoURL) require.NoError(t, err, "repos.EntryByURL()") - err = repos.Close() + require.False(t, trusted, "trusted") + err = reqRepos.Close() require.NoError(t, err, "repos.Close()") - idxFile := indexFile(entry, settings.RepositoryCache) + idxFile := repoIndexFile(entry, settings.RepositoryCache) idx := repo.NewIndexFile() // write empty index file to cause not found error err = idx.WriteFile(idxFile, 0644) require.NoError(t, err, "write empty index file") @@ -209,16 +220,23 @@ func TestRenderUpdateRepositoryIndexIfChartNotFound(t *testing.T) { } func TestRenderUpdateRepositoryIndexIfDependencyNotFound(t *testing.T) { + dir, err := os.MkdirTemp("", "khelm-test-") + require.NoError(t, err) + defer os.RemoveAll(dir) settings := cli.New() + settings.RepositoryCache = dir repoURL := "https://kubernetes-charts.storage.googleapis.com" trust := true - repos, err := reposForURLs(map[string]struct{}{repoURL: {}}, &trust, settings, getter.All(settings)) + repos, err := repositories.New(*settings, getter.All(settings), &trust) + require.NoError(t, err) + reqRepos, err := reposForURLs(map[string]struct{}{repoURL: {}}, repos) require.NoError(t, err, "use repo") - entry, err := repos.Get(repoURL) + entry, trusted, err := reqRepos.Get(repoURL) require.NoError(t, err, "repos.Get()") - err = repos.Close() + require.False(t, trusted, "trusted") + err = reqRepos.Close() require.NoError(t, err, "repos.Close()") - idxFile := indexFile(entry, settings.RepositoryCache) + idxFile := repoIndexFile(entry, settings.RepositoryCache) idx := repo.NewIndexFile() // write empty index file to cause not found error err = idx.WriteFile(idxFile, 0644) require.NoError(t, err, "write empty index file") @@ -329,6 +347,10 @@ func TestRenderRepositoryCredentials(t *testing.T) { } } +func repoIndexFile(entry *repo.Entry, cacheDir string) string { + return filepath.Join(cacheDir, helmpath.CacheIndexFile(entry.Name)) +} + type fakePrivateChartServerHandler struct { repo *repo.Entry config *config.LoaderConfig diff --git a/pkg/helm/repositories.go b/pkg/helm/repositories.go index 0688cc9..04d9977 100644 --- a/pkg/helm/repositories.go +++ b/pkg/helm/repositories.go @@ -1,433 +1,141 @@ package helm import ( - "bytes" "context" - "crypto" - "encoding/hex" "fmt" - "io" "log" "os" - "path/filepath" "sort" - "strings" + "github.com/mgoltzsche/khelm/v2/pkg/repositories" "github.com/pkg/errors" "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/helmpath" "helm.sh/helm/v3/pkg/repo" ) -type untrustedRepoError struct { - error -} - -func (e *untrustedRepoError) Format(s fmt.State, verb rune) { - f, isFormatter := e.error.(interface { - Format(s fmt.State, verb rune) - }) - if isFormatter { - f.Format(s, verb) - return - } - fmt.Fprintf(s, "%s", e.error) -} - -// IsUntrustedRepository return true if the provided error is an untrusted repository error -func IsUntrustedRepository(err error) bool { - _, ok := errors.Cause(err).(*untrustedRepoError) - return ok -} - type repositoryConfig interface { Close() error - FilePath() string + File() string ResolveChartVersion(ctx context.Context, name, version, repo string) (*repo.ChartVersion, error) - Get(repo string) (*repo.Entry, error) + Get(repo string) (*repo.Entry, bool, error) UpdateIndex(context.Context) error - DownloadIndexFilesIfNotExist(context.Context) error - RequireTempHelmHome(bool) - Apply() (repositoryConfig, error) + FetchMissingIndexFiles(context.Context) error + TempRepositories() (repositoryConfig, error) } -func reposForURLs(repoURLs map[string]struct{}, trustAnyRepo *bool, settings *cli.EnvSettings, getters getter.Providers) (repositoryConfig, error) { - repos, err := newRepositories(settings, getters) +func reposForURLs(repoURLs map[string]struct{}, repos repositories.Interface) (repositoryConfig, error) { + repoFile, untrusted, err := repoFileFromURLs(repoURLs, repos) if err != nil { return nil, err } - err = repos.setRepositoriesFromURLs(repoURLs, trustAnyRepo) - if err != nil { - return nil, err + r := &requiredRepos{ + Interface: repos, + repoFile: repoFile, } - return repos, nil + if untrusted { + return newTempRepositories(r) + } + return r, nil } // reposForDependencies create temporary repositories.yaml and configure settings with it. -func reposForDependencies(deps []*chart.Dependency, trustAnyRepo *bool, settings *cli.EnvSettings, getters getter.Providers) (repositoryConfig, error) { +func reposForDependencies(deps []*chart.Dependency, repos repositories.Interface) (repositoryConfig, error) { repoURLs := map[string]struct{}{} for _, d := range deps { repoURLs[d.Repository] = struct{}{} } - repos, err := reposForURLs(repoURLs, trustAnyRepo, settings, getters) - if err != nil { - return nil, err - } - return repos, nil -} - -type repositories struct { - filePath string - repos *repo.File - repoURLMap map[string]*repo.Entry - getters getter.Providers - cacheDir string - entriesAdded bool - indexFiles map[string]*repo.IndexFile -} - -func (f *repositories) RequireTempHelmHome(createTemp bool) { - f.entriesAdded = f.entriesAdded || createTemp -} - -func newRepositories(settings *cli.EnvSettings, getters getter.Providers) (r *repositories, err error) { - r = &repositories{ - filePath: settings.RepositoryConfig, - repoURLMap: map[string]*repo.Entry{}, - getters: getters, - cacheDir: settings.RepositoryCache, - indexFiles: map[string]*repo.IndexFile{}, - } - if !filepath.IsAbs(settings.RepositoryConfig) { - return nil, errors.Errorf("path to repositories.yaml must be absolute but was %q", settings.RepositoryConfig) - } - r.repos, err = repo.LoadFile(settings.RepositoryConfig) - if err != nil { - if !os.IsNotExist(errors.Cause(err)) { - return nil, errors.Wrapf(err, "load %s", settings.RepositoryConfig) - } - r.repos = nil - } else { - for _, e := range r.repos.Repositories { - r.repoURLMap[e.URL] = e - } - } - if err = os.MkdirAll(settings.RepositoryCache, 0750); err != nil { - return nil, errors.Wrap(err, "create repo cache dir") - } - return r, nil -} - -func (f *repositories) repoIndex(ctx context.Context, entry *repo.Entry) (*repo.IndexFile, error) { - idx := f.indexFiles[entry.Name] - if idx != nil { - return idx, nil - } - idxFile := indexFile(entry, f.cacheDir) - idx, err := loadIndexFile(ctx, idxFile) - if err != nil { - if os.IsNotExist(errors.Cause(err)) { - err = downloadIndexFile(ctx, entry, f.cacheDir, f.getters) - if err != nil { - return nil, err - } - idx, err = loadIndexFile(ctx, idxFile) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - f.indexFiles[entry.Name] = idx - return idx, nil + return reposForURLs(repoURLs, repos) } -func loadIndexFile(ctx context.Context, idxFile string) (idx *repo.IndexFile, err error) { - done := make(chan struct{}, 1) - go func() { - idx, err = repo.LoadIndexFile(idxFile) - close(done) - }() - select { - case <-done: - return idx, errors.Wrapf(err, "load repo index file %s", idxFile) - case <-ctx.Done(): - return nil, errors.Wrapf(ctx.Err(), "load repo index file %s", idxFile) - } +type requiredRepos struct { + repositories.Interface + repoFile *repo.File } -func (f *repositories) clearRepoIndex(entry *repo.Entry) { - f.indexFiles[entry.Name] = nil -} - -func (f *repositories) ResolveChartVersion(ctx context.Context, name, version, repoURL string) (*repo.ChartVersion, error) { - entry, err := f.Get(repoURL) - if err != nil { - return nil, err - } - idx, err := f.repoIndex(ctx, entry) +func (r *requiredRepos) TempRepositories() (repositoryConfig, error) { + tr, err := newTempRepositories(r) if err != nil { - return nil, err - } - errMsg := fmt.Sprintf("chart %q", name) - if version != "" { - errMsg = fmt.Sprintf("%s version %q", errMsg, version) - } - cv, err := idx.Get(name, version) - if err != nil { - // Download latest index file and retry lookup if not found - err = downloadIndexFile(ctx, entry, f.cacheDir, f.getters) - if err != nil { - return nil, errors.Wrapf(err, "repo index download after %s not found", errMsg) - } - f.clearRepoIndex(entry) - idx, err := f.repoIndex(ctx, entry) - if err != nil { - return nil, err - } - cv, err = idx.Get(name, version) - if err != nil { - return nil, errors.Errorf("%s not found in repository %s", errMsg, entry.URL) - } - } - - if len(cv.URLs) == 0 { - return nil, errors.Errorf("%s has no downloadable URLs", errMsg) - } - return cv, nil -} - -func (f *repositories) FilePath() string { - return f.filePath -} - -func (f *repositories) Apply() (repositoryConfig, error) { - if !f.entriesAdded { - return f, nil // don't create temp repos + return nil, fmt.Errorf("new temp repositories: %w", err) } - return newTempRepositories(f) + return tr, nil } -func (f *repositories) Close() error { +func (_ *requiredRepos) Close() error { return nil } -func (f *repositories) Get(repo string) (*repo.Entry, error) { - isName := false - if strings.HasPrefix(repo, "alias:") { - repo = repo[6:] - isName = true - } - if strings.HasPrefix(repo, "@") { - repo = repo[1:] - isName = true - } - if isName && f.repos != nil { - if entry := f.repos.Get(repo); entry != nil { - return entry, nil - } - return nil, errors.Errorf("repo name %q is not registered in repositories.yaml", repo) - } - if entry := f.repoURLMap[repo]; entry != nil { - return entry, nil - } - return nil, errors.Errorf("repo URL %q is not registered in repositories.yaml", repo) -} - -func (f *repositories) DownloadIndexFilesIfNotExist(ctx context.Context) error { - for _, r := range f.repos.Repositories { - if _, err := os.Stat(indexFile(r, f.cacheDir)); err == nil { - continue // do not update existing repo index - } - if err := downloadIndexFile(ctx, r, f.cacheDir, f.getters); err != nil { - return errors.Wrap(err, "download repo index") +func (r *requiredRepos) FetchMissingIndexFiles(ctx context.Context) error { + for _, entry := range r.repoFile.Repositories { + err := r.Interface.FetchIndexIfNotExist(ctx, entry) + if err != nil { + return err } } return nil } -func (f *repositories) UpdateIndex(ctx context.Context) error { - for _, r := range f.repos.Repositories { - if err := downloadIndexFile(ctx, r, f.cacheDir, f.getters); err != nil { - return errors.Wrap(err, "download repo index") +func (r *requiredRepos) UpdateIndex(ctx context.Context) error { + for _, entry := range r.repoFile.Repositories { + err := r.Interface.UpdateIndex(ctx, entry) + if err != nil { + return err } } return nil } -func (f *repositories) setRepositoriesFromURLs(repoURLs map[string]struct{}, trustAnyRepo *bool) error { - requiredRepos := make([]*repo.Entry, 0, len(repoURLs)) - repoURLMap := map[string]*repo.Entry{} - for u := range repoURLs { - repo, _ := f.Get(u) - if repo != nil { - u = repo.URL - } else if strings.HasPrefix(u, "alias:") || strings.HasPrefix(u, "@") { - return errors.Errorf("repository %q not found in repositories.yaml", u) - } else if trustAnyRepo != nil && !*trustAnyRepo || trustAnyRepo == nil && f.repos != nil { - err := errors.Errorf("repository %q not found in %s and usage of untrusted repositories is disabled", u, f.filePath) - if f.repos == nil { - err = errors.Errorf("request repository %q: %s does not exist and usage of untrusted repositories is disabled", u, f.filePath) - } - return &untrustedRepoError{err} - } - repoURLMap[u] = repo - } - if f.repos != nil { - for _, entry := range f.repos.Repositories { - if repo := repoURLMap[entry.URL]; repo != nil { - requiredRepos = append(requiredRepos, repo) - } - } +func repoFileFromURLs(repoURLSet map[string]struct{}, repos repositories.Interface) (*repo.File, bool, error) { + repoURLMap := make(map[string]*repo.Entry, len(repoURLSet)) + repoURLs := make([]string, 0, len(repoURLMap)) + for k := range repoURLSet { + repoURLs = append(repoURLs, k) } - f.repos = repo.NewFile() - f.repos.Repositories = requiredRepos - newURLs := make([]string, 0, len(repoURLMap)) - for u, knownRepo := range repoURLMap { - if knownRepo == nil { - newURLs = append(newURLs, u) - f.entriesAdded = true - } - } - sort.Strings(newURLs) - for _, repoURL := range newURLs { - if _, err := f.addRepositoryURL(repoURL); err != nil { - return err + sort.Strings(repoURLs) + untrusted := false + newRepos := repo.NewFile() + for _, u := range repoURLs { + entry, found, err := repos.Get(u) + if err != nil { + return nil, false, err } - } - - // Log repository usage - repoUsage := make([]string, len(f.repos.Repositories)) - for i, entry := range f.repos.Repositories { - if repo := repoURLMap[entry.URL]; repo != nil || trustAnyRepo != nil { + newRepos.Add(entry) + if found { authInfo := "unauthenticated" if entry.Username != "" && entry.Password != "" { authInfo = fmt.Sprintf("as user %q", entry.Username) } - repoUsage[i] = fmt.Sprintf("Using repository %q (%s)", entry.URL, authInfo) + log.Printf("Using repository %q (%s)", u, authInfo) } else { - repoUsage[i] = fmt.Sprintf("WARNING: using untrusted repository %q", entry.URL) + untrusted = true + log.Printf("WARNING: using untrusted repository %q", u) } } - sort.Strings(repoUsage) - for _, msg := range repoUsage { - log.Println(msg) - } - - return nil -} - -func (f *repositories) addRepositoryURL(repoURL string) (*repo.Entry, error) { - for _, repo := range f.repos.Repositories { - f.repoURLMap[repo.URL] = repo - } - name, err := urlToHash(repoURL) - if err != nil { - return nil, err - } - if existing := f.repoURLMap[repoURL]; existing != nil { - return existing, nil - } - entry := &repo.Entry{ - Name: name, - URL: repoURL, - } - f.repos.Add(entry) - f.repoURLMap[entry.URL] = entry - f.entriesAdded = true - return entry, nil + return newRepos, untrusted, nil } type tempRepositories struct { - *repositories + *requiredRepos tmpFile string } -func newTempRepositories(r *repositories) (*tempRepositories, error) { +func newTempRepositories(r *requiredRepos) (*tempRepositories, error) { tmpFile, err := os.CreateTemp("", "helm-repositories-") if err != nil { return nil, errors.WithStack(err) } _ = tmpFile.Close() - err = r.repos.WriteFile(tmpFile.Name(), 0640) + err = r.repoFile.WriteFile(tmpFile.Name(), 0640) return &tempRepositories{r, tmpFile.Name()}, err } -func (f *tempRepositories) FilePath() string { - return f.tmpFile -} - -func (f *tempRepositories) Close() error { - return os.Remove(f.tmpFile) -} - -func downloadIndexFile(ctx context.Context, entry *repo.Entry, cacheDir string, getters getter.Providers) error { - log.Printf("Downloading repository index of %s", entry.URL) - idxFile := indexFile(entry, cacheDir) - err := os.MkdirAll(filepath.Dir(idxFile), 0750) - if err != nil { - return errors.WithStack(err) - } - tmpIdxFile, err := os.CreateTemp(filepath.Dir(idxFile), fmt.Sprintf(".tmp-%s-index", entry.Name)) - if err != nil { - return errors.WithStack(err) - } - tmpIdxFileName := tmpIdxFile.Name() - _ = tmpIdxFile.Close() - - interrupt := ctx.Done() - done := make(chan error, 1) - go func() { - var err error - defer func() { - done <- err - close(done) - if err != nil { - _ = os.Remove(tmpIdxFileName) - } - }() - tmpEntry := *entry - tmpEntry.Name = filepath.Base(tmpIdxFileName) - r, err := repo.NewChartRepository(&tmpEntry, getters) - if err != nil { - err = errors.WithStack(err) - return - } - r.CachePath = filepath.Dir(tmpIdxFileName) - tmpIdxFileName, err = r.DownloadIndexFile() - if err != nil { - err = errors.Wrapf(err, "looks like %q is not a valid chart repository or cannot be reached", entry.URL) - return - } - err = os.Rename(tmpIdxFileName, idxFile) - if os.IsExist(err) { - // Ignore error if file was downloaded by another process concurrently. - // This fixes a race condition, see https://github.com/mgoltzsche/khelm/issues/36 - err = os.Remove(tmpIdxFileName) - } - err = errors.WithStack(err) - }() - select { - case err := <-done: - return err - case <-interrupt: - _ = os.Remove(tmpIdxFileName) - return ctx.Err() - } +func (r *tempRepositories) File() string { + return r.tmpFile } -func indexFile(entry *repo.Entry, cacheDir string) string { - return filepath.Join(cacheDir, helmpath.CacheIndexFile(entry.Name)) +func (r *tempRepositories) Close() error { + return os.Remove(r.tmpFile) } -func urlToHash(str string) (string, error) { - hash := crypto.SHA256.New() - if _, err := io.Copy(hash, bytes.NewReader([]byte(str))); err != nil { - return "", errors.Wrap(err, "urlToHash") - } - name := hex.EncodeToString(hash.Sum(nil)) - return strings.ToLower(strings.TrimRight(name, "=")), nil +func (r *tempRepositories) TempRepositories() (repositoryConfig, error) { + return r, nil } diff --git a/pkg/repositories/repositories.go b/pkg/repositories/repositories.go new file mode 100644 index 0000000..e4f8e6e --- /dev/null +++ b/pkg/repositories/repositories.go @@ -0,0 +1,276 @@ +package repositories + +import ( + "bytes" + "context" + "crypto" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/helmpath" + "helm.sh/helm/v3/pkg/repo" +) + +type untrustedRepoError struct { + error +} + +func IsUntrustedRepository(err error) bool { + var urErr *untrustedRepoError + return errors.As(err, &urErr) +} + +type Interface interface { + File() string + ResolveChartVersion(ctx context.Context, name, version, repoURL string) (*repo.ChartVersion, error) + Get(repo string) (*repo.Entry, bool, error) + FetchIndexIfNotExist(ctx context.Context, entry *repo.Entry) error + UpdateIndex(ctx context.Context, entry *repo.Entry) error +} + +type repositories struct { + file string + repos *repo.File + repoURLMap map[string]*repo.Entry + getters getter.Providers + cacheDir string + indexFiles map[string]*repo.IndexFile + trustAnyRepo *bool +} + +func New(settings cli.EnvSettings, getters getter.Providers, trustAnyRepo *bool) (Interface, error) { + if !filepath.IsAbs(settings.RepositoryConfig) { + return nil, fmt.Errorf("path to repositories.yaml must be absolute but was %q", settings.RepositoryConfig) + } + repoURLMap := map[string]*repo.Entry{} + repos, err := repo.LoadFile(settings.RepositoryConfig) + if err != nil { + if !os.IsNotExist(errors.Unwrap(errors.Unwrap(err))) { + return nil, fmt.Errorf("load repo config: %w", err) + } + repos = nil + } else { + for _, e := range repos.Repositories { + repoURLMap[e.URL] = e + } + } + if err = os.MkdirAll(settings.RepositoryCache, 0750); err != nil { + return nil, fmt.Errorf("create repo cache dir: %w", err) + } + return &repositories{ + file: settings.RepositoryConfig, + repos: repos, + repoURLMap: repoURLMap, + getters: getters, + cacheDir: settings.RepositoryCache, + indexFiles: map[string]*repo.IndexFile{}, + trustAnyRepo: trustAnyRepo, + }, nil +} + +func (r *repositories) File() string { + return r.file +} + +func (r *repositories) ResolveChartVersion(ctx context.Context, name, version, repoURL string) (*repo.ChartVersion, error) { + entry, _, err := r.Get(repoURL) + if err != nil { + return nil, err + } + idx, err := r.index(ctx, entry) + if err != nil { + return nil, err + } + errMsg := fmt.Sprintf("chart %q", name) + if version != "" { + errMsg = fmt.Sprintf("%s version %q", errMsg, version) + } + cv, err := idx.Get(name, version) + if err != nil { + // Download latest index file and retry lookup if not found + idx, err := r.update(ctx, entry) + if err != nil { + return nil, fmt.Errorf("%s not found in repository %s: %w", errMsg, entry.URL, err) + } + cv, err = idx.Get(name, version) + if err != nil { + return nil, fmt.Errorf("%s not found in repository %s", errMsg, entry.URL) + } + } + if len(cv.URLs) == 0 { + return nil, fmt.Errorf("%s has no downloadable URLs", errMsg) + } + return cv, nil +} + +func (r *repositories) Get(repo string) (*repo.Entry, bool, error) { + isName := false + if strings.HasPrefix(repo, "alias:") { + repo = repo[6:] + isName = true + } + if strings.HasPrefix(repo, "@") { + repo = repo[1:] + isName = true + } + if isName { + if r.repos == nil { + return nil, false, fmt.Errorf("resolve repo name %q: no repositories.yaml configured to resolve repo alias with", repo) + } + if e := r.repos.Get(repo); e != nil { + entry := *e + return &entry, true, nil + } + return nil, false, fmt.Errorf("repo name %q is not registered in repositories.yaml", repo) + } + if entry := r.repoURLMap[repo]; entry != nil { + return entry, true, nil + } + if r.trustAnyRepo != nil && !*r.trustAnyRepo || r.trustAnyRepo == nil && r.repos != nil { + err := fmt.Errorf("repository %q not found in %s and usage of untrusted repositories is disabled", repo, r.file) + if r.repos == nil { + err = fmt.Errorf("use repository %s: configuration %s does not exist and usage of untrusted repositories is disabled", repo, r.file) + } + return nil, false, &untrustedRepoError{err} + } + return newEntry(repo), false, nil +} + +func (r *repositories) FetchIndexIfNotExist(ctx context.Context, entry *repo.Entry) error { + _, err := r.index(ctx, entry) + return err +} + +func (r *repositories) index(ctx context.Context, entry *repo.Entry) (*repo.IndexFile, error) { + if idx := r.indexFiles[entry.Name]; idx != nil { + return idx, nil + } + idxFile := indexFile(entry, r.cacheDir) + idx, err := loadIndexFile(ctx, idxFile) + if err != nil { + if os.IsNotExist(errors.Unwrap(err)) { + return r.update(ctx, entry) + } + } + return idx, err +} + +func (r *repositories) UpdateIndex(ctx context.Context, entry *repo.Entry) error { + _, err := r.update(ctx, entry) + return err +} + +func (r *repositories) update(ctx context.Context, entry *repo.Entry) (*repo.IndexFile, error) { + err := downloadIndexFile(ctx, entry, r.cacheDir, r.getters) + if err != nil { + return nil, fmt.Errorf("download index for repo %s: %w", entry.URL, err) + } + idxFile := indexFile(entry, r.cacheDir) + idx, err := loadIndexFile(ctx, idxFile) + if err != nil { + return nil, err + } + r.indexFiles[entry.Name] = idx + return idx, nil +} + +func newEntry(repoURL string) *repo.Entry { + name, err := urlToHash(repoURL) + if err != nil { + panic(fmt.Errorf("hash repo url: %w", err)) + } + return &repo.Entry{ + Name: name, + URL: repoURL, + } +} + +func urlToHash(str string) (string, error) { + hash := crypto.SHA256.New() + if _, err := io.Copy(hash, bytes.NewReader([]byte(str))); err != nil { + return "", err + } + name := hex.EncodeToString(hash.Sum(nil)) + return strings.ToLower(strings.TrimRight(name, "=")), nil +} + +func loadIndexFile(ctx context.Context, idxFile string) (idx *repo.IndexFile, err error) { + done := make(chan struct{}, 1) + go func() { + idx, err = repo.LoadIndexFile(idxFile) + close(done) + }() + select { + case <-done: + if err != nil { + return nil, fmt.Errorf("load repo index file %s: %w", idxFile, err) + } + return idx, nil + case <-ctx.Done(): + return nil, fmt.Errorf("load repo index file %s: %w", idxFile, ctx.Err()) + } +} + +func downloadIndexFile(ctx context.Context, entry *repo.Entry, cacheDir string, getters getter.Providers) error { + log.Printf("Downloading repository index of %s", entry.URL) + idxFile := indexFile(entry, cacheDir) + err := os.MkdirAll(filepath.Dir(idxFile), 0750) + if err != nil { + return err + } + tmpIdxFile, err := os.CreateTemp(filepath.Dir(idxFile), fmt.Sprintf(".tmp-%s-index", entry.Name)) + if err != nil { + return err + } + tmpIdxFileName := tmpIdxFile.Name() + _ = tmpIdxFile.Close() + + interrupt := ctx.Done() + done := make(chan error, 1) + go func() { + var err error + defer func() { + done <- err + close(done) + if err != nil { + _ = os.Remove(tmpIdxFileName) + } + }() + tmpEntry := *entry + tmpEntry.Name = filepath.Base(tmpIdxFileName) + r, e := repo.NewChartRepository(&tmpEntry, getters) + if e != nil { + err = e + return + } + r.CachePath = filepath.Dir(tmpIdxFileName) + tmpIdxFileName, err = r.DownloadIndexFile() + if err != nil { + err = fmt.Errorf("looks like %q is not a valid chart repository or cannot be reached: %w", entry.URL, err) + return + } + err = os.Rename(tmpIdxFileName, idxFile) + if os.IsExist(err) { + err = os.Remove(tmpIdxFileName) + } + }() + select { + case err := <-done: + return err + case <-interrupt: + _ = os.Remove(tmpIdxFileName) + return ctx.Err() + } +} + +func indexFile(entry *repo.Entry, cacheDir string) string { + return filepath.Join(cacheDir, helmpath.CacheIndexFile(entry.Name)) +} diff --git a/pkg/repositories/repositories_test.go b/pkg/repositories/repositories_test.go new file mode 100644 index 0000000..235f9eb --- /dev/null +++ b/pkg/repositories/repositories_test.go @@ -0,0 +1,258 @@ +package repositories + +import ( + "bytes" + "context" + "fmt" + "log" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/helmpath" + "helm.sh/helm/v3/pkg/repo" + "sigs.k8s.io/yaml" +) + +func init() { + log.SetFlags(0) +} + +func TestRepositories(t *testing.T) { + dir, err := os.MkdirTemp("", "khelm-test-repositories-") + require.NoError(t, err) + defer os.RemoveAll(dir) + repoURL := "fake://example.org/myorg/myrepo@some/path?ref=v1.2.3" + repoConfPath := filepath.Join(dir, "repositories.yaml") + repoConf := repo.NewFile() + repoConf.Add(&repo.Entry{ + Name: "fake-repo", + URL: repoURL, + Username: "fake-user", + Password: "fake-password", + }) + b, err := yaml.Marshal(repoConf) + require.NoError(t, err) + err = os.WriteFile(repoConfPath, b, 0600) + require.NoError(t, err) + settings := cli.New() + settings.RepositoryCache = filepath.Join(dir, "cache") + settings.RepositoryConfig = "/fake/non-existing/repositories.yaml" + chartURL := "fake://example.org/myorg/myrepo@some/path/mychart.tgz?ref=v1.2.3" + g := &fakeGetter{ + indexURL: "fake://example.org/myorg/myrepo@some/path/index.yaml?ref=v1.2.3", + chartURL: chartURL, + index: fakeIndexFile(chartURL), + } + trust := true + testee := newTestee(t, settings, &trust, g) + t.Run("File should return repositories config path", func(t *testing.T) { + file := testee.File() + require.Equal(t, settings.RepositoryConfig, file, "File()") + _, err = os.Stat(file) + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + }) + t.Run("Get should resolve untrusted repo when trust-any enabled", func(t *testing.T) { + entry, trusted, err := testee.Get(repoURL) + require.NoError(t, err, "Get()") + require.NotNil(t, entry, "entry returned by Get()") + require.False(t, trusted, "trusted") + }) + t.Run("Get should not resolve untrusted repo", func(t *testing.T) { + newSettings := *settings + newSettings.RepositoryConfig = repoConfPath + trust := false + testee := newTestee(t, &newSettings, &trust, g) + entry, trusted, err := testee.Get(repoURL + "-sth") + require.Error(t, err, "Get()") + require.Truef(t, IsUntrustedRepository(err), "error should indicate the repo is untrusted but was: %s", err) + require.False(t, trusted, "trusted") + require.Nil(t, entry, "entry returned by Get()") + }) + var entry *repo.Entry + t.Run("Get should return credentials for trusted repo", func(t *testing.T) { + newSettings := *settings + newSettings.RepositoryConfig = repoConfPath + trust := false + testee := newTestee(t, &newSettings, &trust, g) + var err error + var trusted bool + entry, trusted, err = testee.Get(repoURL) + require.NoError(t, err, "Get()") + require.NotNil(t, entry, "entry returned by Get()") + require.True(t, trusted, "trusted") + require.Equal(t, "fake-user", entry.Username, "username") + require.Equal(t, "fake-password", entry.Password, "password") + }) + t.Run("FetchIndexIfNotExist should fetch when file does not exist", func(t *testing.T) { + err = testee.FetchIndexIfNotExist(context.Background(), entry) + require.NoError(t, err, "FetchIndexIfNotExist()") + idxFilePath := filepath.Join(settings.RepositoryCache, helmpath.CacheIndexFile(entry.Name)) + b, err := os.ReadFile(idxFilePath) + require.NoError(t, err, "read downloaded repo index file") + idx := repo.NewIndexFile() + err = yaml.Unmarshal(b, idx) + require.NoError(t, err, "unmarshal repo index file") + require.Equal(t, g.index, idx, "repo index") + }) + t.Run("FetchIndexIfNotExist should not fetch when file exists", func(t *testing.T) { + origIdx := g.index + g.index = fakeIndexFile(chartURL) + g.index.Entries["fakechart"][0].Version = "0.1.1" + err = testee.FetchIndexIfNotExist(context.Background(), entry) + require.NoError(t, err, "FetchIndexIfNotExist()") + idxFilePath := filepath.Join(settings.RepositoryCache, helmpath.CacheIndexFile(entry.Name)) + b, err := os.ReadFile(idxFilePath) + require.NoError(t, err, "read downloaded repo index file") + idx := repo.NewIndexFile() + err = yaml.Unmarshal(b, idx) + require.NoError(t, err, "unmarshal repo index file") + require.Equal(t, origIdx, idx, "repo index") + }) + t.Run("FetchIndexIfNotExist should fail when index file does not exist", func(t *testing.T) { + newEntry := *entry + newEntry.Name += "-unknown" + newEntry.URL += "-unknown" + err := testee.FetchIndexIfNotExist(context.Background(), &newEntry) + require.Error(t, err) + }) + t.Run("UpdateIndex should update index if file exists", func(t *testing.T) { + g.index.Entries["fakechart"][0].Version = "0.2.0" + err = testee.UpdateIndex(context.Background(), entry) + require.NoError(t, err, "UpdateIndex()") + idxFilePath := filepath.Join(settings.RepositoryCache, helmpath.CacheIndexFile(entry.Name)) + b, err := os.ReadFile(idxFilePath) + require.NoError(t, err, "read downloaded repo index file") + idx := repo.NewIndexFile() + err = yaml.Unmarshal(b, idx) + require.NoError(t, err, "unmarshal repo index file") + require.Equal(t, g.index, idx, "repo index") + }) + t.Run("UpdateIndex should update index if file does not exist", func(t *testing.T) { + err := os.Remove(filepath.Join(settings.RepositoryCache, helmpath.CacheIndexFile(entry.Name))) + require.NoError(t, err, "remove previously downloaded repo index file") + err = testee.UpdateIndex(context.Background(), entry) + require.NoError(t, err, "UpdateIndex()") + idxFilePath := filepath.Join(settings.RepositoryCache, helmpath.CacheIndexFile(entry.Name)) + b, err := os.ReadFile(idxFilePath) + require.NoError(t, err, "read downloaded repo index file") + idx := repo.NewIndexFile() + err = yaml.Unmarshal(b, idx) + require.NoError(t, err, "unmarshal repo index file") + require.Equal(t, g.index, idx, "repo index") + }) + t.Run("ResolveChart should resolve chart version", func(t *testing.T) { + g.index = fakeIndexFile(chartURL) + chartVersion, err := testee.ResolveChartVersion(context.Background(), "fakechart", "0.1.0", repoURL) + require.NoError(t, err) + require.NotNil(t, chartVersion) + require.Equal(t, "fakechart", chartVersion.Name, "name") + require.Equal(t, "0.1.0", chartVersion.Version, "version") + require.Equal(t, []string{chartURL}, chartVersion.URLs, "urls") + }) + t.Run("ResolveChart should resolve chart version range", func(t *testing.T) { + g.index = fakeIndexFile(chartURL) + chartVersion, err := testee.ResolveChartVersion(context.Background(), "fakechart", "0.x.x", repoURL) + require.NoError(t, err) + require.NotNil(t, chartVersion) + require.Equal(t, "fakechart", chartVersion.Name, "name") + require.Equal(t, "0.1.0", chartVersion.Version, "version") + require.Equal(t, []string{chartURL}, chartVersion.URLs, "urls") + }) + t.Run("ResolveChart should resolve empty chart version", func(t *testing.T) { + g.index = fakeIndexFile(chartURL) + chartVersion, err := testee.ResolveChartVersion(context.Background(), "fakechart", "", repoURL) + require.NoError(t, err) + require.NotNil(t, chartVersion) + require.Equal(t, "fakechart", chartVersion.Name, "name") + require.Equal(t, "0.1.0", chartVersion.Version, "version") + require.Equal(t, []string{chartURL}, chartVersion.URLs, "urls") + }) + t.Run("ResolveChart should resolve repo alias", func(t *testing.T) { + g.index = fakeIndexFile(chartURL) + newSettings := *settings + newSettings.RepositoryConfig = repoConfPath + trust := false + testee := newTestee(t, &newSettings, &trust, g) + chartVersion, err := testee.ResolveChartVersion(context.Background(), "fakechart", "0.1.0", "alias:fake-repo") + require.NoError(t, err) + require.NotNil(t, chartVersion) + require.Equal(t, "fakechart", chartVersion.Name, "name") + require.Equal(t, "0.1.0", chartVersion.Version, "version") + require.Equal(t, []string{chartURL}, chartVersion.URLs, "urls") + }) + t.Run("ResolveChart should resolve repo at alias", func(t *testing.T) { + g.index = fakeIndexFile(chartURL) + newSettings := *settings + newSettings.RepositoryConfig = repoConfPath + trust := false + testee := newTestee(t, &newSettings, &trust, g) + chartVersion, err := testee.ResolveChartVersion(context.Background(), "fakechart", "0.1.0", "@fake-repo") + require.NoError(t, err) + require.NotNil(t, chartVersion) + require.Equal(t, "fakechart", chartVersion.Name, "name") + require.Equal(t, "0.1.0", chartVersion.Version, "version") + require.Equal(t, []string{chartURL}, chartVersion.URLs, "urls") + }) + t.Run("ResolveChart should fail when chart does not exist", func(t *testing.T) { + g.index = fakeIndexFile(chartURL) + _, err := testee.ResolveChartVersion(context.Background(), "fakechart", "1.x.x", repoURL) + require.Error(t, err) + }) +} + +func newTestee(t *testing.T, settings *cli.EnvSettings, trust *bool, g *fakeGetter) Interface { + fakeGetterConstructor := func(_ ...getter.Option) (getter.Getter, error) { return g, nil } + r, err := New(*settings, getter.Providers([]getter.Provider{{ + Schemes: []string{"fake"}, + New: fakeGetterConstructor, + }}), trust) + require.NoError(t, err) + return r +} + +func fakeIndexFile(chartURL string) *repo.IndexFile { + return &repo.IndexFile{ + APIVersion: "v2", + Entries: map[string]repo.ChartVersions{ + "fakechart": []*repo.ChartVersion{ + { + Metadata: &chart.Metadata{ + APIVersion: "v2", + Name: "fakechart", + Version: "0.1.0", + }, + URLs: []string{chartURL}, + }, + }, + }, + PublicKeys: []string{}, + } +} + +type fakeGetter struct { + indexURL string + chartURL string + index *repo.IndexFile +} + +func (g *fakeGetter) Get(location string, _ ...getter.Option) (*bytes.Buffer, error) { + var buf bytes.Buffer + if location == g.indexURL { + b, err := yaml.Marshal(g.index) + if err != nil { + panic(err) + } + _, _ = buf.Write(b) + return &buf, nil + } else if location == g.chartURL { + _, _ = buf.WriteString("fake chart tgz") + return &buf, nil + } + return nil, fmt.Errorf("unexpected location %q", location) +}