feat: custom resolver (#13)

* test: custom resolver prototype

Signed-off-by: Nico Braun <[email protected]>

* refactor: move default file associations to krm package

Signed-off-by: Nico Braun <[email protected]>

* fix: fix errors from rebase

Signed-off-by: Nico Braun <[email protected]>

---------

Signed-off-by: Nico Braun <[email protected]>

internal/krm/renderer.go

@@ -10,6 +10,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
+	"sigs.k8s.io/kustomize/kyaml/utils"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 
 	"github.com/bluebrown/kobold/internal/events"
@@ -29,16 +30,12 @@ type ImageNodeHandlerFunc func(source, parent string, imgNode *yaml.MapNode) err
 // the resolver is responsible for finding one or more image node in a given yaml document
 type Resolver func(node *yaml.RNode, source string, handleImage ImageNodeHandlerFunc) error
 
-// the resolver selector should return the correct resolver based on the file
-// for example for a docker-compose.yaml, the compose resolver should be returned
-type ResolverSelector func(ctx context.Context, source string) Resolver
-
 // the renderer is the high level struct used with the krm framework.
 // its render function runs a kio pipeline using a custom filter based
 // on the renderer options
 type renderer struct {
 	skipfn           kio.LocalPackageSkipFileFunc
-	selector         ResolverSelector
+	selector         *ResolverSelector
 	defaultRegistry  string
 	imageNodeHandler *ImageNodeHandler
 	writer           kio.Writer
@@ -56,7 +53,7 @@ func WithScopes(scopes []string) RendererOption {
 }
 
 // the selector determines which resolver to use for a given file name
-func WithSelector(selector ResolverSelector) RendererOption {
+func WithSelector(selector *ResolverSelector) RendererOption {
 	return func(r *renderer) {
 		r.selector = selector
 	}
@@ -93,7 +90,7 @@ func NewRenderer(opts ...RendererOption) renderer {
 	}
 
 	if r.selector == nil {
-		r.selector = NewSelector(kobold.DefaultAssociations)
+		r.selector = NewSelector(nil, nil)
 	}
 
 	if r.defaultRegistry == "" {
@@ -156,7 +153,7 @@ type filter struct {
 	context  context.Context
 	Events   []events.PushData
 	Changes  []Change
-	selector ResolverSelector
+	selector *ResolverSelector
 	handler  *ImageNodeHandler
 }
 
@@ -174,7 +171,7 @@ func (fn *filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) {
 
 		// select the resolver based on the source and resolve the image nodes with it
 		// once an image node is found, the imageNodeHandler is invoked
-		resolver := fn.selector(fn.context, source)
+		resolver := fn.selector.Select(fn.context, source)
 		if resolver == nil {
 			log.Warn().Str("source", source).Msg("no matching selector")
 			continue
@@ -274,33 +271,96 @@ func resolveKo(node *yaml.RNode, source string, handleImage ImageNodeHandlerFunc
 	})
 }
 
-func NewSelector(fa []kobold.FileTypeSpec) ResolverSelector {
-	return func(ctx context.Context, source string) Resolver {
-		base := filepath.Base(source)
-		var res Resolver
-		for _, a := range fa {
-			ok, err := filepath.Match(a.Pattern, base)
+func NewCustomResolver(name string, paths []string) Resolver {
+	matchers := make([]yaml.Filter, len(paths))
+
+	imageFields := make([]string, len(paths))
+
+	for i, path := range paths {
+		// separate the last part of the path selector to use it to lookup
+		// the image map node once the path
+		smartPath := utils.SmarterPathSplitter(path, ".")
+		imageFields[i] = smartPath[len(smartPath)-1]
+
+		matchers[i] = &yaml.PathMatcher{
+			Path: smartPath[:len(smartPath)-1],
+		}
+	}
+
+	// try each path in the list, but dont stop on first match (for now)
+	return func(node *yaml.RNode, source string, handleImage ImageNodeHandlerFunc) error {
+		for i, matcher := range matchers {
+			matches, err := node.Pipe(matcher)
 			if err != nil {
-				log.Ctx(ctx).Warn().Err(err).Msg("failed to match filetype")
+				return err
+			}
+			if matches == nil {
 				continue
 			}
-			if ok {
-				res = lookupResolver(a.Kind)
-				break
+			err = matches.VisitElements(func(node *yaml.RNode) error {
+				imgNode := node.Field(imageFields[i])
+				if imgNode == nil {
+					return nil
+				}
+				return handleImage(source, "", imgNode)
+			})
+			if err != nil {
+				return err
 			}
 		}
-		return res
+		return nil
+	}
+}
+
+// the resolver selector should return the correct resolver based on the file
+// for example for a docker-compose.yaml, the compose resolver should be returned
+type ResolverSelector struct {
+	resolvers    map[string]Resolver
+	associations []kobold.FileTypeSpec
+}
+
+func NewSelector(resolvers []kobold.ResolverSpec, associations []kobold.FileTypeSpec) *ResolverSelector {
+	resolverMap := map[string]Resolver{
+		"ko":         resolveKo,
+		"compose":    resolveCompose,
+		"kubernetes": resolveKube,
+	}
+
+	for _, res := range resolvers {
+		resolverMap[res.Name] = NewCustomResolver(res.Name, res.Paths)
+	}
+
+	// TODO: merge defaults with user associations ?!
+	if len(associations) == 0 {
+		associations = []kobold.FileTypeSpec{
+			{Kind: "ko", Pattern: ".ko.yaml"},
+			{Kind: "compose", Pattern: "*compose*.y?ml"},
+			{Kind: "kubernetes", Pattern: "*"},
+		}
+	}
+
+	return &ResolverSelector{
+		resolvers:    resolverMap,
+		associations: associations,
 	}
 }
 
-func lookupResolver(kind kobold.FileTypeKind) Resolver {
-	switch kind {
-	case kobold.FileTypeKubernetes:
-		return resolveKube
-	case kobold.FileTypeCompose:
-		return resolveCompose
-	case kobold.FileTypeKo:
-		return resolveKo
+func (s ResolverSelector) Select(ctx context.Context, source string) Resolver {
+	base := filepath.Base(source)
+	var res Resolver
+	for _, a := range s.associations {
+		ok, err := filepath.Match(a.Pattern, base)
+		if err != nil {
+			log.Ctx(ctx).Warn().Err(err).Msg("failed to match filetype")
+			continue
+		}
+		if ok {
+			res, ok = s.resolvers[a.Kind]
+			if !ok {
+				log.Ctx(ctx).Warn().Str("resolver", a.Kind).Msg("resolver does not exist")
+			}
+			break
+		}
 	}
-	return nil
+	return res
 }

internal/krm/renderer_test.go

@@ -5,12 +5,18 @@ import (
 	"testing"
 
 	"github.com/bluebrown/kobold/internal/events"
+	"github.com/bluebrown/kobold/kobold"
 	"github.com/google/go-containerregistry/pkg/name"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
 	"sigs.k8s.io/kustomize/kyaml/kio"
 )
 
-func testPipe(caseDir string, events ...events.PushData) (filesys.FileSystem, error) {
+type testPipeOptions struct {
+	associations []kobold.FileTypeSpec
+	resolvers    []kobold.ResolverSpec
+}
+
+func testPipe(caseDir string, opts testPipeOptions, events ...events.PushData) (filesys.FileSystem, error) {
 	outFs := filesys.MakeFsInMemory()
 	w := kio.LocalPackageWriter{
 		PackagePath: "/",
@@ -19,7 +25,10 @@ func testPipe(caseDir string, events ...events.PushData) (filesys.FileSystem, er
 		},
 	}
 
-	rend := NewRenderer(WithWriter(w))
+	rend := NewRenderer(
+		WithWriter(w),
+		WithSelector(NewSelector(opts.resolvers, opts.associations)),
+	)
 
 	if _, err := rend.Render(context.Background(), "testdata/"+caseDir, events); err != nil {
 		return nil, err
@@ -37,6 +46,7 @@ func Test_renderer_Render(t *testing.T) {
 	tests := []struct {
 		name                 string
 		giveDir              string
+		giveOpts             testPipeOptions
 		giveEvents           []events.PushData
 		wantSourceFieldValue map[string][]wantFieldValue
 	}{
@@ -121,7 +131,7 @@ func Test_renderer_Render(t *testing.T) {
 		// needs to use .krm ignore to ignore invalid yaml portions
 		// {
 		// 	name:    "helm skip errors",
-		// 	giveDir: "helm",
+		// 	giveDir: "helm-skip-errors",
 		// 	giveEvents: []events.PushData{
 		// 		{Image: "index.docker.io/bluebrown/busybox", Tag: "latest", Digest: "sha256:3b3128d9df6bbbcc92e2358e596c9fbd722a437a62bafbc51607970e9e3b8869"},
 		// 	},
@@ -208,13 +218,41 @@ func Test_renderer_Render(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:    "custom-resolver-helm",
+			giveDir: "custom-resolver-helm",
+			giveOpts: testPipeOptions{
+				resolvers: []kobold.ResolverSpec{
+					{Name: "my-helm", Paths: []string{"path.to.image", "another.path"}},
+				},
+				associations: []kobold.FileTypeSpec{{Kind: "my-helm", Pattern: "values.yaml"}},
+			},
+			giveEvents: []events.PushData{
+				{Image: "index.docker.io/bluebrown/echoserver", Tag: "latest", Digest: "sha256:3b3128d9df6bbbcc92e2358e596c9fbd722a437a62bafbc51607970e9e3b8869"},
+				{Image: "test.azurecr.io/nginx", Tag: "latest", Digest: "sha256:220611111e8c9bbe242e9dc1367c0fa89eef83f26203ee3f7c3764046e02b248"},
+			},
+			wantSourceFieldValue: map[string][]wantFieldValue{
+				"values.yaml": {
+					{
+						rnodeIndex: 0,
+						field:      "path.to.image",
+						value:      "index.docker.io/bluebrown/echoserver:latest@sha256:3b3128d9df6bbbcc92e2358e596c9fbd722a437a62bafbc51607970e9e3b8869",
+					},
+					{
+						rnodeIndex: 0,
+						field:      "another.path",
+						value:      "test.azurecr.io/nginx:latest@sha256:220611111e8c9bbe242e9dc1367c0fa89eef83f26203ee3f7c3764046e02b248",
+					},
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			fs, err := testPipe(tt.giveDir, tt.giveEvents...)
+			fs, err := testPipe(tt.giveDir, tt.giveOpts, tt.giveEvents...)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/krm/testdata/custom-resolver-helm/values.yaml

@@ -0,0 +1,6 @@
+path:
+  to:
+    image: docker.io/bluebrown/echoserver:latest # kobold: tag: latest; type: exact
+
+another:
+  path: test.azurecr.io/nginx # kobold: tag: latest; type: exact

internal/server/mux.go

@@ -95,19 +95,14 @@ func (g generator) Generate(conf *kobold.NormalizedConfig) (http.Handler, error)
 
 		bot := gitbot.NewGitbot(sub.Name, repo, sub.Branch, prClient)
 
-		ro := []krm.RendererOption{
+		renderer := krm.NewRenderer(
 			krm.WithScopes(sub.Scopes),
+			krm.WithSelector(krm.NewSelector(conf.Resolvers, sub.FileAssociations)),
 			// these 2 could be part of the subscription config
 			// for now, they will be global for all configs
 			krm.WithDefaultRegistry(g.defaultRegistry),
 			krm.WithImagerefTemplate(g.imagerefTemplate),
-		}
-
-		if len(sub.FileAssociations) > 0 {
-			ro = append(ro, krm.WithSelector(krm.NewSelector(sub.FileAssociations)))
-		}
-
-		renderer := krm.NewRenderer(ro...)
+		)
 
 		// TODO: check if sub with given name already exists and warn user
 		subChan := NewSubscriber(

kobold/config.go

@@ -19,6 +19,9 @@ type NormalizedConfig struct {
 	// both title and description are parsed as template string and executed
 	// with an array of changes as context
 	CommitMessage CommitMessageSpec `json:"commitMessage,omitempty"`
+	// list of custom path resolvers to find image refs
+	// this allows the user to lookup images in arbitrary paths
+	Resolvers []ResolverSpec `json:"resolvers,omitempty"`
 }
 
 type CommitMessageSpec struct {
@@ -78,25 +81,11 @@ const (
 	StrategyPullRequest Strategy = "pull-request"
 )
 
-type FileTypeKind string
-
-const (
-	FileTypeKubernetes FileTypeKind = "kubernetes"
-	FileTypeCompose    FileTypeKind = "docker-compose"
-	FileTypeKo         FileTypeKind = "ko-build"
-)
-
 type FileTypeSpec struct {
-	Kind    FileTypeKind
+	Kind    string
 	Pattern string
 }
 
-var DefaultAssociations = []FileTypeSpec{
-	{Kind: FileTypeKo, Pattern: ".ko.yaml"},
-	{Kind: FileTypeCompose, Pattern: "*compose*.y?ml"},
-	{Kind: FileTypeKubernetes, Pattern: "*"},
-}
-
 type EndpointRef struct {
 	Name string `json:"name,omitempty"`
 }
@@ -114,3 +103,8 @@ type SubscriptionSpec struct {
 	Scopes           []string       `json:"scopes,omitempty"`
 	FileAssociations []FileTypeSpec `json:"fileAssociations,omitempty"`
 }
+
+type ResolverSpec struct {
+	Name  string   `json:"name,omitempty"`
+	Paths []string `json:"paths,omitempty"`
+}