feat: support multiple cache paths

Support multiple cache paths. Signed-off-by: Noel Georgi <[email protected]>
siderolabs · Mar 6, 2024 · 6eaf7ab · 6eaf7ab
1 parent 6db0478
commit 6eaf7ab
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 23 deletions.
diff --git a/README.md b/README.md
@@ -196,7 +196,9 @@ steps:
         destination: bison.tar.xz
         sha256: 075cef2e814642e30e10e8155e93022e4a91ca38a65aa1d5467d4e969f97f338
         sha512: 00b448db8abe91b07e32ff5273c6617bc1350d806f92073a9472f4c2f0de5d22c152795674171b74f2eb9eff8d36f8173b82dacb215601bb071ae39404d4a8a2
-    cache: "/.cache" # cache mount to be used across builds
+    cachePaths:
+      - /.cache/go-build
+      - /go/pkg
     prepare:
       - tar -xJf bison.tar.xz --strip-components=1
       - mkdir build
@@ -320,7 +322,7 @@ Top-level keys describing phases are (all phases are optional):
 
 - `sources` (download)
 - `env` (environment variables)
-- `cache` (a cache mount to be used across builds)
+- `cachePaths` (a list of cache mount paths to be used across builds)
 - `prepare` (shell script)
 - `build` (shell script)
 - `install` (shell script)

diff --git a/go.mod b/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.0
 	github.com/otiai10/copy v1.14.0
+	github.com/siderolabs/gen v0.4.7
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/oauth2 v0.18.0

diff --git a/go.sum b/go.sum
@@ -95,6 +95,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/siderolabs/gen v0.4.7 h1:lM69UYggT7yzpubf7hEFaNujPdY55Y9zvQf/NC18GvA=
+github.com/siderolabs/gen v0.4.7/go.mod h1:4PBYMdXxTg292IDRq4CGn5AymyDxJVEDvobVKDqFBEA=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=

diff --git a/internal/pkg/convert/node.go b/internal/pkg/convert/node.go
@@ -13,6 +13,7 @@ import (
 	"github.com/moby/buildkit/client/llb"
 	"github.com/opencontainers/go-digest"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/siderolabs/gen/xslices"
 
 	"github.com/siderolabs/bldr/internal/pkg/constants"
 	"github.com/siderolabs/bldr/internal/pkg/environment"
@@ -270,18 +271,16 @@ func (node *NodeLLB) stepScripts(root llb.State, i int, step v1alpha2.Step) llb.
 		for _, instruction := range script.Instructions {
 			runOptions := append([]llb.RunOption(nil), node.Graph.commonRunOptions...)
 
-			if step.CachePath != "" {
-				runOptions = append(runOptions,
-					llb.AddMount(
-						step.CachePath,
-						llb.Scratch(),
-						llb.AsPersistentCacheDir(
-							path.Clean(node.Graph.Options.CacheIDNamespace+"/"+step.CachePath),
-							llb.CacheMountShared,
-						),
+			runOptions = append(runOptions, xslices.Map(step.CachePaths, func(p string) llb.RunOption {
+				return llb.AddMount(
+					p,
+					llb.Scratch(),
+					llb.AsPersistentCacheDir(
+						path.Clean(node.Graph.Options.CacheIDNamespace+"/"+p),
+						llb.CacheMountShared,
 					),
 				)
-			}
+			})...)
 
 			runOptions = append(runOptions,
 				llb.Args([]string{

diff --git a/internal/pkg/integration/testdata/cache/check-cache/pkg.yaml b/internal/pkg/integration/testdata/cache/check-cache/pkg.yaml
@@ -2,8 +2,9 @@ name: check-cache
 dependencies:
   - stage: fill-cache
 steps:
-- cache: /.cache
-  prepare:
+- cachePaths:
+    - /.cache
+  test:
     -  "[ -e /.cache/foo ]"
 finalize:
   - from: /pkg

diff --git a/internal/pkg/integration/testdata/cache/fill-cache/pkg.yaml b/internal/pkg/integration/testdata/cache/fill-cache/pkg.yaml
@@ -1,6 +1,7 @@
 name: fill-cache
 steps:
-- cache: /.cache
+- cachePaths:
+    - /.cache
   prepare:
     -  echo "foo" > /.cache/foo
 

diff --git a/internal/pkg/types/v1alpha2/steps.go b/internal/pkg/types/v1alpha2/steps.go
@@ -28,14 +28,14 @@ func (steps Steps) Validate() error {
 // Steps are executed sequentially, each step runs in its own
 // empty temporary directory.
 type Step struct {
-	Env       Environment  `yaml:"env,omitempty"`
-	CachePath string       `yaml:"cache,omitempty"`
-	TmpDir    string       `yaml:"-"`
-	Sources   Sources      `yaml:"sources,omitempty"`
-	Prepare   Instructions `yaml:"prepare,omitempty"`
-	Build     Instructions `yaml:"build,omitempty"`
-	Install   Instructions `yaml:"install,omitempty"`
-	Test      Instructions `yaml:"test,omitempty"`
+	Env        Environment  `yaml:"env,omitempty"`
+	CachePaths []string     `yaml:"cachePaths,omitempty"`
+	TmpDir     string       `yaml:"-"`
+	Sources    Sources      `yaml:"sources,omitempty"`
+	Prepare    Instructions `yaml:"prepare,omitempty"`
+	Build      Instructions `yaml:"build,omitempty"`
+	Install    Instructions `yaml:"install,omitempty"`
+	Test       Instructions `yaml:"test,omitempty"`
 }
 
 // Validate the step.