feat: support platform override

Support setting `platform` for dependencies and also allow overriding
`platform` for a specific package.

Signed-off-by: Noel Georgi <[email protected]>

Dockerfile

@@ -1,8 +1,8 @@
-# syntax = docker/dockerfile-upstream:1.6.0-labs
+# syntax = docker/dockerfile-upstream:1.7.0-labs
 
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-05T05:52:53Z by kres latest.
+# Generated on 2024-03-06T10:32:46Z by kres latest.
 
 ARG TOOLCHAIN
 

Makefile

@@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2024-03-05T05:52:53Z by kres latest.
+# Generated on 2024-03-06T10:32:46Z by kres latest.
 
 # common variables
 
@@ -14,15 +14,15 @@ WITH_RACE ?= false
 REGISTRY ?= ghcr.io
 USERNAME ?= siderolabs
 REGISTRY_AND_USERNAME ?= $(REGISTRY)/$(USERNAME)
-PROTOBUF_GO_VERSION ?= 1.32.0
+PROTOBUF_GO_VERSION ?= 1.33.0
 GRPC_GO_VERSION ?= 1.3.0
 GRPC_GATEWAY_VERSION ?= 2.19.1
 VTPROTOBUF_VERSION ?= 0.6.0
 DEEPCOPY_VERSION ?= v0.5.6
 GOLANGCILINT_VERSION ?= v1.56.2
 GOFUMPT_VERSION ?= v0.6.0
-GO_VERSION ?= 1.22.0
-GOIMPORTS_VERSION ?= v0.18.0
+GO_VERSION ?= 1.22.1
+GOIMPORTS_VERSION ?= v0.19.0
 GO_BUILDFLAGS ?=
 GO_LDFLAGS ?=
 CGO_ENABLED ?= 0

README.md

@@ -227,6 +227,8 @@ Additionally, hermetic text functions from [Sprig](http://masterminds.github.io/
 On the root level, following properties are available:
 
 - `name` (*str*, *required*): name of the package, also used to reference this package from other packages as dependency.
+- `platform` (*str*, *optional*): platform override for the build.
+  If not set, defaults to the platform of the build.
 - `variant` (*str*, *optional*): variant of the base image of the build.
   Two variants are available:
   - `alpine`: Alpine Linux 3.16 image with `bash` package pre-installed
@@ -270,6 +272,8 @@ Properties:
   Contents of the stage are poured into the build at the location specified with `to:` parameter.
 - `image` (*str*, *external dependency*): reference to the registry container image this package depends on.
   Contents of the image are poured into the build at the location specified with `to:` parameter.
+- `platform` (*str*, *optional*): platform to pull the image for.
+  If not set, defaults to the platform of the build.
 - `runtime` (*bool*, *optional*): if set, marks dependency as runtime.
   This means that when this package is pulled in into the build, all the runtime dependencies are pulled in automatically as well.
   This also applies to transitive runtime dependencies.

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alessio/shellescape v1.4.2
-	github.com/containerd/containerd v1.7.11
+	github.com/containerd/containerd v1.7.13
 	github.com/emicklei/dot v1.6.1
 	github.com/google/go-github/v60 v60.0.0
 	github.com/hashicorp/go-multierror v1.1.1

go.sum

@@ -12,8 +12,8 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7
 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
 github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
-github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw=
-github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE=
+github.com/containerd/containerd v1.7.13 h1:wPYKIeGMN8vaggSKuV1X0wZulpMz4CrgEsZdaCyB6Is=
+github.com/containerd/containerd v1.7.13/go.mod h1:zT3up6yTRfEUa6+GsITYIJNgSVL9NQ4x4h1RPzk0Wu4=
 github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM=
 github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=

internal/pkg/convert/graph.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 
 	"github.com/moby/buildkit/client/llb"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"github.com/siderolabs/bldr/internal/pkg/constants"
 	"github.com/siderolabs/bldr/internal/pkg/environment"
@@ -29,19 +30,24 @@ type GraphLLB struct {
 	LocalContext llb.State
 
 	baseImageProcessor llbProcessor
-	cache              map[*solver.PackageNode]llb.State
+	cache              map[cacheState]llb.State
 
 	commonRunOptions []llb.RunOption
 }
 
+type cacheState struct {
+	*solver.PackageNode
+	Platform string
+}
+
 type llbProcessor func(llb.State) llb.State
 
 // NewGraphLLB creates new GraphLLB and initializes shared images.
 func NewGraphLLB(graph *solver.PackageGraph, options *environment.Options) *GraphLLB {
 	result := &GraphLLB{
 		PackageGraph: graph,
 		Options:      options,
-		cache:        make(map[*solver.PackageNode]llb.State),
+		cache:        make(map[cacheState]llb.State),
 	}
 
 	if options.ProxyEnv != nil {
@@ -55,6 +61,23 @@ func NewGraphLLB(graph *solver.PackageGraph, options *environment.Options) *Grap
 	return result
 }
 
+func (graph *GraphLLB) convertPlatform(platform string) *v1.Platform {
+	switch platform {
+	case "linux/amd64":
+		return &v1.Platform{
+			OS:           "linux",
+			Architecture: "amd64",
+		}
+	case "linux/arm64":
+		return &v1.Platform{
+			OS:           "linux",
+			Architecture: "arm64",
+		}
+	default:
+		return nil
+	}
+}
+
 func (graph *GraphLLB) buildBaseImages() {
 	graph.BaseImages = make(map[v1alpha2.Variant]llb.State)
 
@@ -86,10 +109,20 @@ func (graph *GraphLLB) buildBaseImages() {
 		return addEnv(addPkg(root))
 	}
 
-	graph.BaseImages[v1alpha2.Alpine] = graph.baseImageProcessor(llb.Image(
+	alpineBase := llb.Image(
 		constants.DefaultBaseImage,
 		llb.WithCustomName(graph.Options.CommonPrefix+"base"),
-	).Run(
+	)
+
+	if platform := graph.convertPlatform(graph.Root.Pkg.Platform); platform != nil {
+		alpineBase = llb.Image(
+			constants.DefaultBaseImage,
+			llb.WithCustomName(graph.Options.CommonPrefix+"base"),
+			llb.Platform(*platform),
+		)
+	}
+
+	graph.BaseImages[v1alpha2.Alpine] = graph.baseImageProcessor(alpineBase).Run(
 		append(graph.commonRunOptions,
 			llb.Shlex("apk --no-cache --update add bash"),
 			llb.WithCustomName(graph.Options.CommonPrefix+"base-apkinstall"),
@@ -99,16 +132,25 @@ func (graph *GraphLLB) buildBaseImages() {
 			llb.Args([]string{"ln", "-svf", "/bin/bash", "/bin/sh"}),
 			llb.WithCustomName(graph.Options.CommonPrefix+"base-symlink"),
 		)...,
-	).Root())
+	).Root()
 
 	graph.BaseImages[v1alpha2.Scratch] = graph.baseImageProcessor(llb.Scratch())
 }
 
 func (graph *GraphLLB) buildChecksummer() {
-	graph.Checksummer = llb.Image(
+	alpineBase := llb.Image(
 		constants.DefaultBaseImage,
 		llb.WithCustomName(graph.Options.CommonPrefix+"cksum"),
-	).Run(
+	)
+
+	if platform := graph.convertPlatform(graph.Root.Pkg.Platform); platform != nil {
+		alpineBase = llb.Image(
+			constants.DefaultBaseImage,
+			llb.WithCustomName(graph.Options.CommonPrefix+"cksum"),
+			llb.Platform(*platform))
+	}
+
+	graph.Checksummer = alpineBase.Run(
 		append(graph.commonRunOptions,
 			llb.Shlex("apk --no-cache --update add coreutils"),
 			llb.WithCustomName(graph.Options.CommonPrefix+"cksum-apkinstall"),
@@ -132,7 +174,7 @@ func (graph *GraphLLB) buildLocalContext() {
 
 // Build converts package graph to LLB.
 func (graph *GraphLLB) Build() (llb.State, error) {
-	return NewNodeLLB(graph.Root, graph).Build()
+	return NewNodeLLB(graph.Root, graph).Build(graph.Root.Pkg.Platform)
 }
 
 // Marshal returns marshaled LLB.

internal/pkg/convert/node.go

@@ -57,6 +57,20 @@ type NodeLLB struct {
 
 	Graph  *GraphLLB
 	Prefix string
+	Platform
+}
+
+// Platform wraps v1.Platform to provide Stringer interface.
+type Platform struct {
+	v1.Platform
+}
+
+func (p Platform) String() string {
+	if p.OS == "" && p.Architecture == "" {
+		return ""
+	}
+
+	return p.OS + "/" + p.Architecture
 }
 
 // NewNodeLLB wraps PackageNode for LLB conversion.
@@ -115,7 +129,7 @@ func (node *NodeLLB) convertPlatform(platform string) (v1.Platform, error) {
 
 func (node *NodeLLB) convertDependency(dep solver.PackageDependency) (depState llb.State, srcName string, err error) {
 	if dep.IsInternal() {
-		depState, err = NewNodeLLB(dep.Node, node.Graph).Build()
+		depState, err = NewNodeLLB(dep.Node, node.Graph).Build(node.Pkg.Platform)
 		if err != nil {
 			return llb.Scratch(), "", err
 		}
@@ -160,11 +174,18 @@ func (node *NodeLLB) dependencies(root llb.State) (llb.State, error) {
 	stages := []llb.State{root}
 
 	for _, dep := range deps {
-		if _, alreadyProcessed := seen[dep.ID()]; alreadyProcessed {
+		depID := dep.ID() + node.Platform.String()
+
+		// set dep platform to node platform if not set
+		if dep.Platform == "" {
+			dep.Platform = node.Platform.String()
+		}
+
+		if _, alreadyProcessed := seen[depID]; alreadyProcessed {
 			continue
 		}
 
-		seen[dep.ID()] = struct{}{}
+		seen[depID] = struct{}{}
 
 		depState, srcName, err := node.convertDependency(dep)
 		if err != nil {
@@ -327,8 +348,22 @@ func (node *NodeLLB) finalize(root llb.State) llb.State {
 }
 
 // Build converts PackageNode to buildkit LLB.
-func (node *NodeLLB) Build() (llb.State, error) {
-	if state, ok := node.Graph.cache[node.PackageNode]; ok {
+func (node *NodeLLB) Build(platform string) (llb.State, error) {
+	cacheSt := cacheState{
+		PackageNode: node.PackageNode,
+	}
+
+	if platform != "" {
+		v1Platform, err := node.convertPlatform(platform)
+		if err != nil {
+			return llb.Scratch(), err
+		}
+
+		node.Platform = Platform{v1Platform}
+		cacheSt.Platform = node.Platform.String()
+	}
+
+	if state, ok := node.Graph.cache[cacheSt]; ok {
 		return state, nil
 	}
 
@@ -348,7 +383,7 @@ func (node *NodeLLB) Build() (llb.State, error) {
 
 	root = node.finalize(root)
 
-	node.Graph.cache[node.PackageNode] = root
+	node.Graph.cache[cacheSt] = root
 
 	return root, nil
 }

internal/pkg/integration/testdata/arch-amd64-with-arm64-execution/Pkgfile

@@ -0,0 +1,3 @@
+# syntax = SHEBANG
+
+format: v1alpha2

internal/pkg/integration/testdata/arch-amd64-with-arm64-execution/final/pkg.yaml

@@ -0,0 +1,9 @@
+---
+name: final
+platform: linux/arm64
+steps:
+- test:
+    - test `uname -m` = "aarch64"
+finalize:
+  - from: /
+    to: /

internal/pkg/integration/testdata/arch-amd64-with-arm64-execution/test.yaml

@@ -0,0 +1,17 @@
+---
+run:
+  - name: docker-amd64
+    runner: docker
+    platform: linux/amd64
+    target: final
+    expect: success
+  - name: llb-amd64
+    runner: llb
+    platform: linux/amd64
+    target: final
+    expect: success
+  - name: buildkit-amd64
+    runner: buildkit
+    platform: linux/amd64
+    target: final
+    expect: success

internal/pkg/integration/testdata/arch-amd64/final/pkg.yaml

@@ -1,5 +1,12 @@
 ---
 name: final
+dependencies:
+  # this arm64 variant of the image  only contains the u-boot binary,
+  # so this is a good test to ensure that we can copy the binary from
+  # the arm64 image to the x86_64 image
+  - image: ghcr.io/siderolabs/u-boot:v1.7.0-alpha.0-31-gcb39126
+    platform: linux/arm64
+    from: /rpi_generic
 steps:
 - test:
     - test "${BUILD:-x}" = "x86_64-linux-musl"

internal/pkg/types/v1alpha2/pkg.go

@@ -24,6 +24,7 @@ type Pkg struct {
 	Shell        Shell           `yaml:"shell,omitempty"`
 	BaseDir      string          `yaml:"-"`
 	FileName     string          `yaml:"-"`
+	Platform     string          `yaml:"platform,omitempty"`
 	Install      Install         `yaml:"install,omitempty"`
 	Dependencies Dependencies    `yaml:"dependencies,omitempty"`
 	Steps        Steps           `yaml:"steps,omitempty"`