Add nix-snapshotter support to the embedded containerd

Signed-off-by: Edgar Lee <[email protected]>
k3s-io · Oct 24, 2024 · ab340f4 · ab340f4
1 parent 9be4076
commit ab340f4
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 2 deletions.
diff --git a/go.mod b/go.mod
@@ -115,6 +115,7 @@ require (
 	github.com/opencontainers/runc v1.1.13
 	github.com/opencontainers/selinux v1.11.0
 	github.com/otiai10/copy v1.7.0
+	github.com/pdtpartners/nix-snapshotter v0.2.1
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.20.1
 	github.com/prometheus/common v0.55.0

diff --git a/go.sum b/go.sum
@@ -1388,6 +1388,8 @@ github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pdtpartners/nix-snapshotter v0.2.1 h1:NVmZbcDvD5UWNs4oF4vMmhRTlcf596+/fSPb6koiGKs=
+github.com/pdtpartners/nix-snapshotter v0.2.1/go.mod h1:MKa+V5fH15XmLCDt+s8qRQeIAaadaJ3/4+/oD7f0K0k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -2229,8 +2231,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
-gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
 gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go
@@ -594,6 +594,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 						nodeConfig.Containerd.Root)
 				}
 				nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+			case "nix":
+				if err := containerd.NixSupported(nodeConfig.Containerd.Root); err != nil {
+					return nil, errors.Wrapf(err, "\"nix\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
+						nodeConfig.Containerd.Root)
+				}
+				nodeConfig.AgentConfig.ImageServiceSocket = "/run/k3s/nix-snapshotter/nix-snapshotter.sock"
 			}
 		} else {
 			nodeConfig.AgentConfig.ImageServiceSocket = nodeConfig.ContainerRuntimeEndpoint

diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go
@@ -16,6 +16,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/opencontainers/runc/libcontainer/userns"
+	"github.com/pdtpartners/nix-snapshotter/pkg/nix"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -117,3 +118,7 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return stargz.Supported(root)
 }
+
+func NixSupported(root string) error {
+	return nix.Supported(root)
+}
diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go
@@ -64,3 +64,7 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return errors.Wrapf(util3.ErrUnsupportedPlatform, "stargz is not supported")
 }
+
+func NixSupported(root string) error {
+	return errors.Wrapf(util3.ErrUnsupportedPlatform, "nix is not supported")
+}
diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go
@@ -35,7 +35,7 @@ version = 2
 {{- if .NodeConfig.AgentConfig.Snapshotter }}
 [plugins."io.containerd.grpc.v1.cri".containerd]
   snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}"
-  disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}}
+  disable_snapshot_annotations = {{ if or (eq .NodeConfig.AgentConfig.Snapshotter "stargz") (eq .NodeConfig.AgentConfig.Snapshotter "nix") }}false{{else}}true{{end}}
   {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}}
 {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}
 {{ if .NodeConfig.AgentConfig.ImageServiceSocket }}
@@ -60,6 +60,14 @@ enable_keychain = true
 {{end}}
 {{end}}
 {{end}}
+{{ if eq .NodeConfig.AgentConfig.Snapshotter "nix" }}
+[plugins."io.containerd.snapshotter.v1.nix"]
+address = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}"
+image_service.enable = true
+[[plugins."io.containerd.transfer.v1.local".unpack_config]]
+platform = "linux/amd64"
+snapshotter = "nix"
+{{end}}
 {{end}}
 
 {{- if not .NodeConfig.NoFlannel }}

diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go
@@ -32,4 +32,5 @@ import (
 	_ "github.com/containerd/fuse-overlayfs-snapshotter/plugin"
 	_ "github.com/containerd/stargz-snapshotter/service/plugin"
 	_ "github.com/containerd/zfs/plugin"
+	_ "github.com/pdtpartners/nix-snapshotter/pkg/plugin"
 )
diff --git a/pkg/containerd/utility_linux.go b/pkg/containerd/utility_linux.go
@@ -6,6 +6,7 @@ import (
 	"github.com/containerd/containerd/snapshots/overlay/overlayutils"
 	fuseoverlayfs "github.com/containerd/fuse-overlayfs-snapshotter"
 	stargz "github.com/containerd/stargz-snapshotter/service"
+	"github.com/pdtpartners/nix-snapshotter/pkg/nix"
 )
 
 func OverlaySupported(root string) error {
@@ -19,3 +20,7 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return stargz.Supported(root)
 }
+
+func NixSupported(root string) error {
+	return nix.Supported(root)
+}
diff --git a/pkg/containerd/utility_windows.go b/pkg/containerd/utility_windows.go
@@ -19,3 +19,7 @@ func FuseoverlayfsSupported(root string) error {
 func StargzSupported(root string) error {
 	return errors.Wrapf(util2.ErrUnsupportedPlatform, "stargz is not supported")
 }
+
+func NixSupported(root string) error {
+	return errors.Wrapf(util2.ErrUnsupportedPlatform, "nix is not supported")
+}