From 57503ccd3bbba510401dbde67340b71eee0adfee Mon Sep 17 00:00:00 2001 From: Artem Chernyshev Date: Mon, 25 Dec 2023 21:22:02 +0300 Subject: [PATCH] feat: generate extensions descriptions file as part of extensions image Add all authors and descriptions to the `descriptions.yaml` file, then add this file to the published `extensions` docker image. Signed-off-by: Artem Chernyshev --- .github/workflows/ci.yaml | 5 +- .gitignore | 3 +- .kres.yaml | 36 ++-- Makefile | 21 +- extensions.yaml | 370 ----------------------------------- internal/extensions/pkg.yaml | 2 + 6 files changed, 31 insertions(+), 406 deletions(-) delete mode 100644 extensions.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b850ba61..b8e683db 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-12-19T16:28:49Z by kres latest. +# Generated on 2023-12-25T14:50:46Z by kres latest. name: default concurrency: @@ -90,9 +90,6 @@ jobs: }) return resp.data.labels.map(label => label.name) - - name: check-dirty - run: | - make check-dirty - name: extensions if: github.event_name != 'pull_request' env: diff --git a/.gitignore b/.gitignore index 734b7cba..97fda4af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-11-02T15:50:53Z by kres latest. +# Generated on 2023-12-25T17:44:28Z by kres latest. _out internal/extensions/image-digests +internal/extensions/descriptions.yaml diff --git a/.kres.yaml b/.kres.yaml index 5f7cbc9a..2d5d63c4 100644 --- a/.kres.yaml +++ b/.kres.yaml @@ -46,6 +46,7 @@ kind: common.Build spec: ignoredPaths: - "internal/extensions/image-digests" + - "internal/extensions/descriptions.yaml" --- kind: auto.CustomSteps spec: @@ -58,6 +59,8 @@ spec: toplevel: true - name: internal/extensions/image-digests toplevel: true + - name: internal/extensions/descriptions.yaml + toplevel: true - name: sign-images toplevel: true - name: extensions-info @@ -75,7 +78,7 @@ spec: - name: PKGS defaultValue: v1.7.0-alpha.0-7-g4c59641 depends: - - internal/extensions/image-digests + - internal/extensions/descriptions.yaml script: - | @$(MAKE) docker-$@ TARGET_ARGS="--tag=$(EXTENSIONS_IMAGE_REF) --push=$(PUSH)" @@ -100,42 +103,33 @@ spec: @$(foreach target,$(NONFREE_TARGETS),echo $(REGISTRY)/$(USERNAME)/$(target):$(shell $(ARTIFACTS)/bldr eval --target $(target) --build-arg TAG=$(TAG) '{{.VERSION}}' 2>/dev/null) >> _out/extensions-metadata;) --- kind: custom.Step -name: extensions-info -spec: - makefile: - enabled: true - phony: true - depends: - - $(ARTIFACTS)/bldr - script: - - | - @find ./ -name "manifest.yaml" -print0 | env LC_ALL=en_US sort -z | xargs -r0 -I{} sh -c 'echo "---\\n$$(cat {})"' > extensions.yaml ---- -kind: custom.Step -name: check-dirty +name: internal/extensions/image-digests spec: makefile: enabled: true phony: true depends: - - extensions-info + - extensions-metadata script: - | - @if test -n "`git status --porcelain`"; then echo "Source tree is dirty"; git status; git diff; exit 1 ; fi - ghaction: - enabled: true + @echo "Generating image digests..." + @cat _out/extensions-metadata | xargs -I{} sh -c 'echo {}@$$(crane digest {})' > internal/extensions/image-digests --- kind: custom.Step -name: internal/extensions/image-digests +name: internal/extensions/descriptions.yaml spec: makefile: enabled: true phony: true depends: - - extensions-metadata + - internal/extensions/image-digests script: - | - @cat _out/extensions-metadata | xargs -I{} sh -c 'echo {}@$$(crane digest {})' > internal/extensions/image-digests + @echo "Generating image descriptions..." + @echo -n "" > internal/extensions/descriptions.yaml + @for image in $(shell cat internal/extensions/image-digests); do \ + crane export $$image - | tar x -O --occurrence=1 manifest.yaml | yq -r ". += {\"$$image\": {\"author\": .metadata.author, \"description\": .metadata.description}} | del(.metadata, .version)" - >> internal/extensions/descriptions.yaml; \ + done --- kind: custom.Step name: sign-images diff --git a/Makefile b/Makefile index da302367..f0410e75 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT. # -# Generated on 2023-12-19T15:09:45Z by kres latest. +# Generated on 2023-12-25T18:19:39Z by kres latest. # common variables @@ -163,12 +163,8 @@ $(ARTIFACTS)/bldr: $(ARTIFACTS) ## Downloads bldr binary. deps.png: ## Generates a dependency graph of the Pkgfile. @$(BLDR) graph | dot -Tpng -o deps.png -.PHONY: check-dirty -check-dirty: extensions-info - @if test -n "`git status --porcelain`"; then echo "Source tree is dirty"; git status; git diff; exit 1 ; fi - .PHONY: extensions -extensions: internal/extensions/image-digests +extensions: internal/extensions/descriptions.yaml @$(MAKE) docker-$@ TARGET_ARGS="--tag=$(EXTENSIONS_IMAGE_REF) --push=$(PUSH)" .PHONY: extensions-metadata @@ -179,8 +175,17 @@ extensions-metadata: $(ARTIFACTS)/bldr .PHONY: internal/extensions/image-digests internal/extensions/image-digests: extensions-metadata + @echo "Generating image digests..." @cat _out/extensions-metadata | xargs -I{} sh -c 'echo {}@$$(crane digest {})' > internal/extensions/image-digests +.PHONY: internal/extensions/descriptions.yaml +internal/extensions/descriptions.yaml: internal/extensions/image-digests + @echo "Generating image descriptions..." + @echo -n "" > internal/extensions/descriptions.yaml + @for image in $(shell cat internal/extensions/image-digests); do \ + crane export $$image - | tar x -O --occurrence=1 manifest.yaml | yq -r ". += {\"$$image\": {\"author\": .metadata.author, \"description\": .metadata.description}} | del(.metadata, .version)" - >> internal/extensions/descriptions.yaml; \ + done + .PHONY: sign-images sign-images: @for image in $(shell crane export $(EXTENSIONS_IMAGE_REF) | tar x --to-stdout image-digests) $(EXTENSIONS_IMAGE_REF)@$$(crane digest $(EXTENSIONS_IMAGE_REF)); do \ @@ -189,10 +194,6 @@ sign-images: cosign sign --yes $$image; \ done -.PHONY: extensions-info -extensions-info: $(ARTIFACTS)/bldr - @find ./ -name "manifest.yaml" -print0 | env LC_ALL=en_US sort -z | xargs -r0 -I{} sh -c 'echo "---\\n$$(cat {})"' > extensions.yaml - .PHONY: rekres rekres: @docker pull $(KRES_IMAGE) diff --git a/extensions.yaml b/extensions.yaml deleted file mode 100644 index b928f45a..00000000 --- a/extensions.yaml +++ /dev/null @@ -1,370 +0,0 @@ ---- -version: v1alpha1 -metadata: - name: ecr-credential-provider - version: "$VERSION" - author: Florian Ströger - description: | - This system extension provides a binary which implements Kubelet's CredentialProvider API - to authenticate against AWS' Elastic Container Registry and pull images. - compatibility: - talos: - version: ">= v1.6.0" ---- -version: v1alpha1 -metadata: - name: gvisor - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides gVisor using containerd's runtime handler. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: stargz-snapshotter - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides Stargz Snapshotter using containerd's runtime handler. - compatibility: - talos: - version: ">= v1.6.0" ---- -version: v1alpha1 -metadata: - name: chelsio-drivers - version: "$VERSION" - author: SideroLabs - description: | - This system extension provides Chelsio network drivers. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: gasket - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides google gasket driver kernel modules built against a specific Talos version. - This driver is required for PCIe and M.2 Google Coral accelerators. There are 2 kernel modules ("gasket" - and "apex") required to enable this driver. - compatibility: - talos: - version: ">= v1.2.0" ---- -version: v1alpha1 -metadata: - name: mellanox-ofed - version: "$VERSION" - author: SideroLabs - description: | - This system extension provides kernel module driver for Mellanox OFED built against a specific Talos version. - compatibility: - talos: - version: ">= v1.3.0" ---- -version: v1alpha1 -metadata: - name: thunderbolt - version: "$VERSION" - author: Igor Rzegocki - description: | - This system extension provides Thunderbolt/USB4 drivers kernel modules built against a specific Talos version. - This driver enables Thunderbolg/USB4 devices, including networking. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: usb-modem - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides USB modem drivers kernel modules built against a specific Talos version. - This driver is required for USB modems to function. This extension includes all the drivers needed to - operate any USB modem under Linux, but your device might not require all of them. Read your device's - docs to learn which drivers you need, or just enable them all as a starting point. - compatibility: - talos: - version: ">= v1.4.5" ---- -version: v1alpha1 -metadata: - name: hello-world-service - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides an example Talos extension service. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: amd-ucode - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides AMD microcode binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: amdgpu-firmware - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides AMDGPU firmware binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: bnx2-bnx2x - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides bnx2 and bnx2x binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: chelsio-firmware - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides Chelsio NIC firmware binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: i915-ucode - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides Intel GPU microcode binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: intel-ice-firmware - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides Intel Ice firmware binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: intel-ucode - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides Intel microcode binaries. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: qlogic-firmware - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides firmware for QLogic devices. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: qemu-guest-agent - version: "$VERSION" - author: Markus Reiter - description: | - This system extension provides the QEMU Guest Agent service. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: xe-guest-utilities - version: "$VERSION" - author: Cas de Reuver - description: | - xe-guest-utilities communicates information and metrics with the Xen host. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: binfmt-misc - version: "$VERSION" - author: Serge Logvinov - description: | - This system extension provides kernel module driver for binfmt-misc built against a specific Talos version. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: tailscale - version: "$VERSION" - author: Beau Trepp - description: | - Tailscale connects your team's devices and development environments for easy access to remote resources. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: nonfree-kmod-nvidia - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides nvidia proprietary kernel modules built against a specific Talos version. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: nvidia-container-toolkit - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides nvidia runtime and it's dependencies using NVIDIA's runtime handler. - compatibility: - talos: - version: ">= v1.2.0" ---- -version: v1alpha1 -metadata: - name: nvidia-fabricmanager - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides the Nvidia fabricmanager for GPU's that need NVLink support. - compatibility: - talos: - version: ">= v1.2.0" ---- -version: v1alpha1 -metadata: - name: nvidia-open-gpu-kernel-modules - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides nvidia open source driver kernel modules built against a specific Talos version. - compatibility: - talos: - version: ">= v1.2.0" ---- -version: v1alpha1 -metadata: - name: nut-client - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides the network-ups-tools upsmon service. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: btrfs - version: "$VERSION" - author: Enno Boland - description: | - This system extension provides kernel module driver for BTRFS built against a specific Talos version. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: drbd - version: "$VERSION" - author: Alexander Prentki - description: | - This system extension provides kernel module driver for DRBD built against a specific Talos version. - compatibility: - talos: - version: ">= v1.2.0" ---- -version: v1alpha1 -metadata: - name: fuse3 - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides fuse3 functionality. - compatibility: - talos: - version: ">= v1.0.0" ---- -version: v1alpha1 -metadata: - name: iscsi-tools - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides iscsi-tools. - compatibility: - talos: - version: ">= v1.1.0" ---- -version: v1alpha1 -metadata: - name: mdadm - version: "$VERSION" - author: Serge Logvinov - description: | - This system extension provides mdadm binary. - compatibility: - talos: - version: ">= v1.5.0" ---- -version: v1alpha1 -metadata: - name: zfs - version: "$VERSION" - author: Andrei Kvapil, Enix - description: | - This system extension provides kernel module driver for ZFS built against a specific Talos version. - compatibility: - talos: - version: ">= v1.6.0" ---- -version: v1alpha1 -metadata: - name: util-linux-tools - version: "$VERSION" - author: Sidero Labs - description: | - This system extension provides a minimal util-linux package. - compatibility: - talos: - version: ">= v1.0.0" diff --git a/internal/extensions/pkg.yaml b/internal/extensions/pkg.yaml index 2d3535be..fd3355e6 100644 --- a/internal/extensions/pkg.yaml +++ b/internal/extensions/pkg.yaml @@ -4,3 +4,5 @@ variant: scratch finalize: - from: /pkg/image-digests to: /image-digests + - from: /pkg/descriptions.yaml + to: /descriptions.yaml