Add Github runner to build zfs-release RPM (#93)

Add a runner to build the zfs-release RPM and then package it up in artifacts. This re-uses a stripped down version of the QEMU runners from ZFS: openzfs/zfs@bca9b64 Signed-off-by: Tony Hutter <[email protected]> Reviewed-by: Brian Behlendorf <[email protected]>
zfsonlinux · Nov 21, 2024 · 24cfa67 · 24cfa67
1 parent ae83f1e
commit 24cfa67
Show file tree

Hide file tree

Showing 5 changed files with 422 additions and 0 deletions.
diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup qemu instance on action runner
+######################################################################
+
+set -eu
+
+# install needed packages
+export DEBIAN_FRONTEND="noninteractive"
+sudo apt-get -y update
+sudo apt-get install -y axel cloud-image-utils daemonize guestfs-tools \
+  ksmtuned virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
+
+# generate ssh keys
+rm -f ~/.ssh/id_ed25519
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
+
+# we expect RAM shortage
+cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
+# /etc/ksmtuned.conf - Configuration file for ksmtuned
+# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm
+KSM_MONITOR_INTERVAL=60
+
+# Millisecond sleep between ksm scans for 16Gb server.
+# Smaller servers sleep more, bigger sleep less.
+KSM_SLEEP_MSEC=30
+
+KSM_NPAGES_BOOST=0
+KSM_NPAGES_DECAY=0
+KSM_NPAGES_MIN=1000
+KSM_NPAGES_MAX=25000
+
+KSM_THRES_COEF=80
+KSM_THRES_CONST=8192
+
+LOGFILE=/var/log/ksmtuned.log
+DEBUG=1
+EOF
+sudo systemctl restart ksm
+sudo systemctl restart ksmtuned
+
+# not needed
+sudo systemctl stop docker.socket
+sudo systemctl stop multipathd.socket
+
+# remove default swapfile and /mnt
+sudo swapoff -a
+sudo umount -l /mnt
+DISK="/dev/disk/cloud/azure_resource-part1"
+sudo sed -e "s|^$DISK.*||g" -i /etc/fstab
+sudo wipefs -aq $DISK
+sudo systemctl daemon-reload
+
+sudo modprobe loop
+sudo modprobe zfs
+
+# partition the disk as needed
+DISK="/dev/disk/cloud/azure_resource"
+sudo sgdisk --zap-all $DISK
+sudo sgdisk -p \
+ -n 1:0:+16G -c 1:"swap" \
+ -n 2:0:0    -c 2:"tests" \
+$DISK
+sync
+sleep 1
+
+# swap with same size as RAM
+sudo mkswap $DISK-part1
+sudo swapon $DISK-part1
+
+# 60GB data disk
+SSD1="$DISK-part2"
+
+# 10GB data disk on ext4
+sudo fallocate -l 10G /test.ssd1
+SSD2=$(sudo losetup -b 4096 -f /test.ssd1 --show)
+
+# adjust zfs module parameter and create pool
+exec 1>/dev/null
+ARC_MIN=$((1024*1024*256))
+ARC_MAX=$((1024*1024*512))
+echo $ARC_MIN | sudo tee /sys/module/zfs/parameters/zfs_arc_min
+echo $ARC_MAX | sudo tee /sys/module/zfs/parameters/zfs_arc_max
+echo 1 | sudo tee /sys/module/zfs/parameters/zvol_use_blk_mq
+sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 \
+  -O relatime=off -O atime=off -O xattr=sa -O compression=lz4 \
+  -O mountpoint=/mnt/tests
+
+# no need for some scheduler
+for i in /sys/block/s*/queue/scheduler; do
+  echo "none" | sudo tee $i > /dev/null
+done
diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh
@@ -0,0 +1,225 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 2) start qemu with some operating system, init via cloud-init
+######################################################################
+
+set -eu
+
+# short name used in zfs-qemu.yml
+OS="$1"
+
+# OS variant (virt-install --os-variant list)
+OSv=$OS
+
+# compressed with .zst extension
+REPO="https://github.com/mcmilk/openzfs-freebsd-images"
+FREEBSD="$REPO/releases/download/v2024-10-05"
+URLzs=""
+
+# Ubuntu mirrors
+#UBMIRROR="https://cloud-images.ubuntu.com"
+#UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
+UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
+
+# default nic model for vm's
+NIC="virtio"
+
+case "$OS" in
+  almalinux8)
+    OSNAME="AlmaLinux 8"
+    URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  almalinux9)
+    OSNAME="AlmaLinux 9"
+    URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  archlinux)
+    OSNAME="Archlinux"
+    URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
+    # dns sometimes fails with that url  :/
+    echo "89.187.191.12  geo.mirror.pkgbuild.com" | sudo tee /etc/hosts > /dev/null
+    ;;
+  centos-stream9)
+    OSNAME="CentOS Stream 9"
+    URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
+    ;;
+  debian11)
+    OSNAME="Debian 11"
+    URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
+    ;;
+  debian12)
+    OSNAME="Debian 12"
+    URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+    ;;
+  fedora40)
+    OSNAME="Fedora 40"
+    OSv="fedora-unknown"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
+    ;;
+  fedora41)
+    OSNAME="Fedora 41"
+    OSv="fedora-unknown"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"
+    ;;
+  freebsd13-3r)
+    OSNAME="FreeBSD 13.3-RELEASE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.3-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    NIC="rtl8139"
+    ;;
+  freebsd13-4r)
+    OSNAME="FreeBSD 13.4-RELEASE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.4-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    NIC="rtl8139"
+    ;;
+  freebsd14-0r)
+    OSNAME="FreeBSD 14.0-RELEASE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.0-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14-1r)
+    OSNAME="FreeBSD 14.1-RELEASE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd13-4s)
+    OSNAME="FreeBSD 13.4-STABLE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.4-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14-1s)
+    OSNAME="FreeBSD 14.1-STABLE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd15-0c)
+    OSNAME="FreeBSD 15.0-CURRENT"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  tumbleweed)
+    OSNAME="openSUSE Tumbleweed"
+    OSv="opensusetumbleweed"
+    MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
+    URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
+    ;;
+  ubuntu20)
+    OSNAME="Ubuntu 20.04"
+    OSv="ubuntu20.04"
+    URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img"
+    ;;
+  ubuntu22)
+    OSNAME="Ubuntu 22.04"
+    OSv="ubuntu22.04"
+    URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
+    ;;
+  ubuntu24)
+    OSNAME="Ubuntu 24.04"
+    OSv="ubuntu24.04"
+    URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img"
+    ;;
+  *)
+    echo "Wrong value for OS variable!"
+    exit 111
+    ;;
+esac
+
+# environment file
+ENV="/var/tmp/env.txt"
+echo "ENV=$ENV" >> $ENV
+
+# result path
+echo 'RESPATH="/var/tmp/test_results"' >> $ENV
+
+# FreeBSD 13 has problems with: e1000+virtio
+echo "NIC=$NIC" >> $ENV
+
+# freebsd15 -> used in zfs-qemu.yml
+echo "OS=$OS" >> $ENV
+
+# freebsd14.0 -> used for virt-install
+echo "OSv=\"$OSv\"" >> $ENV
+
+# FreeBSD 15 (Current) -> used for summary
+echo "OSNAME=\"$OSNAME\"" >> $ENV
+
+sudo mkdir -p "/mnt/tests"
+sudo chown -R $(whoami) /mnt/tests
+
+# we are downloading via axel, curl and wget are mostly slower and
+# require more return value checking
+IMG="/mnt/tests/cloudimg.qcow2"
+if [ ! -z "$URLzs" ]; then
+  echo "Loading image $URLzs ..."
+  time axel -q -o "$IMG.zst" "$URLzs"
+  zstd -q -d --rm "$IMG.zst"
+else
+  echo "Loading image $URL ..."
+  time axel -q -o "$IMG" "$URL"
+fi
+
+DISK="/dev/zvol/zpool/openzfs"
+FORMAT="raw"
+sudo zfs create -ps -b 64k -V 80g zpool/openzfs
+while true; do test -b $DISK && break; sleep 1; done
+echo "Importing VM image to zvol..."
+sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M
+rm -f $IMG
+
+PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
+cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: $OS
+
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+sudo virsh net-update default add ip-dhcp-host \
+  "<host mac='52:54:00:83:79:00' ip='192.168.122.10'/>" --live --config
+
+sudo virt-install \
+  --os-variant $OSv \
+  --name "openzfs" \
+  --cpu host-passthrough \
+  --virt-type=kvm --hvm \
+  --vcpus=4,sockets=1 \
+  --memory $((1024*12)) \
+  --memballoon model=virtio \
+  --graphics none \
+  --network bridge=virbr0,model=$NIC,mac='52:54:00:83:79:00' \
+  --cloud-init user-data=/tmp/user-data \
+  --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+  --import --noautoconsole >/dev/null
+
+# in case the directory isn't there already
+mkdir -p $HOME/.ssh
+
+cat <<EOF >> $HOME/.ssh/config
+# no questions please
+StrictHostKeyChecking no
+
+# small timeout, used in while loops later
+ConnectTimeout 1
+EOF
diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 3) install dependencies for compiling and loading
+######################################################################
+
+set -eu
+
+sudo dnf install rpm-tools
+
+exit 0
diff --git a/.github/workflows/scripts/qemu-4-build.sh b/.github/workflows/scripts/qemu-4-build.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 4) configure and build openzfs modules
+######################################################################
+
+set -eu
+
+cd zfsonlinux.github.com/zfs-release
+sudo yum -y install rpm-build
+mkdir -p ~/rpmbuild/{BUILDROOT,SPECS,RPMS,SRPMS,SOURCES,BUILD}
+cp RPM-GPG-KEY-openzfs* *.repo ~/rpmbuild/SOURCES
+cp zfs-release.spec ~/rpmbuild/SPECS/
+rpmbuild -ba ~/rpmbuild/SPECS/zfs-release.spec
+
+# building the zfs module was ok
+echo 0 > /var/tmp/build-exitcode.txt
+
+# reset cloud-init configuration and poweroff
+sudo cloud-init clean --logs
+exit 0