From 33fb44c0d2ccd7c54dcf399c32221367473bd65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 30 Nov 2023 09:09:19 +0000 Subject: [PATCH 01/13] bin/ubuntu-core-initramfs: add ancillary install files function and use it to install some of files in the initramfs. --- bin/ubuntu-core-initramfs | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 1f6bc512..4d142550 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -276,6 +276,19 @@ def add_modules_from_file(dest_d, kernel_root, modules_d, fw_d, conf_file, db, ) +def install_files(files, dest_dir): + proc_env = os.environ.copy() + proc_env["LP_PRELOAD"] = "" + subprocess.check_call( + [ + "/usr/lib/dracut/dracut-install", + "-D", dest_dir, + "--ldd", "--all", + ] + files, + env=proc_env) + + +def install_systemd_files(dest_dir): def create_initrd(parser, args): rootfs = "/" if not args.kerneldir: @@ -306,14 +319,7 @@ def create_initrd(parser, args): snapd_files = [os.path.join(snapd_lib, "snap-bootstrap"), os.path.join(snapd_lib, "info"), "/lib/systemd/system/snapd.recovery-chooser-trigger.service"] - for snapd_f in snapd_files: - subprocess.check_call( - [ - "/usr/lib/dracut/dracut-install", - "-D", main, - "--ldd", snapd_f, - ] - ) + install_files(snapd_files, main) # Copy features for feature in args.features: # Add feature files @@ -327,15 +333,10 @@ def create_initrd(parser, args): add_modules_from_file(main, kernel_root, modules, firmware, extra_modules, db) if "fips" in args.features: - subprocess.check_call( - [ - "/usr/lib/dracut/dracut-install", - "-D", main, - "--ldd", "--all", + install_files([ "/usr/bin/kcapi-hasher", "/usr/bin/.kcapi-hasher.hmac", - ] + glob.glob("/usr/lib/*/.libkcapi.so.*.hmac") - ) + ] + glob.glob("/usr/lib/*/.libkcapi.so.*.hmac"), main) # Update epoch pathlib.Path("%s/main/usr/lib/clock-epoch" % d).touch() From ecb0b6e431ba6bef3e68b364447248ef898230e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 30 Nov 2023 09:10:37 +0000 Subject: [PATCH 02/13] bin/ubuntu-core-initramfs: install systemd files Install systemd files from the system where the script is run instead of using the ones included in the ubuntu-core-initramfs package. --- bin/ubuntu-core-initramfs | 47 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 4d142550..30b73bf9 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -289,6 +289,49 @@ def install_files(files, dest_dir): def install_systemd_files(dest_dir): + # Build list of files and directories + out = subprocess.check_output( + [ + r"dpkg -L systemd | " + r"grep -E '(^/lib/systemd/system|/modprobe\.d/|/sysctl\.d/|" + r"/rules\.d/|/tmpfiles\.d/|/bin/)'", + ], + shell=True) + subprocess.check_output( + [ + r"dpkg -L systemd-sysv | grep sbin/", + ], + shell=True) + subprocess.check_output( + [ + r"dpkg -L udev | grep -E '(bin/|lib/|rules\.d/)'", + ], + shell=True) + files = out.decode("utf-8").splitlines() + files.append("/var/lib/systemd/") + # Filter out things we don't want + to_remove = re.compile(".*systemd-gpt-auto-generator|" + ".*proc-sys-fs-binfmt_misc.automount|" + ".*systemd-pcrphase.*") + filtered = [i for i in files if not to_remove.match(i)] + # Install + install_files(filtered, dest_dir) + + # Local modifications + # This hack should be removed with PR#113 + subprocess.check_call([r"sed -i '/^After=/" + "{;s, *plymouth-start[.]service *, ,;/" + "^After= *$$/d;}' " + + os.path.join(dest_dir, + "usr/lib/systemd/system/systemd-ask-password-*")], + shell=True) + # Generate hw database (/usr/lib/udev/hwdb.bin) for udev and + # remove redundant definitions after that. + subprocess.check_call(["systemd-hwdb", "--root", dest_dir, + "update", "--usr", "--strict"]) + subprocess.check_call(["rm -rf " + os.path.join(dest_dir, + "usr/lib/udev/hwdb.d")], + shell=True) + + def create_initrd(parser, args): rootfs = "/" if not args.kerneldir: @@ -314,6 +357,8 @@ def create_initrd(parser, args): db = ModuleDb(modules) main = os.path.join(d, "main") os.makedirs(main, exist_ok=True) + # Copy systemd bits + install_systemd_files(main) # Copy snapd bits snapd_lib = path_join_make_rel_paths(rootfs, "/usr/lib/snapd") snapd_files = [os.path.join(snapd_lib, "snap-bootstrap"), @@ -460,7 +505,7 @@ def main(): efi_parser.add_argument("--stub", help="path to stub") if suffix: efi_parser.set_defaults( - stub="/usr/lib/ubuntu-core-initramfs/efi/linux%s.efi.stub" % suffix + stub="/usr/lib/systemd/boot/efi/linux%s.efi.stub" % suffix ) efi_parser.add_argument("--kernel", help="path to kernel", default="/boot/vmlinuz") efi_parser.add_argument( From cb0a3ac0ffc521d1decdc004bde2e5ce94483599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 30 Nov 2023 16:42:49 +0000 Subject: [PATCH 03/13] bin/ubuntu-core-initramfs: install all from run system --- bin/ubuntu-core-initramfs | 114 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 30b73bf9..98b139ab 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -288,6 +288,20 @@ def install_files(files, dest_dir): env=proc_env) +# This is useful if the destination path inside dest_dir is different +# from the current file path. +def install_file_to_path(file, dest_dir, dest_path): + proc_env = os.environ.copy() + proc_env["LP_PRELOAD"] = "" + subprocess.check_call( + [ + "/usr/lib/dracut/dracut-install", + "-D", dest_dir, + "--ldd", file, dest_path, + ], + env=proc_env) + + def install_systemd_files(dest_dir): # Build list of files and directories out = subprocess.check_output( @@ -332,7 +346,103 @@ def install_systemd_files(dest_dir): shell=True) +def install_busybox(dest_dir): + # Use busybox shell instead of dash + os.remove(os.path.join(dest_dir, "usr/bin/sh")) + os.remove(os.path.join(dest_dir, "usr/bin/dash")) + install_file_to_path("/usr/lib/initramfs-tools/bin/busybox", dest_dir, + "usr/bin/busybox") + # Create links to commands + bb_cmd = os.path.join(dest_dir, "usr/bin/busybox") + out = subprocess.check_output([bb_cmd, "--list-long"]) + cmds = out.decode("utf-8").splitlines() + # Remove command we do not want + to_remove = ["busybox", "reboot", "mount", "umount"] + cmds = [c for c in cmds if c not in to_remove] + for c in cmds: + os.symlink("busybox", os.path.join(dest_dir, "usr/bin", c)) + + +def install_misc(dest_dir): + # dmsetup rules + rules = subprocess.check_output([r"dpkg -L dmsetup | grep rules.d"], + shell=True).decode("utf-8").splitlines() + install_files(rules, dest_dir) + # Other needed stuff + out = subprocess.check_output(["dpkg-architecture", "-q", + "DEB_BUILD_MULTIARCH"]).decode("utf-8") + deb_arch = out.splitlines()[0] + files = [ + "/bin/kmod", + "/bin/mount", + "/sbin/sulogin", + "/bin/tar", + "/lib/" + deb_arch + "/libgcc_s.so.1", + "/lib/systemd/system/dbus.service", + "/lib/systemd/system/dbus.socket", + "/lib/systemd/system/plymouth-start.service", + "/lib/systemd/system/plymouth-switch-root.service", + "/lib/systemd/systemd-bootchart", + "/sbin/cryptsetup", + "/sbin/dmsetup", + "/sbin/e2fsck", + "/sbin/fsck", + "/bin/umount", + "/sbin/fsck.vfat", + "/sbin/fsck.vfat", + "/sbin/mkfs.ext4", + "/sbin/mkfs.vfat", + "/sbin/sfdisk", + "/usr/bin/dbus-daemon", + "/usr/bin/mountpoint", + "/usr/bin/partx", + "/usr/bin/plymouth", + "/usr/bin/unsquashfs", + "/usr/lib/" + deb_arch + "/plymouth/label-ft.so", + "/usr/lib/" + deb_arch + "/plymouth/script.so", + "/usr/lib/" + deb_arch + "/plymouth/two-step.so", + "/usr/sbin/depmod", + "/usr/sbin/insmod", + "/usr/sbin/lsmod", + "/usr/sbin/modinfo", + "/usr/sbin/modprobe", + "/usr/sbin/plymouthd", + "/usr/sbin/rmmod", + "/usr/share/dbus-1/system.conf", + "/usr/share/libdrm/amdgpu.ids", + ] + files += glob.glob("/lib/" + deb_arch + "/libnss_compat.so.*") + files += glob.glob("/lib/" + deb_arch + "/libnss_files.so.*") + files += glob.glob("/usr/lib/" + deb_arch + "/plymouth/renderers/*.so") + files += glob.glob("/usr/share/plymouth/themes/bgrt/*") + files += glob.glob("/usr/share/plymouth/themes/spinner/*") + install_files(files, dest_dir) + # Links for fsck + os.symlink("e2fsck", os.path.join(dest_dir, "usr/sbin", "fsck.ext4")) + # Get deps for shared objects that have not the exec bit set and that + # are loaded with dlopen (which means that they are not pull by --ldd + # option, instead use --resolvelazy). + proc_env = os.environ.copy() + proc_env["LP_PRELOAD"] = "" + to_resolve = [ + "/usr/lib/" + deb_arch + "/plymouth/label-ft.so", + "/usr/lib/" + deb_arch + "/plymouth/script.so", + "/usr/lib/" + deb_arch + "/plymouth/two-step.so", + ] + to_resolve += glob.glob("/usr/lib/" + deb_arch + "/plymouth/renderers/*.so") + subprocess.check_call( + [ + "/usr/lib/dracut/dracut-install", + "-D", dest_dir, + "--resolvelazy", + ] + to_resolve, + env=proc_env) + # Build ld cache + subprocess.check_call(["ldconfig", "-r", dest_dir]) + + def create_initrd(parser, args): + # TODO generate microcode instead of shipping in debian package rootfs = "/" if not args.kerneldir: args.kerneldir = "/lib/modules/%s" % args.kernelver @@ -359,6 +469,10 @@ def create_initrd(parser, args): os.makedirs(main, exist_ok=True) # Copy systemd bits install_systemd_files(main) + # copy busybox + install_busybox(main) + # Other miscelanea stuff + install_misc(main) # Copy snapd bits snapd_lib = path_join_make_rel_paths(rootfs, "/usr/lib/snapd") snapd_files = [os.path.join(snapd_lib, "snap-bootstrap"), From e862f666a72866a411795776a7fa13d0fff5f688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 30 Nov 2023 17:08:12 +0000 Subject: [PATCH 04/13] factory: remove folders that we do not really need to ship These folders get created anyway by create-initrd. --- factory/bin | 1 - factory/lib | 1 - factory/lib64 | 1 - factory/sbin | 1 - factory/usr/sbin | 1 - 5 files changed, 5 deletions(-) delete mode 120000 factory/bin delete mode 120000 factory/lib delete mode 120000 factory/lib64 delete mode 120000 factory/sbin delete mode 120000 factory/usr/sbin diff --git a/factory/bin b/factory/bin deleted file mode 120000 index 1e881eda..00000000 --- a/factory/bin +++ /dev/null @@ -1 +0,0 @@ -usr/bin \ No newline at end of file diff --git a/factory/lib b/factory/lib deleted file mode 120000 index 0d5487ba..00000000 --- a/factory/lib +++ /dev/null @@ -1 +0,0 @@ -usr/lib \ No newline at end of file diff --git a/factory/lib64 b/factory/lib64 deleted file mode 120000 index d4801c6b..00000000 --- a/factory/lib64 +++ /dev/null @@ -1 +0,0 @@ -usr/lib64 \ No newline at end of file diff --git a/factory/sbin b/factory/sbin deleted file mode 120000 index 1e881eda..00000000 --- a/factory/sbin +++ /dev/null @@ -1 +0,0 @@ -usr/bin \ No newline at end of file diff --git a/factory/usr/sbin b/factory/usr/sbin deleted file mode 120000 index c5e82d74..00000000 --- a/factory/usr/sbin +++ /dev/null @@ -1 +0,0 @@ -bin \ No newline at end of file From 483a0fa427aed9def059fe16a4b6c3a3cba56edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 30 Nov 2023 17:46:02 +0000 Subject: [PATCH 05/13] debian: move build deps to normal dependencies As now we pull most of the files at run time. --- debian/control | 58 +++++++++++++++++----------- debian/install | 1 - debian/rules | 103 ++----------------------------------------------- 3 files changed, 38 insertions(+), 124 deletions(-) diff --git a/debian/control b/debian/control index 8455b05e..39473b4f 100644 --- a/debian/control +++ b/debian/control @@ -2,37 +2,49 @@ Source: ubuntu-core-initramfs Section: utils Priority: optional Maintainer: Dimitri John Ledkov -Build-Depends: debhelper-compat (= 13), dh-python, python3:any, dracut-core, busybox-initramfs, +Build-Depends: debhelper-compat (= 13), dh-python, amd64-microcode [amd64], - dbus, - dmsetup, - dosfstools, - e2fsprogs, + cpio, fakeroot, - fdisk, fonts-ubuntu, - git, - intel-microcode [amd64], - kmod, - libgcc-s1, - mount, - plymouth-label-ft, - plymouth-theme-spinner, - squashfs-tools, - systemd, - systemd-boot-efi, - systemd-bootchart, - cryptsetup-bin, - systemd-sysv, - tar, - udev, - util-linux + intel-microcode [amd64] Standards-Version: 4.4.1 Homepage: https://launchpad.net/ubuntu-core-initramfs Package: ubuntu-core-initramfs Architecture: amd64 arm64 armhf riscv64 -Depends: ${python3:Depends}, ${misc:Depends}, dracut-core (>= 051-1), zstd, sbsigntool, snapd (>= 2.50+20.04), linux-firmware, llvm, kcapi-tools (>= 1.4.0-1ubuntu3) +Depends: ${python3:Depends}, ${misc:Depends}, dracut-core (>= 051-1), + python3:any, + dracut-core, + busybox-initramfs, + zstd, + sbsigntool, + snapd (>= 2.50+20.04), + linux-firmware, + llvm, + kcapi-tools (>= 1.4.0-1ubuntu3), + dbus, + dmsetup, + dosfstools, + dpkg-dev, + e2fsprogs, + fakeroot, + fdisk, + git, + kmod, + libgcc-s1, + mount, + plymouth-label-ft, + plymouth-theme-spinner, + squashfs-tools, + systemd, + systemd-boot-efi, + systemd-bootchart, + cryptsetup-bin, + systemd-sysv, + tar, + udev, + util-linux Description: standard embedded initrd Standard embedded initrd implementation to be used with Ubuntu Core systems. Currently targetting creating BLS Type2 like binaries. diff --git a/debian/install b/debian/install index 118ab793..7f37d569 100644 --- a/debian/install +++ b/debian/install @@ -2,6 +2,5 @@ bin/ubuntu-core-initramfs usr/bin postinst.d etc/kernel/ snakeoil/* usr/lib/ubuntu-core-initramfs/snakeoil/ debian/tmp/* usr/lib/ubuntu-core-initramfs/main -debian/tmp-efi/* usr/lib/ubuntu-core-initramfs/efi/ modules usr/lib/ubuntu-core-initramfs/ fips usr/lib/ubuntu-core-initramfs/ diff --git a/debian/rules b/debian/rules index 8e454b7a..6da56309 100755 --- a/debian/rules +++ b/debian/rules @@ -6,55 +6,10 @@ include /usr/share/dpkg/default.mk %: dh $@ -INSTALL_FILES_FROM_HOST= \ - /bin/kmod \ - /bin/mount \ - /sbin/sulogin \ - /bin/tar \ - /lib/$(DEB_HOST_MULTIARCH)/libgcc_s.so.1 \ - /lib/$(DEB_HOST_MULTIARCH)/libnss_compat.so.* \ - /lib/$(DEB_HOST_MULTIARCH)/libnss_files.so.* \ - /lib/systemd/system/dbus.service \ - /lib/systemd/system/dbus.socket \ - /lib/systemd/system/plymouth-start.service \ - /lib/systemd/system/plymouth-switch-root.service \ - /lib/systemd/systemd-bootchart \ - /sbin/cryptsetup \ - /sbin/dmsetup \ - /sbin/e2fsck \ - /sbin/fsck /bin/umount \ - /sbin/fsck.vfat \ - /sbin/fsck.vfat \ - /sbin/mkfs.ext4 \ - /sbin/mkfs.vfat \ - /sbin/sfdisk \ - /usr/bin/dbus-daemon \ - /usr/bin/mountpoint \ - /usr/bin/partx \ - /usr/bin/plymouth \ - /usr/bin/unsquashfs \ - /usr/lib/$(DEB_HOST_MULTIARCH)/plymouth/label-ft.so \ - /usr/lib/$(DEB_HOST_MULTIARCH)/plymouth/renderers/*.so \ - /usr/lib/$(DEB_HOST_MULTIARCH)/plymouth/script.so \ - /usr/lib/$(DEB_HOST_MULTIARCH)/plymouth/two-step.so \ - /usr/sbin/depmod \ - /usr/sbin/insmod \ - /usr/sbin/lsmod \ - /usr/sbin/modinfo \ - /usr/sbin/modprobe \ - /usr/sbin/plymouthd \ - /usr/sbin/rmmod \ - /usr/share/dbus-1/system.conf \ - /usr/share/libdrm/amdgpu.ids \ - /usr/share/plymouth/themes/bgrt/* \ - /usr/share/plymouth/themes/spinner/* - override_dh_auto_install: - rm -rf debian/tmp debian/tmp-efi - mkdir debian/tmp debian/tmp-efi + rm -rf debian/tmp + mkdir debian/tmp cp -ar factory/* debian/tmp - mkdir -p debian/tmp/usr/bin - mkdir -p debian/tmp/usr/lib64 # splash functionality mkdir -p debian/tmp/usr/share/plymouth/themes/ @@ -66,61 +21,9 @@ override_dh_auto_install: touch debian/tmp/etc/machine-id - # Include all needed systemd bits - set -e; \ - for f in $$(dpkg -L systemd | \ - grep -E '(^/lib/systemd/system|/modprobe\.d/|/sysctl\.d/|/rules\.d/|/tmpfiles\.d/|/bin/)') \ - $$(dpkg -L systemd-sysv | grep sbin/) \ - /var/lib/systemd/ \ - $$(dpkg -L udev | grep -E '(bin/|lib/|rules\.d/)') \ - ; \ - do \ - LD_PRELOAD= \ - /usr/lib/dracut/dracut-install \ - -D $(CURDIR)/debian/tmp --ldd $$f; \ - done - # But we don't won't to automount the rootfs - rm -f --verbose debian/tmp/usr/lib/systemd/system-generators/systemd-gpt-auto-generator - # or this (binfmt_misc module is not in the initramfs) - rm debian/tmp/usr/lib/systemd/system/sysinit.target.wants/proc-sys-fs-binfmt_misc.automount - # Remove all pcrphase related bits for the moment (services and binary) - find debian/tmp/ -type f,l -name \*systemd-pcrphase\* -delete - - # This hack should be removed with #113 - sed -i '/^After=/{;s, *plymouth-start[.]service *, ,;/^After= *$$/d;}' debian/tmp/usr/lib/systemd/system/systemd-ask-password-* - - # Generate hw database (/usr/lib/udev/hwdb.bin) for udev and - # remove redundant definitions after that. - systemd-hwdb --root debian/tmp update --usr --strict - rm -rf debian/tmp/usr/lib/udev/hwdb.d - - # Use busybox shell instead of dash as in the past - rm debian/tmp/usr/bin/sh debian/tmp/usr/bin/dash - /usr/lib/dracut/dracut-install -D $(CURDIR)/debian/tmp --ldd /usr/lib/initramfs-tools/bin/busybox usr/bin/busybox - set -e; \ - for alias in `debian/tmp/usr/bin/busybox --list-long | grep -v -e busybox -e reboot -e mount -e umount`; do \ - ln -v -s busybox debian/tmp/usr/bin/$$alias; \ - done - - LD_PRELOAD= /usr/lib/dracut/dracut-install -D $(CURDIR)/debian/tmp --ldd $(INSTALL_FILES_FROM_HOST) - dpkg -L dmsetup | grep rules.d | xargs -L1 /usr/lib/dracut/dracut-install -D $(CURDIR)/debian/tmp --ldd - ln -v -s e2fsck debian/tmp/usr/sbin/fsck.ext4 - - set -e; \ - for e in $$(find debian/tmp -type f -executable) \ - /usr/lib/$(DEB_HOST_MULTIARCH)/plymouth/*.so \ - /usr/lib/$(DEB_HOST_MULTIARCH)/plymouth/renderers/*.so; do \ - LD_PRELOAD= \ - /usr/lib/dracut/dracut-install -D $(CURDIR)/debian/tmp --resolvelazy $$e ; \ - done - ldconfig -r debian/tmp - - # Include stub (used when building kernel.efi) - install /usr/lib/systemd/boot/efi/linux*.efi.stub $(CURDIR)/debian/tmp-efi/ - override_dh_install: dh_install - rm -rf debian/ubuntu-core-initramfs/usr/lib/ubuntu-core-initramfs/main/usr/lib/systemd/boot + ifeq ($(DEB_HOST_ARCH),amd64) mkdir -p debian/ubuntu-core-initramfs/usr/lib/ubuntu-core-initramfs/early/ debian/generate-x86-microcode debian/ubuntu-core-initramfs/usr/lib/ubuntu-core-initramfs/early/microcode.cpio From 614308b1d0817f9d717c6eb432f7761a5e63e42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 11:24:38 +0000 Subject: [PATCH 06/13] bin/ubuntu-core-initramfs: fix name of env var --- bin/ubuntu-core-initramfs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 98b139ab..b6010dba 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -278,8 +278,8 @@ def add_modules_from_file(dest_d, kernel_root, modules_d, fw_d, conf_file, db, def install_files(files, dest_dir): proc_env = os.environ.copy() - proc_env["LP_PRELOAD"] = "" subprocess.check_call( + proc_env["LD_PRELOAD"] = "" [ "/usr/lib/dracut/dracut-install", "-D", dest_dir, @@ -292,8 +292,8 @@ def install_files(files, dest_dir): # from the current file path. def install_file_to_path(file, dest_dir, dest_path): proc_env = os.environ.copy() - proc_env["LP_PRELOAD"] = "" subprocess.check_call( + proc_env["LD_PRELOAD"] = "" [ "/usr/lib/dracut/dracut-install", "-D", dest_dir, @@ -423,7 +423,7 @@ def install_misc(dest_dir): # are loaded with dlopen (which means that they are not pull by --ldd # option, instead use --resolvelazy). proc_env = os.environ.copy() - proc_env["LP_PRELOAD"] = "" + proc_env["LD_PRELOAD"] = "" to_resolve = [ "/usr/lib/" + deb_arch + "/plymouth/label-ft.so", "/usr/lib/" + deb_arch + "/plymouth/script.so", From 3bab853133503393affae5fe6e2b4ffa1c2b7cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 11:25:34 +0000 Subject: [PATCH 07/13] bin/ubuntu-core-initramfs: import used functions from subprocess For clearer calls. --- bin/ubuntu-core-initramfs | 66 +++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index b6010dba..95c38913 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -1,7 +1,7 @@ #!/usr/bin/python3 import argparse import os -import subprocess +from subprocess import run, check_call, check_output import tempfile import pathlib import platform @@ -259,7 +259,7 @@ def add_modules_from_file(dest_d, kernel_root, modules_d, fw_d, conf_file, db, if not mod.builtin: db.mark_installed(module, conf_file) - subprocess.check_call( + check_call( [ "/usr/lib/dracut/dracut-install", "-D", @@ -278,8 +278,8 @@ def add_modules_from_file(dest_d, kernel_root, modules_d, fw_d, conf_file, db, def install_files(files, dest_dir): proc_env = os.environ.copy() - subprocess.check_call( proc_env["LD_PRELOAD"] = "" + check_call( [ "/usr/lib/dracut/dracut-install", "-D", dest_dir, @@ -292,8 +292,8 @@ def install_files(files, dest_dir): # from the current file path. def install_file_to_path(file, dest_dir, dest_path): proc_env = os.environ.copy() - subprocess.check_call( proc_env["LD_PRELOAD"] = "" + check_call( [ "/usr/lib/dracut/dracut-install", "-D", dest_dir, @@ -331,19 +331,19 @@ def install_systemd_files(dest_dir): # Local modifications # This hack should be removed with PR#113 - subprocess.check_call([r"sed -i '/^After=/" - "{;s, *plymouth-start[.]service *, ,;/" - "^After= *$$/d;}' " - + os.path.join(dest_dir, - "usr/lib/systemd/system/systemd-ask-password-*")], - shell=True) + check_call([r"sed -i '/^After=/" + r"{;s, *plymouth-start[.]service *, ,;/" + r"^After= *$$/d;}' " + + os.path.join(dest_dir, + "usr/lib/systemd/system/systemd-ask-password-*")], + shell=True) # Generate hw database (/usr/lib/udev/hwdb.bin) for udev and # remove redundant definitions after that. - subprocess.check_call(["systemd-hwdb", "--root", dest_dir, - "update", "--usr", "--strict"]) - subprocess.check_call(["rm -rf " + os.path.join(dest_dir, - "usr/lib/udev/hwdb.d")], - shell=True) + check_call(["systemd-hwdb", "--root", dest_dir, + "update", "--usr", "--strict"]) + check_call(["rm -rf " + os.path.join(dest_dir, + "usr/lib/udev/hwdb.d")], + shell=True) def install_busybox(dest_dir): @@ -354,7 +354,7 @@ def install_busybox(dest_dir): "usr/bin/busybox") # Create links to commands bb_cmd = os.path.join(dest_dir, "usr/bin/busybox") - out = subprocess.check_output([bb_cmd, "--list-long"]) + out = check_output([bb_cmd, "--list-long"]) cmds = out.decode("utf-8").splitlines() # Remove command we do not want to_remove = ["busybox", "reboot", "mount", "umount"] @@ -365,12 +365,12 @@ def install_busybox(dest_dir): def install_misc(dest_dir): # dmsetup rules - rules = subprocess.check_output([r"dpkg -L dmsetup | grep rules.d"], - shell=True).decode("utf-8").splitlines() + rules = check_output([r"dpkg -L dmsetup | grep rules.d"], + shell=True).decode("utf-8").splitlines() install_files(rules, dest_dir) # Other needed stuff - out = subprocess.check_output(["dpkg-architecture", "-q", - "DEB_BUILD_MULTIARCH"]).decode("utf-8") + out = check_output(["dpkg-architecture", "-q", + "DEB_BUILD_MULTIARCH"]).decode("utf-8") deb_arch = out.splitlines()[0] files = [ "/bin/kmod", @@ -430,7 +430,7 @@ def install_misc(dest_dir): "/usr/lib/" + deb_arch + "/plymouth/two-step.so", ] to_resolve += glob.glob("/usr/lib/" + deb_arch + "/plymouth/renderers/*.so") - subprocess.check_call( + check_call( [ "/usr/lib/dracut/dracut-install", "-D", dest_dir, @@ -438,7 +438,7 @@ def install_misc(dest_dir): ] + to_resolve, env=proc_env) # Build ld cache - subprocess.check_call(["ldconfig", "-r", dest_dir]) + check_call(["ldconfig", "-r", dest_dir]) def create_initrd(parser, args): @@ -459,10 +459,10 @@ def create_initrd(parser, args): modules = os.path.join(kernel_root, "usr", "lib", "modules") os.makedirs(modules, exist_ok=True) modules = os.path.join(modules, args.kernelver) - subprocess.check_call(["cp", "-ar", args.kerneldir, modules]) + check_call(["cp", "-ar", args.kerneldir, modules]) firmware = os.path.join(kernel_root, "usr", "lib", "firmware") - subprocess.check_call(["cp", "-ar", args.firmwaredir, firmware]) + check_call(["cp", "-ar", args.firmwaredir, firmware]) db = ModuleDb(modules) main = os.path.join(d, "main") @@ -484,7 +484,7 @@ def create_initrd(parser, args): # Add feature files feature_path = os.path.join(args.skeleton, feature) if os.path.isdir(feature_path): - subprocess.check_call(["cp", "-aT", feature_path, main]) + check_call(["cp", "-aT", feature_path, main]) # Add feature kernel modules extra_modules = os.path.join(args.skeleton, "modules", feature, "extra-modules.conf") @@ -505,7 +505,7 @@ def create_initrd(parser, args): warn_discoverable=True) for modulesf in ["modules.order", "modules.builtin", "modules.builtin.bin", "modules.builtin.modinfo"]: - subprocess.check_call( + check_call( [ "/usr/lib/dracut/dracut-install", "-D", @@ -515,13 +515,13 @@ def create_initrd(parser, args): os.path.join("usr/lib/modules", args.kernelver, modulesf), ] ) - subprocess.check_call(["depmod", "-a", "-b", main, args.kernelver]) + check_call(["depmod", "-a", "-b", main, args.kernelver]) with open(args.output, "wb") as output: for early in glob.iglob("%s/early/*.cpio" % args.skeleton): with open(early, "rb") as f: shutil.copyfileobj(f, output) output.write( - subprocess.run( + run( "find . | cpio --create --quiet --format='newc' --owner=0:0 | zstd -1 -T0", cwd=main, capture_output=True, @@ -547,7 +547,7 @@ def create_efi(parser, args): if platform.machine() == "aarch64": import gzip - raw_kernel=tempfile.NamedTemporaryFile(mode='wb') + raw_kernel = tempfile.NamedTemporaryFile(mode='wb') try: with gzip.open(args.kernel, 'rb') as comp_kernel: shutil.copyfileobj(comp_kernel, raw_kernel) @@ -584,9 +584,9 @@ def create_efi(parser, args): args.stub, args.output, ] - subprocess.check_call(objcopy_cmd) + check_call(objcopy_cmd) if not args.unsigned: - subprocess.check_call( + check_call( [ "sbsign", "--key", @@ -602,7 +602,7 @@ def create_efi(parser, args): def main(): - kernelver = subprocess.check_output( + kernelver = check_output( ["uname", "-r"], universal_newlines=True ).strip() suffix = {"x86_64": "x64", @@ -671,7 +671,7 @@ def main(): "--output", help="path to output", default="/boot/ubuntu-core-initramfs.img" ) initrd_parser.set_defaults( - kernelver=subprocess.check_output( + kernelver=check_output( ["uname", "-r"], universal_newlines=True ).strip() ) From 91f59aacb11097625c42e96fced25dab992596a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 11:45:42 +0000 Subject: [PATCH 08/13] bin/ubuntu-core-initramfs: obtain files to install by filtering in the script instead of by using grep. --- bin/ubuntu-core-initramfs | 60 ++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 95c38913..4e9d6b2f 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -304,24 +304,44 @@ def install_file_to_path(file, dest_dir, dest_path): def install_systemd_files(dest_dir): # Build list of files and directories - out = subprocess.check_output( - [ - r"dpkg -L systemd | " - r"grep -E '(^/lib/systemd/system|/modprobe\.d/|/sysctl\.d/|" - r"/rules\.d/|/tmpfiles\.d/|/bin/)'", - ], - shell=True) + subprocess.check_output( - [ - r"dpkg -L systemd-sysv | grep sbin/", - ], - shell=True) + subprocess.check_output( - [ - r"dpkg -L udev | grep -E '(bin/|lib/|rules\.d/)'", - ], - shell=True) - files = out.decode("utf-8").splitlines() + + out = check_output(["dpkg", "-L", "systemd", "systemd-sysv"]) + lines = out.decode("utf-8").splitlines() + # From systemd, we pull + # * units configuration + # * Executables + # * Module load options + # * Configuration of kernel parameters + # * udev rules + # * Configuration for systemd-tmpfiles + # TODO: some of this can be cleaned up + to_include = re.compile(r"^/lib/systemd/system|" + r".*/modprobe\.d/|" + r".*/sysctl\.d/|" + r".*/rules\.d/|" + r".*/tmpfiles\.d/|" + r".*bin/|" + r".*sbin/" + ) + files = [i for i in lines if to_include.match(i)] + + out = check_output(["dpkg", "-L", "udev"]) + lines = out.decode("utf-8").splitlines() + # From udev, we pull + # * Executables + # * systemd configuration (units, tmpfiles) + # * udev rules + # * hwdb + # TODO: some of this can be cleaned up + to_include = re.compile(r".*bin/|" + r".*lib/|" + r".*rules\.d/" + ) + files += [i for i in lines if to_include.match(i)] + files.append("/var/lib/systemd/") - # Filter out things we don't want + + # Filter out some units we don't want to_remove = re.compile(".*systemd-gpt-auto-generator|" ".*proc-sys-fs-binfmt_misc.automount|" ".*systemd-pcrphase.*") @@ -365,9 +385,11 @@ def install_busybox(dest_dir): def install_misc(dest_dir): # dmsetup rules - rules = check_output([r"dpkg -L dmsetup | grep rules.d"], - shell=True).decode("utf-8").splitlines() + rules = check_output(["dpkg", "-L", "dmsetup"]).decode("utf-8").splitlines() + to_include = re.compile(r".*rules.d/") + rules = [i for i in rules if to_include.match(i)] install_files(rules, dest_dir) + # Other needed stuff out = check_output(["dpkg-architecture", "-q", "DEB_BUILD_MULTIARCH"]).decode("utf-8") From 07b37461baa53ea3c3d47ccac3c70681c94d3067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 13:29:34 +0000 Subject: [PATCH 09/13] bin/ubuntu-core-initramfs: pull busybox first dracut-install installs the interpreters detected in shebangs (see resolve_deps() in dracut-install.c) - but we want busybox, not the system default. --- bin/ubuntu-core-initramfs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 4e9d6b2f..349ab012 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -367,9 +367,6 @@ def install_systemd_files(dest_dir): def install_busybox(dest_dir): - # Use busybox shell instead of dash - os.remove(os.path.join(dest_dir, "usr/bin/sh")) - os.remove(os.path.join(dest_dir, "usr/bin/dash")) install_file_to_path("/usr/lib/initramfs-tools/bin/busybox", dest_dir, "usr/bin/busybox") # Create links to commands @@ -489,10 +486,12 @@ def create_initrd(parser, args): db = ModuleDb(modules) main = os.path.join(d, "main") os.makedirs(main, exist_ok=True) + # copy busybox first so we get already the shell interpreter we + # want (busybox) instead of dracut-install pulling the systemd + # default (dash) when it pulls a shell script later. + install_busybox(main) # Copy systemd bits install_systemd_files(main) - # copy busybox - install_busybox(main) # Other miscelanea stuff install_misc(main) # Copy snapd bits From c6fb236834f1d51693b571909e6093a5d574f9b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 13:56:56 +0000 Subject: [PATCH 10/13] bin/ubuntu-core-initramfs: use DEB_HOST_MULTIARCH instead of *_BUILD_* as we are interested in the run system now. --- bin/ubuntu-core-initramfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 349ab012..b8cbfcdb 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -389,7 +389,7 @@ def install_misc(dest_dir): # Other needed stuff out = check_output(["dpkg-architecture", "-q", - "DEB_BUILD_MULTIARCH"]).decode("utf-8") + "DEB_HOST_MULTIARCH"]).decode("utf-8") deb_arch = out.splitlines()[0] files = [ "/bin/kmod", From 45c73bdad5835069c0537ce700c3b7993cb66905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 13:57:59 +0000 Subject: [PATCH 11/13] bin/ubuntu-core-initramfs: add /usr prefix to files to include we want to avoid symbolic links. --- bin/ubuntu-core-initramfs | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index b8cbfcdb..9f444fb3 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -392,26 +392,26 @@ def install_misc(dest_dir): "DEB_HOST_MULTIARCH"]).decode("utf-8") deb_arch = out.splitlines()[0] files = [ - "/bin/kmod", - "/bin/mount", - "/sbin/sulogin", - "/bin/tar", - "/lib/" + deb_arch + "/libgcc_s.so.1", - "/lib/systemd/system/dbus.service", - "/lib/systemd/system/dbus.socket", - "/lib/systemd/system/plymouth-start.service", - "/lib/systemd/system/plymouth-switch-root.service", - "/lib/systemd/systemd-bootchart", - "/sbin/cryptsetup", - "/sbin/dmsetup", - "/sbin/e2fsck", - "/sbin/fsck", - "/bin/umount", - "/sbin/fsck.vfat", - "/sbin/fsck.vfat", - "/sbin/mkfs.ext4", - "/sbin/mkfs.vfat", - "/sbin/sfdisk", + "/usr/bin/kmod", + "/usr/bin/mount", + "/usr/sbin/sulogin", + "/usr/bin/tar", + "/usr/lib/" + deb_arch + "/libgcc_s.so.1", + "/usr/lib/systemd/system/dbus.service", + "/usr/lib/systemd/system/dbus.socket", + "/usr/lib/systemd/system/plymouth-start.service", + "/usr/lib/systemd/system/plymouth-switch-root.service", + "/usr/lib/systemd/systemd-bootchart", + "/usr/sbin/cryptsetup", + "/usr/sbin/dmsetup", + "/usr/sbin/e2fsck", + "/usr/sbin/fsck", + "/usr/bin/umount", + "/usr/sbin/fsck.vfat", + "/usr/sbin/fsck.vfat", + "/usr/sbin/mkfs.ext4", + "/usr/sbin/mkfs.vfat", + "/usr/sbin/sfdisk", "/usr/bin/dbus-daemon", "/usr/bin/mountpoint", "/usr/bin/partx", From 86c7aa4378c73fa1a9e06fc62c56a8d89a4392a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 1 Dec 2023 14:02:06 +0000 Subject: [PATCH 12/13] bin/ubuntu-core-initramfs: add TODO about fips feature --- bin/ubuntu-core-initramfs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index 9f444fb3..a5d688b5 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -512,6 +512,9 @@ def create_initrd(parser, args): if os.path.exists(extra_modules): add_modules_from_file(main, kernel_root, modules, firmware, extra_modules, db) + # TODO xnox: fips actually needs additional runtime dependencies than + # this, and currently forked in fips PPA. we need to figure out how to + # make ubuntu-core-initramfs-fips a reality. if "fips" in args.features: install_files([ "/usr/bin/kcapi-hasher", From 913abb84bb2603a744e79c3f3b32934bc2139499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Fri, 8 Dec 2023 16:27:54 +0000 Subject: [PATCH 13/13] bin/ubuntu-core-initramfs: address review comments --- bin/ubuntu-core-initramfs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/ubuntu-core-initramfs b/bin/ubuntu-core-initramfs index a5d688b5..a5c31a49 100755 --- a/bin/ubuntu-core-initramfs +++ b/bin/ubuntu-core-initramfs @@ -302,11 +302,16 @@ def install_file_to_path(file, dest_dir, dest_path): env=proc_env) +# Returns as a list the files contained in a list of deb packages. +def package_files(pkgs): + out = check_output(["dpkg", "-L"] + pkgs) + return out.decode("utf-8").splitlines() + + def install_systemd_files(dest_dir): # Build list of files and directories - out = check_output(["dpkg", "-L", "systemd", "systemd-sysv"]) - lines = out.decode("utf-8").splitlines() + lines = package_files(["systemd", "systemd-sysv"]) # From systemd, we pull # * units configuration # * Executables @@ -325,8 +330,7 @@ def install_systemd_files(dest_dir): ) files = [i for i in lines if to_include.match(i)] - out = check_output(["dpkg", "-L", "udev"]) - lines = out.decode("utf-8").splitlines() + lines = package_files(["udev"]) # From udev, we pull # * Executables # * systemd configuration (units, tmpfiles) @@ -353,7 +357,7 @@ def install_systemd_files(dest_dir): # This hack should be removed with PR#113 check_call([r"sed -i '/^After=/" r"{;s, *plymouth-start[.]service *, ,;/" - r"^After= *$$/d;}' " + r"^After= *$/d;}' " + os.path.join(dest_dir, "usr/lib/systemd/system/systemd-ask-password-*")], shell=True) @@ -361,9 +365,7 @@ def install_systemd_files(dest_dir): # remove redundant definitions after that. check_call(["systemd-hwdb", "--root", dest_dir, "update", "--usr", "--strict"]) - check_call(["rm -rf " + os.path.join(dest_dir, - "usr/lib/udev/hwdb.d")], - shell=True) + shutil.rmtree(os.path.join(dest_dir, "usr/lib/udev/hwdb.d")) def install_busybox(dest_dir): @@ -373,8 +375,8 @@ def install_busybox(dest_dir): bb_cmd = os.path.join(dest_dir, "usr/bin/busybox") out = check_output([bb_cmd, "--list-long"]) cmds = out.decode("utf-8").splitlines() - # Remove command we do not want - to_remove = ["busybox", "reboot", "mount", "umount"] + # Remove commands we do not want + to_remove = ["busybox", "reboot", "mount", "umount", "modinfo"] cmds = [c for c in cmds if c not in to_remove] for c in cmds: os.symlink("busybox", os.path.join(dest_dir, "usr/bin", c)) @@ -382,7 +384,7 @@ def install_busybox(dest_dir): def install_misc(dest_dir): # dmsetup rules - rules = check_output(["dpkg", "-L", "dmsetup"]).decode("utf-8").splitlines() + rules = package_files(["dmsetup"]) to_include = re.compile(r".*rules.d/") rules = [i for i in rules if to_include.match(i)] install_files(rules, dest_dir)