many: when building initramfs, pull files from the run system

bin/ubuntu-core-initramfs

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 import argparse
 import os
-import subprocess
+from subprocess import run, check_call, check_output
 import tempfile
 import pathlib
 import platform
@@ -259,7 +259,7 @@ def add_modules_from_file(dest_d, kernel_root, modules_d, fw_d, conf_file, db,
                     if not mod.builtin:
                         db.mark_installed(module, conf_file)
 
-                subprocess.check_call(
+                check_call(
                     [
                         "/usr/lib/dracut/dracut-install",
                         "-D",
@@ -276,7 +276,194 @@ def add_modules_from_file(dest_d, kernel_root, modules_d, fw_d, conf_file, db,
                 )
 
 
+def install_files(files, dest_dir):
+    proc_env = os.environ.copy()
+    proc_env["LD_PRELOAD"] = ""
+    check_call(
+        [
+            "/usr/lib/dracut/dracut-install",
+            "-D", dest_dir,
+            "--ldd", "--all",
+        ] + files,
+        env=proc_env)
+
+
+# This is useful if the destination path inside dest_dir is different
+# from the current file path.
+def install_file_to_path(file, dest_dir, dest_path):
+    proc_env = os.environ.copy()
+    proc_env["LD_PRELOAD"] = ""
+    check_call(
+        [
+            "/usr/lib/dracut/dracut-install",
+            "-D", dest_dir,
+            "--ldd", file, dest_path,
+        ],
+        env=proc_env)
+
+
+# Returns as a list the files contained in a list of deb packages.
+def package_files(pkgs):
+    out = check_output(["dpkg", "-L"] + pkgs)
+    return out.decode("utf-8").splitlines()
+
+
+def install_systemd_files(dest_dir):
+    # Build list of files and directories
+
+    lines = package_files(["systemd", "systemd-sysv"])
+    # From systemd, we pull
+    # * units configuration
+    # * Executables
+    # * Module load options
+    # * Configuration of kernel parameters
+    # * udev rules
+    # * Configuration for systemd-tmpfiles
+    # TODO: some of this can be cleaned up
+    to_include = re.compile(r"^/lib/systemd/system|"
+                            r".*/modprobe\.d/|"
+                            r".*/sysctl\.d/|"
+                            r".*/rules\.d/|"
+                            r".*/tmpfiles\.d/|"
+                            r".*bin/|"
+                            r".*sbin/"
+                            )
+    files = [i for i in lines if to_include.match(i)]
+
+    lines = package_files(["udev"])
+    # From udev, we pull
+    # * Executables
+    # * systemd configuration (units, tmpfiles)
+    # * udev rules
+    # * hwdb
+    # TODO: some of this can be cleaned up
+    to_include = re.compile(r".*bin/|"
+                            r".*lib/|"
+                            r".*rules\.d/"
+                            )
+    files += [i for i in lines if to_include.match(i)]
+
+    files.append("/var/lib/systemd/")
+
+    # Filter out some units we don't want
+    to_remove = re.compile(".*systemd-gpt-auto-generator|"
+                           ".*proc-sys-fs-binfmt_misc.automount|"
+                           ".*systemd-pcrphase.*")
+    filtered = [i for i in files if not to_remove.match(i)]
+    # Install
+    install_files(filtered, dest_dir)
+
+    # Local modifications
+    # This hack should be removed with PR#113
+    check_call([r"sed -i '/^After=/"
+                r"{;s, *plymouth-start[.]service *, ,;/"
+                r"^After= *$/d;}' "
+                + os.path.join(dest_dir,
+                               "usr/lib/systemd/system/systemd-ask-password-*")],
+               shell=True)
+    # Generate hw database (/usr/lib/udev/hwdb.bin) for udev and
+    # remove redundant definitions after that.
+    check_call(["systemd-hwdb", "--root", dest_dir,
+                "update", "--usr", "--strict"])
+    shutil.rmtree(os.path.join(dest_dir, "usr/lib/udev/hwdb.d"))
+
+
+def install_busybox(dest_dir):
+    install_file_to_path("/usr/lib/initramfs-tools/bin/busybox", dest_dir,
+                         "usr/bin/busybox")
+    # Create links to commands
+    bb_cmd = os.path.join(dest_dir, "usr/bin/busybox")
+    out = check_output([bb_cmd, "--list-long"])
+    cmds = out.decode("utf-8").splitlines()
+    # Remove commands we do not want
+    to_remove = ["busybox", "reboot", "mount", "umount", "modinfo"]
+    cmds = [c for c in cmds if c not in to_remove]
+    for c in cmds:
+        os.symlink("busybox", os.path.join(dest_dir, "usr/bin", c))
+
+
+def install_misc(dest_dir):
+    # dmsetup rules
+    rules = package_files(["dmsetup"])
+    to_include = re.compile(r".*rules.d/")
+    rules = [i for i in rules if to_include.match(i)]
+    install_files(rules, dest_dir)
+
+    # Other needed stuff
+    out = check_output(["dpkg-architecture", "-q",
+                        "DEB_HOST_MULTIARCH"]).decode("utf-8")
+    deb_arch = out.splitlines()[0]
+    files = [
+        "/usr/bin/kmod",
+        "/usr/bin/mount",
+        "/usr/sbin/sulogin",
+        "/usr/bin/tar",
+        "/usr/lib/" + deb_arch + "/libgcc_s.so.1",
+        "/usr/lib/systemd/system/dbus.service",
+        "/usr/lib/systemd/system/dbus.socket",
+        "/usr/lib/systemd/system/plymouth-start.service",
+        "/usr/lib/systemd/system/plymouth-switch-root.service",
+        "/usr/lib/systemd/systemd-bootchart",
+        "/usr/sbin/cryptsetup",
+        "/usr/sbin/dmsetup",
+        "/usr/sbin/e2fsck",
+        "/usr/sbin/fsck",
+        "/usr/bin/umount",
+        "/usr/sbin/fsck.vfat",
+        "/usr/sbin/fsck.vfat",
+        "/usr/sbin/mkfs.ext4",
+        "/usr/sbin/mkfs.vfat",
+        "/usr/sbin/sfdisk",
+        "/usr/bin/dbus-daemon",
+        "/usr/bin/mountpoint",
+        "/usr/bin/partx",
+        "/usr/bin/plymouth",
+        "/usr/bin/unsquashfs",
+        "/usr/lib/" + deb_arch + "/plymouth/label-ft.so",
+        "/usr/lib/" + deb_arch + "/plymouth/script.so",
+        "/usr/lib/" + deb_arch + "/plymouth/two-step.so",
+        "/usr/sbin/depmod",
+        "/usr/sbin/insmod",
+        "/usr/sbin/lsmod",
+        "/usr/sbin/modinfo",
+        "/usr/sbin/modprobe",
+        "/usr/sbin/plymouthd",
+        "/usr/sbin/rmmod",
+        "/usr/share/dbus-1/system.conf",
+        "/usr/share/libdrm/amdgpu.ids",
+    ]
+    files += glob.glob("/lib/" + deb_arch + "/libnss_compat.so.*")
+    files += glob.glob("/lib/" + deb_arch + "/libnss_files.so.*")
+    files += glob.glob("/usr/lib/" + deb_arch + "/plymouth/renderers/*.so")
+    files += glob.glob("/usr/share/plymouth/themes/bgrt/*")
+    files += glob.glob("/usr/share/plymouth/themes/spinner/*")
+    install_files(files, dest_dir)
+    # Links for fsck
+    os.symlink("e2fsck", os.path.join(dest_dir, "usr/sbin", "fsck.ext4"))
+    # Get deps for shared objects that have not the exec bit set and that
+    # are loaded with dlopen (which means that they are not pull by --ldd
+    # option, instead use --resolvelazy).
+    proc_env = os.environ.copy()
+    proc_env["LD_PRELOAD"] = ""
+    to_resolve = [
+        "/usr/lib/" + deb_arch + "/plymouth/label-ft.so",
+        "/usr/lib/" + deb_arch + "/plymouth/script.so",
+        "/usr/lib/" + deb_arch + "/plymouth/two-step.so",
+    ]
+    to_resolve += glob.glob("/usr/lib/" + deb_arch + "/plymouth/renderers/*.so")
+    check_call(
+        [
+            "/usr/lib/dracut/dracut-install",
+            "-D", dest_dir,
+            "--resolvelazy",
+        ] + to_resolve,
+        env=proc_env)
+    # Build ld cache
+    check_call(["ldconfig", "-r", dest_dir])
+
+
 def create_initrd(parser, args):
+    # TODO generate microcode instead of shipping in debian package
     rootfs = "/"
     if not args.kerneldir:
         args.kerneldir = "/lib/modules/%s" % args.kernelver
@@ -293,49 +480,48 @@ def create_initrd(parser, args):
         modules = os.path.join(kernel_root, "usr", "lib", "modules")
         os.makedirs(modules, exist_ok=True)
         modules = os.path.join(modules, args.kernelver)
-        subprocess.check_call(["cp", "-ar", args.kerneldir, modules])
+        check_call(["cp", "-ar", args.kerneldir, modules])
 
         firmware = os.path.join(kernel_root, "usr", "lib", "firmware")
-        subprocess.check_call(["cp", "-ar", args.firmwaredir, firmware])
+        check_call(["cp", "-ar", args.firmwaredir, firmware])
 
         db = ModuleDb(modules)
         main = os.path.join(d, "main")
         os.makedirs(main, exist_ok=True)
+        # copy busybox first so we get already the shell interpreter we
+        # want (busybox) instead of dracut-install pulling the systemd
+        # default (dash) when it pulls a shell script later.
+        install_busybox(main)
+        # Copy systemd bits
+        install_systemd_files(main)
+        # Other miscelanea stuff
+        install_misc(main)
         # Copy snapd bits
         snapd_lib = path_join_make_rel_paths(rootfs, "/usr/lib/snapd")
         snapd_files = [os.path.join(snapd_lib, "snap-bootstrap"),
                        os.path.join(snapd_lib, "info"),
                        "/lib/systemd/system/snapd.recovery-chooser-trigger.service"]
-        for snapd_f in snapd_files:
-            subprocess.check_call(
-                [
-                    "/usr/lib/dracut/dracut-install",
-                    "-D", main,
-                    "--ldd", snapd_f,
-                ]
-            )
+        install_files(snapd_files, main)
         # Copy features
         for feature in args.features:
             # Add feature files
             feature_path = os.path.join(args.skeleton, feature)
             if os.path.isdir(feature_path):
-                subprocess.check_call(["cp", "-aT", feature_path, main])
+                check_call(["cp", "-aT", feature_path, main])
             # Add feature kernel modules
             extra_modules = os.path.join(args.skeleton, "modules", feature,
                                          "extra-modules.conf")
             if os.path.exists(extra_modules):
                 add_modules_from_file(main, kernel_root, modules, firmware, extra_modules, db)
 
+        # TODO xnox: fips actually needs additional runtime dependencies than
+        # this, and currently forked in fips PPA. we need to figure out how to
+        # make ubuntu-core-initramfs-fips a reality.
         if "fips" in args.features:
-            subprocess.check_call(
-                [
-                    "/usr/lib/dracut/dracut-install",
-                    "-D", main,
-                    "--ldd", "--all",
+            install_files([
                     "/usr/bin/kcapi-hasher",
                     "/usr/bin/.kcapi-hasher.hmac",
-                ] + glob.glob("/usr/lib/*/.libkcapi.so.*.hmac")
-            )
+            ] + glob.glob("/usr/lib/*/.libkcapi.so.*.hmac"), main)
 
         # Update epoch
         pathlib.Path("%s/main/usr/lib/clock-epoch" % d).touch()
@@ -345,7 +531,7 @@ def create_initrd(parser, args):
                                   warn_discoverable=True)
 
         for modulesf in ["modules.order", "modules.builtin", "modules.builtin.bin", "modules.builtin.modinfo"]:
-            subprocess.check_call(
+            check_call(
                 [
                     "/usr/lib/dracut/dracut-install",
                     "-D",
@@ -355,13 +541,13 @@ def create_initrd(parser, args):
                     os.path.join("usr/lib/modules", args.kernelver, modulesf),
                 ]
             )
-        subprocess.check_call(["depmod", "-a", "-b", main, args.kernelver])
+        check_call(["depmod", "-a", "-b", main, args.kernelver])
         with open(args.output, "wb") as output:
             for early in glob.iglob("%s/early/*.cpio" % args.skeleton):
                 with open(early, "rb") as f:
                     shutil.copyfileobj(f, output)
             output.write(
-                subprocess.run(
+                run(
                     "find . | cpio --create --quiet --format='newc' --owner=0:0 | zstd -1 -T0",
                     cwd=main,
                     capture_output=True,
@@ -387,7 +573,7 @@ def create_efi(parser, args):
 
         if platform.machine() == "aarch64":
             import gzip
-            raw_kernel=tempfile.NamedTemporaryFile(mode='wb')
+            raw_kernel = tempfile.NamedTemporaryFile(mode='wb')
             try:
                 with gzip.open(args.kernel, 'rb') as comp_kernel:
                     shutil.copyfileobj(comp_kernel, raw_kernel)
@@ -424,9 +610,9 @@ def create_efi(parser, args):
             args.stub,
             args.output,
         ]
-    subprocess.check_call(objcopy_cmd)
+    check_call(objcopy_cmd)
     if not args.unsigned:
-        subprocess.check_call(
+        check_call(
             [
                 "sbsign",
                 "--key",
@@ -442,7 +628,7 @@ def create_efi(parser, args):
 
 
 def main():
-    kernelver = subprocess.check_output(
+    kernelver = check_output(
         ["uname", "-r"], universal_newlines=True
     ).strip()
     suffix = {"x86_64": "x64",
@@ -459,7 +645,7 @@ def main():
     efi_parser.add_argument("--stub", help="path to stub")
     if suffix:
         efi_parser.set_defaults(
-            stub="/usr/lib/ubuntu-core-initramfs/efi/linux%s.efi.stub" % suffix
+            stub="/usr/lib/systemd/boot/efi/linux%s.efi.stub" % suffix
         )
     efi_parser.add_argument("--kernel", help="path to kernel", default="/boot/vmlinuz")
     efi_parser.add_argument(
@@ -511,7 +697,7 @@ def main():
         "--output", help="path to output", default="/boot/ubuntu-core-initramfs.img"
     )
     initrd_parser.set_defaults(
-        kernelver=subprocess.check_output(
+        kernelver=check_output(
             ["uname", "-r"], universal_newlines=True
         ).strip()
     )