From ab074d414974182aa5c43b115cd66869fdcc16d9 Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Tue, 27 Jun 2023 16:00:35 +0200 Subject: [PATCH 1/7] blsspec.py: find /boot/loader instead of /boot/entry_token /boot/entry_token does not exist if all kernels are installed in the uki layout Signed-off-by: Andrew Ammerlaan --- ecleankernel/layout/blspec.py | 66 ++++++++++++++++++----------------- test/test_layout_blspec.py | 2 ++ 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/ecleankernel/layout/blspec.py b/ecleankernel/layout/blspec.py index b371c14..301650d 100644 --- a/ecleankernel/layout/blspec.py +++ b/ecleankernel/layout/blspec.py @@ -45,7 +45,7 @@ def __init__(self, for path in ("etc/kernel/entry-token", "etc/machine-id"): try: with open(root / path) as f: - kernel_id = f.read().strip() + self.kernel_id = f.read().strip() break except FileNotFoundError: pass @@ -53,11 +53,12 @@ def __init__(self, raise LayoutNotFound("/etc/machine-id not found") for d in self.potential_dirs: - self.bootdir = root / d / kernel_id - if self.bootdir.is_dir(): + bootloaderdir = root / d / "loader" + if bootloaderdir.is_dir(): + self.blsdir = root / d / self.kernel_id return else: - raise LayoutNotFound(f"/boot/[EFI/]{kernel_id} not found") + raise LayoutNotFound("/boot/[EFI/]loader not found") def find_kernels(self, exclusions: typing.Container[KernelFileType] = [], @@ -72,35 +73,36 @@ def find_kernels(self, # collect from /boot kernels: typing.Dict[str, Kernel] = {} - for ver in os.listdir(self.bootdir): - if ver.startswith('.'): - continue - dir_path = self.bootdir / ver - if dir_path.is_symlink() or not dir_path.is_dir(): - continue - - k = Kernel(ver) - for fn in os.listdir(dir_path): - if fn.startswith('.'): + if self.blsdir.is_dir(): + for ver in os.listdir(self.blsdir): + if ver.startswith('.'): continue - path = dir_path / fn - ftype = self.name_map.get(fn, KernelFileType.MISC) - fobj = GenericFile(path, ftype) - if ftype == KernelFileType.KERNEL: - try: - kobj = KernelImage(path) - except UnrecognizedKernelError as err: - logging.debug( - f"Unrecognized potential kernel image: {err}") - else: - # associate the module directory - k.all_files.extend( - module_dict.get(kobj.internal_version, [])) - fobj = kobj - if ftype not in exclusions: - k.all_files.append(fobj) - k.all_files.append(EmptyDirectory(dir_path)) - kernels[ver] = k + dir_path = self.blsdir / ver + if dir_path.is_symlink() or not dir_path.is_dir(): + continue + + k = Kernel(ver) + for fn in os.listdir(dir_path): + if fn.startswith('.'): + continue + path = dir_path / fn + ftype = self.name_map.get(fn, KernelFileType.MISC) + fobj = GenericFile(path, ftype) + if ftype == KernelFileType.KERNEL: + try: + kobj = KernelImage(path) + except UnrecognizedKernelError as err: + logging.debug( + f"Unrecognized potential kernel image: {err}") + else: + # associate the module directory + k.all_files.extend( + module_dict.get(kobj.internal_version, [])) + fobj = kobj + if ftype not in exclusions: + k.all_files.append(fobj) + k.all_files.append(EmptyDirectory(dir_path)) + kernels[ver] = k # merge unassociated modules into kernel groups for mkv, fobjs in module_dict.items(): diff --git a/test/test_layout_blspec.py b/test/test_layout_blspec.py index 97f0fbc..6641339 100644 --- a/test/test_layout_blspec.py +++ b/test/test_layout_blspec.py @@ -50,6 +50,7 @@ def create_layout(self, f"boot/{subdir}/1.2.2/.hidden-blocker", f"boot/{subdir}/1.2.1/initrd", f"boot/{subdir}/1.2.1/linux", + f"boot/{subdir}/../loader/entries", 'etc/machine-id', 'lib/modules/1.2.4/test.ko', 'lib/modules/1.2.3/test.ko', @@ -104,6 +105,7 @@ def assert_kernels(self, f'boot/{subdir}{self.machine_id}/1.2.1/linux': k121, f'boot/{subdir}{self.machine_id}/1.2.1': k121, f'boot/{subdir}{self.machine_id}': True, + f"boot/{subdir}/loader/entries": True, 'etc/machine-id': True, 'lib/modules/1.2.4/test.ko': k124, 'lib/modules/1.2.4': k124, From 9dcbec2b4cb3d3405bcc674b4224223e92ef955f Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Tue, 27 Jun 2023 16:06:59 +0200 Subject: [PATCH 2/7] blsspec.py: move appending of kernel files into its own function In preparation of adding support for the uki layout Signed-off-by: Andrew Ammerlaan --- ecleankernel/layout/blspec.py | 51 ++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/ecleankernel/layout/blspec.py b/ecleankernel/layout/blspec.py index 301650d..3747d64 100644 --- a/ecleankernel/layout/blspec.py +++ b/ecleankernel/layout/blspec.py @@ -60,6 +60,33 @@ def __init__(self, else: raise LayoutNotFound("/boot/[EFI/]loader not found") + def append_kernel_files(self, + ftype: KernelFileType, + path: Path, + k: Kernel, + ver: str, + module_dict: dict, + exclusions: typing.Container[KernelFileType] = [], + ) -> Kernel: + fobj = GenericFile(path, ftype) + + if ftype == KernelFileType.KERNEL: + try: + kobj = KernelImage(path) + except UnrecognizedKernelError as err: + logging.debug( + f"Unrecognized potential kernel image: {err}") + else: + fobj = kobj + # associate the module directory + k.all_files.extend( + module_dict.get(kobj.internal_version, [])) + + if ftype not in exclusions: + k.all_files.append(fobj) + + return k + def find_kernels(self, exclusions: typing.Container[KernelFileType] = [], ) -> typing.List[Kernel]: @@ -82,27 +109,15 @@ def find_kernels(self, continue k = Kernel(ver) + for fn in os.listdir(dir_path): if fn.startswith('.'): continue - path = dir_path / fn - ftype = self.name_map.get(fn, KernelFileType.MISC) - fobj = GenericFile(path, ftype) - if ftype == KernelFileType.KERNEL: - try: - kobj = KernelImage(path) - except UnrecognizedKernelError as err: - logging.debug( - f"Unrecognized potential kernel image: {err}") - else: - # associate the module directory - k.all_files.extend( - module_dict.get(kobj.internal_version, [])) - fobj = kobj - if ftype not in exclusions: - k.all_files.append(fobj) - k.all_files.append(EmptyDirectory(dir_path)) - kernels[ver] = k + kernels[ver] = self.append_kernel_files( + self.name_map.get(fn, KernelFileType.MISC), + dir_path / fn, + k, ver, module_dict, exclusions) + kernels[ver].all_files.append(EmptyDirectory(dir_path)) # merge unassociated modules into kernel groups for mkv, fobjs in module_dict.items(): From a23d1ed0382b8fc844c4c714eadc121100ddfab6 Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Tue, 27 Jun 2023 16:12:21 +0200 Subject: [PATCH 3/7] blspec.py: support the type 2 unified kernel image layout Signed-off-by: Andrew Ammerlaan --- ecleankernel/layout/blspec.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ecleankernel/layout/blspec.py b/ecleankernel/layout/blspec.py index 3747d64..cb5d9a9 100644 --- a/ecleankernel/layout/blspec.py +++ b/ecleankernel/layout/blspec.py @@ -55,7 +55,12 @@ def __init__(self, for d in self.potential_dirs: bootloaderdir = root / d / "loader" if bootloaderdir.is_dir(): + # Type 1 entries (linux+initrd) are in + # $BOOT/ENTRY-TOKEN/KERNEL-VERSION/ self.blsdir = root / d / self.kernel_id + # Type 2 entries (uki's) are in + # $BOOT/EFI/Linux/ENTRY-TOKEN-KERNEL-VERSION.efi + self.ukidir = root / d / "EFI" / "Linux" return else: raise LayoutNotFound("/boot/[EFI/]loader not found") @@ -119,6 +124,27 @@ def find_kernels(self, k, ver, module_dict, exclusions) kernels[ver].all_files.append(EmptyDirectory(dir_path)) + # collect from ESP/Linux + if self.ukidir.is_dir(): + for file in os.listdir(self.ukidir): + if (not file.startswith(self.kernel_id) or + not file.endswith(".efi")): + # This file is not an efi file or does not belong to us + continue + ver = file.removeprefix(self.kernel_id + + "-").removesuffix(".efi") + + if ver in kernels: + raise LayoutNotFound( + "Same kernel version installed in bls and uki layout") + + kernels[ver] = self.append_kernel_files( + KernelFileType.KERNEL, + self.ukidir / file, + Kernel(ver), + ver, module_dict, + exclusions) + # merge unassociated modules into kernel groups for mkv, fobjs in module_dict.items(): if any(mkv == k.real_kv for k in kernels.values()): From 8ce056e29f2272b2ce4d141c1db66b62e804e902 Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Fri, 7 Jul 2023 11:30:02 +0200 Subject: [PATCH 4/7] test_layout_blspec.py: add some uki's to the test spec Signed-off-by: Andrew Ammerlaan --- test/test_layout_blspec.py | 251 +++++++++++++++++++++++++------------ 1 file changed, 170 insertions(+), 81 deletions(-) diff --git a/test/test_layout_blspec.py b/test/test_layout_blspec.py index 6641339..ba1734d 100644 --- a/test/test_layout_blspec.py +++ b/test/test_layout_blspec.py @@ -39,19 +39,23 @@ def create_layout(self, entry_token: bool = False, ) -> tempfile.TemporaryDirectory: subdir = 'EFI/' if efi_subdir else '' - subdir += self.entry_token if entry_token else self.machine_id + entry = self.entry_token if entry_token else self.machine_id test_spec = [ - f"boot/{subdir}/1.2.4/initrd", - f"boot/{subdir}/1.2.3/initrd", - f"boot/{subdir}/1.2.3/linux", - f"boot/{subdir}/1.2.3/misc", - f"boot/{subdir}/1.2.2/initrd", - f"boot/{subdir}/1.2.2/linux", - f"boot/{subdir}/1.2.2/.hidden-blocker", - f"boot/{subdir}/1.2.1/initrd", - f"boot/{subdir}/1.2.1/linux", - f"boot/{subdir}/../loader/entries", + f"boot/{subdir}/EFI/Linux/{entry}-1.2.6.efi", + f"boot/{subdir}/EFI/Linux/{entry}-1.2.5.efi", + f"boot/{subdir}/{entry}/1.2.4/initrd", + f"boot/{subdir}/{entry}/1.2.3/initrd", + f"boot/{subdir}/{entry}/1.2.3/linux", + f"boot/{subdir}/{entry}/1.2.3/misc", + f"boot/{subdir}/{entry}/1.2.2/initrd", + f"boot/{subdir}/{entry}/1.2.2/linux", + f"boot/{subdir}/{entry}/1.2.2/.hidden-blocker", + f"boot/{subdir}/{entry}/1.2.1/initrd", + f"boot/{subdir}/{entry}/1.2.1/linux", + f"boot/{subdir}/loader/entries", 'etc/machine-id', + 'lib/modules/1.2.6/test.ko', + 'lib/modules/1.2.5/test.ko', 'lib/modules/1.2.4/test.ko', 'lib/modules/1.2.3/test.ko', 'lib/modules/1.2.2/test.ko', @@ -72,9 +76,13 @@ def create_layout(self, if entry_token: with open(path / "etc/kernel/entry-token", "w") as f: f.write(f"{self.entry_token}\n") - write_bzImage(bootsub / '1.2.3/linux', b'1.2.3 test') - write_bzImage(bootsub / '1.2.2/linux', b'1.2.2 test') - write_bzImage(bootsub / '1.2.1/linux', b'1.2.1 test') + write_bzImage(bootsub / f"EFI/Linux/{entry}-1.2.6.efi", b'1.2.6 test') + write_bzImage(bootsub / f"EFI/Linux/{entry}-1.2.5.efi", b'1.2.5 test') + write_bzImage(bootsub / f"{entry}/1.2.3/linux", b'1.2.3 test') + write_bzImage(bootsub / f"{entry}/1.2.2/linux", b'1.2.2 test') + write_bzImage(bootsub / f"{entry}/1.2.1/linux", b'1.2.1 test') + os.symlink('../../../usr/src/linux', modules / '1.2.6/build') + os.symlink('../../../usr/src/linux', modules / '1.2.5/build') os.symlink('../../../usr/src/linux', modules / '1.2.3/build') os.symlink('../../../usr/src/linux', modules / '1.2.2/build') @@ -83,6 +91,8 @@ def create_layout(self, def assert_kernels(self, root: Path, efi_subdir: bool = False, + k126: bool = True, + k125: bool = True, k124: bool = True, k123: bool = True, k122: bool = True, @@ -91,6 +101,8 @@ def assert_kernels(self, """Assert whether specified kernels were removed or kept""" subdir = 'EFI/' if efi_subdir else '' files = { + f'boot/{subdir}/EFI/Linux/{self.machine_id}-1.2.6.efi': k126, + f'boot/{subdir}/EFI/Linux/{self.machine_id}-1.2.5.efi': k125, f'boot/{subdir}{self.machine_id}/1.2.4/initrd': k124, f'boot/{subdir}{self.machine_id}/1.2.4': k124, f'boot/{subdir}{self.machine_id}/1.2.3/initrd': k123, @@ -107,6 +119,10 @@ def assert_kernels(self, f'boot/{subdir}{self.machine_id}': True, f"boot/{subdir}/loader/entries": True, 'etc/machine-id': True, + 'lib/modules/1.2.6/test.ko': k126, + 'lib/modules/1.2.6': k126, + 'lib/modules/1.2.5/test.ko': k125, + 'lib/modules/1.2.5': k125, 'lib/modules/1.2.4/test.ko': k124, 'lib/modules/1.2.4': k124, 'lib/modules/1.2.3/test.ko': k123, @@ -116,8 +132,8 @@ def assert_kernels(self, 'lib/modules/1.2.1/test.ko': k121, 'lib/modules/1.2.1': k121, 'lib/modules': True, - 'usr/src/linux/Makefile': k123 or k122, - 'usr/src/linux': k123 or k122, + 'usr/src/linux/Makefile': k126 or k125 or k123 or k122, + 'usr/src/linux': k126 or k125 or k123 or k122, 'usr/src': True, } expected_files = [f for f, exp in files.items() if exp] @@ -162,50 +178,66 @@ def test_accept_no_machine_id(self) -> None: def test_find_modules(self) -> None: with self.create_layout() as td: path = Path(td) - boot = path / f'boot/{self.machine_id}' + blspath = path / f"boot/{self.machine_id}" + ukipath = path / "boot/EFI/Linux" modules = path / 'lib/modules' self.assertEqual( sorted(kernel_paths( BlSpecLayout(root=path).find_kernels())), [('1.2.1', - [EmptyDirectory(boot / '1.2.1'), - GenericFile(boot / '1.2.1/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.1/linux'), + [EmptyDirectory(blspath / '1.2.1'), + GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.1/linux'), ModuleDirectory(modules / '1.2.1'), ], '1.2.1'), ('1.2.2', - [EmptyDirectory(boot / '1.2.2'), - GenericFile(boot / '1.2.2/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.2/linux'), + [EmptyDirectory(blspath / '1.2.2'), + GenericFile(blspath / '1.2.2/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.2/linux'), ModuleDirectory(modules / '1.2.2'), GenericFile(modules / '1.2.2/../../../usr/src/linux', KFT.BUILD), ], '1.2.2'), ('1.2.3', - [EmptyDirectory(boot / '1.2.3'), - GenericFile(boot / '1.2.3/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.3/linux'), - GenericFile(boot / '1.2.3/misc', KFT.MISC), + [EmptyDirectory(blspath / '1.2.3'), + GenericFile(blspath / '1.2.3/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.3/linux'), + GenericFile(blspath / '1.2.3/misc', KFT.MISC), ModuleDirectory(modules / '1.2.3'), GenericFile(modules / '1.2.3/../../../usr/src/linux', KFT.BUILD), ], '1.2.3'), ('1.2.4', - [EmptyDirectory(boot / '1.2.4'), - GenericFile(boot / '1.2.4/initrd', KFT.INITRAMFS), + [EmptyDirectory(blspath / '1.2.4'), + GenericFile(blspath / '1.2.4/initrd', KFT.INITRAMFS), ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), + ModuleDirectory(modules / '1.2.5'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), + ('1.2.6', + [KernelImage(ukipath / f"{self.machine_id}-1.2.6.efi"), + ModuleDirectory(modules / '1.2.6'), + GenericFile(modules / '1.2.6/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.6'), ]) def test_exclude_misc(self) -> None: with self.create_layout() as td: path = Path(td) - boot = path / f'boot/{self.machine_id}' + blspath = path / f"boot/{self.machine_id}" + ukipath = path / "boot/EFI/Linux" modules = path / 'lib/modules' self.assertEqual( @@ -213,42 +245,57 @@ def test_exclude_misc(self) -> None: BlSpecLayout(root=path).find_kernels( exclusions=[KFT.MISC]))), [('1.2.1', - [EmptyDirectory(boot / '1.2.1'), - GenericFile(boot / '1.2.1/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.1/linux'), + [EmptyDirectory(blspath / '1.2.1'), + GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.1/linux'), ModuleDirectory(modules / '1.2.1'), ], '1.2.1'), ('1.2.2', - [EmptyDirectory(boot / '1.2.2'), - GenericFile(boot / '1.2.2/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.2/linux'), + [EmptyDirectory(blspath / '1.2.2'), + GenericFile(blspath / '1.2.2/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.2/linux'), ModuleDirectory(modules / '1.2.2'), GenericFile(modules / '1.2.2/../../../usr/src/linux', KFT.BUILD), ], '1.2.2'), ('1.2.3', - [EmptyDirectory(boot / '1.2.3'), - GenericFile(boot / '1.2.3/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.3/linux'), + [EmptyDirectory(blspath / '1.2.3'), + GenericFile(blspath / '1.2.3/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.3/linux'), ModuleDirectory(modules / '1.2.3'), GenericFile(modules / '1.2.3/../../../usr/src/linux', KFT.BUILD), ], '1.2.3'), ('1.2.4', - [EmptyDirectory(boot / '1.2.4'), - GenericFile(boot / '1.2.4/initrd', KFT.INITRAMFS), + [EmptyDirectory(blspath / '1.2.4'), + GenericFile(blspath / '1.2.4/initrd', KFT.INITRAMFS), ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), + ModuleDirectory(modules / '1.2.5'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), + ('1.2.6', + [KernelImage(ukipath / f"{self.machine_id}-1.2.6.efi"), + ModuleDirectory(modules / '1.2.6'), + GenericFile(modules / '1.2.6/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.6'), ]) def test_exclude_modules(self) -> None: with self.create_layout() as td: path = Path(td) - boot = path / f'boot/{self.machine_id}' + blspath = path / f"boot/{self.machine_id}" + ukipath = path / "boot/EFI/Linux" modules = path / 'lib/modules' self.assertEqual( @@ -256,39 +303,52 @@ def test_exclude_modules(self) -> None: BlSpecLayout(root=path).find_kernels( exclusions=[KFT.MODULES]))), [('1.2.1', - [EmptyDirectory(boot / '1.2.1'), - GenericFile(boot / '1.2.1/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.1/linux'), + [EmptyDirectory(blspath / '1.2.1'), + GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.1/linux'), ], '1.2.1'), ('1.2.2', - [EmptyDirectory(boot / '1.2.2'), - GenericFile(boot / '1.2.2/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.2/linux'), + [EmptyDirectory(blspath / '1.2.2'), + GenericFile(blspath / '1.2.2/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.2/linux'), GenericFile(modules / '1.2.2/../../../usr/src/linux', KFT.BUILD), ], '1.2.2'), ('1.2.3', - [EmptyDirectory(boot / '1.2.3'), - GenericFile(boot / '1.2.3/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.3/linux'), - GenericFile(boot / '1.2.3/misc', KFT.MISC), + [EmptyDirectory(blspath / '1.2.3'), + GenericFile(blspath / '1.2.3/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.3/linux'), + GenericFile(blspath / '1.2.3/misc', KFT.MISC), GenericFile(modules / '1.2.3/../../../usr/src/linux', KFT.BUILD), ], '1.2.3'), ('1.2.4', - [EmptyDirectory(boot / '1.2.4'), - GenericFile(boot / '1.2.4/initrd', KFT.INITRAMFS), + [EmptyDirectory(blspath / '1.2.4'), + GenericFile(blspath / '1.2.4/initrd', KFT.INITRAMFS), ], None), + ('1.2.5', + [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), + ('1.2.6', + [KernelImage(ukipath / f"{self.machine_id}-1.2.6.efi"), + GenericFile(modules / '1.2.6/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.6'), ]) def test_exclude_build(self) -> None: with self.create_layout() as td: path = Path(td) - boot = path / f'boot/{self.machine_id}' + blspath = path / f"boot/{self.machine_id}" + ukipath = path / "boot/EFI/Linux" modules = path / 'lib/modules' self.assertEqual( @@ -296,76 +356,101 @@ def test_exclude_build(self) -> None: BlSpecLayout(root=path).find_kernels( exclusions=[KFT.BUILD]))), [('1.2.1', - [EmptyDirectory(boot / '1.2.1'), - GenericFile(boot / '1.2.1/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.1/linux'), + [EmptyDirectory(blspath / '1.2.1'), + GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.1/linux'), ModuleDirectory(modules / '1.2.1'), ], '1.2.1'), ('1.2.2', - [EmptyDirectory(boot / '1.2.2'), - GenericFile(boot / '1.2.2/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.2/linux'), + [EmptyDirectory(blspath / '1.2.2'), + GenericFile(blspath / '1.2.2/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.2/linux'), ModuleDirectory(modules / '1.2.2'), ], '1.2.2'), ('1.2.3', - [EmptyDirectory(boot / '1.2.3'), - GenericFile(boot / '1.2.3/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.3/linux'), - GenericFile(boot / '1.2.3/misc', KFT.MISC), + [EmptyDirectory(blspath / '1.2.3'), + GenericFile(blspath / '1.2.3/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.3/linux'), + GenericFile(blspath / '1.2.3/misc', KFT.MISC), ModuleDirectory(modules / '1.2.3'), ], '1.2.3'), ('1.2.4', - [EmptyDirectory(boot / '1.2.4'), - GenericFile(boot / '1.2.4/initrd', KFT.INITRAMFS), + [EmptyDirectory(blspath / '1.2.4'), + GenericFile(blspath / '1.2.4/initrd', KFT.INITRAMFS), ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), + ModuleDirectory(modules / '1.2.5'), + ], + '1.2.5'), + ('1.2.6', + [KernelImage(ukipath / f"{self.machine_id}-1.2.6.efi"), + ModuleDirectory(modules / '1.2.6'), + ], + '1.2.6'), ]) def test_find_modules_EFI(self) -> None: with self.create_layout(efi_subdir=True) as td: path = Path(td) - boot = path / f'boot/EFI/{self.machine_id}' + blspath = path / f"boot/EFI/{self.machine_id}" + ukipath = path / "boot/EFI/EFI/Linux" modules = path / 'lib/modules' self.assertEqual( sorted(kernel_paths( BlSpecLayout(root=path).find_kernels())), [('1.2.1', - [EmptyDirectory(boot / '1.2.1'), - GenericFile(boot / '1.2.1/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.1/linux'), + [EmptyDirectory(blspath / '1.2.1'), + GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.1/linux'), ModuleDirectory(modules / '1.2.1'), ], '1.2.1'), ('1.2.2', - [EmptyDirectory(boot / '1.2.2'), - GenericFile(boot / '1.2.2/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.2/linux'), + [EmptyDirectory(blspath / '1.2.2'), + GenericFile(blspath / '1.2.2/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.2/linux'), ModuleDirectory(modules / '1.2.2'), GenericFile(modules / '1.2.2/../../../usr/src/linux', KFT.BUILD), ], '1.2.2'), ('1.2.3', - [EmptyDirectory(boot / '1.2.3'), - GenericFile(boot / '1.2.3/initrd', KFT.INITRAMFS), - KernelImage(boot / '1.2.3/linux'), - GenericFile(boot / '1.2.3/misc', KFT.MISC), + [EmptyDirectory(blspath / '1.2.3'), + GenericFile(blspath / '1.2.3/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.3/linux'), + GenericFile(blspath / '1.2.3/misc', KFT.MISC), ModuleDirectory(modules / '1.2.3'), GenericFile(modules / '1.2.3/../../../usr/src/linux', KFT.BUILD), ], '1.2.3'), ('1.2.4', - [EmptyDirectory(boot / '1.2.4'), - GenericFile(boot / '1.2.4/initrd', KFT.INITRAMFS), + [EmptyDirectory(blspath / '1.2.4'), + GenericFile(blspath / '1.2.4/initrd', KFT.INITRAMFS), ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), + ModuleDirectory(modules / '1.2.5'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), + ('1.2.6', + [KernelImage(ukipath / f"{self.machine_id}-1.2.6.efi"), + ModuleDirectory(modules / '1.2.6'), + GenericFile(modules / '1.2.6/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.6'), ]) def test_main_remove(self) -> None: @@ -395,7 +480,9 @@ def test_main_remove_all(self) -> None: k121=False, k122=False, k123=False, - k124=False) + k124=False, + k125=False, + k126=False) def test_main_remove_all_pretend(self) -> None: with self.create_layout() as td: @@ -413,6 +500,8 @@ def test_main_remove_n2(self) -> None: 0) self.assert_kernels(Path(td), k121=False, + k122=False, + k123=False, k124=False) def test_main_remove_n2_pretend(self) -> None: From 5d52d9195b1de6764bd481188b72dabe2ccbc2eb Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Mon, 10 Jul 2023 11:16:09 +0200 Subject: [PATCH 5/7] kernel.py: add layout attribute Signed-off-by: Andrew Ammerlaan --- ecleankernel/__main__.py | 8 ++++---- ecleankernel/kernel.py | 13 +++++++++---- ecleankernel/layout/blspec.py | 4 ++-- test/test_layout_std.py | 8 ++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/ecleankernel/__main__.py b/ecleankernel/__main__.py index 307e2a6..9aebd56 100644 --- a/ecleankernel/__main__.py +++ b/ecleankernel/__main__.py @@ -241,7 +241,7 @@ def main(argv: typing.List[str]) -> int: key=sorter.key, reverse=True) for k in ordered: - print(f'{k.version} [{k.real_kv}]') + print(f'{k.layout} {k.version} [{k.real_kv}]') for kf in sorted(k.all_files, key=lambda f: f.path): print(f'- {kf.ftype.value}: {kf.path}') ts = time.strftime("%Y-%m-%d %H:%M:%S", @@ -292,7 +292,7 @@ def main(argv: typing.List[str]) -> int: get_removable_files(removals, kernels)) for k, reason, files in file_removals: - print(f'- {k.version}: {", ".join(reason)}') + print(f'- {k.layout} {k.version}: {", ".join(reason)}') for kf in k.all_files: if kf.path in files: sign = '-' @@ -315,7 +315,7 @@ def main(argv: typing.List[str]) -> int: for k, reason in list(removals.items()): while args.ask: - ans = input(f'Remove {k.version} ' + ans = input(f'Remove {k.layout} {k.version} ' f'({", ".join(reason)})? [Yes/No]' ).lower() if 'yes'.startswith(ans): @@ -330,7 +330,7 @@ def main(argv: typing.List[str]) -> int: get_removable_files(removals, kernels)) for k, reason, files in file_removals: - print(f'* Removing kernel {k.version} ' + print(f'* Removing kernel {k.layout} {k.version} ' f'({", ".join(reason)})') if has_kernel_install: diff --git a/ecleankernel/kernel.py b/ecleankernel/kernel.py index 25a7e81..1d6b49d 100644 --- a/ecleankernel/kernel.py +++ b/ecleankernel/kernel.py @@ -36,11 +36,14 @@ class Kernel(object): all_files: typing.List[GenericFile] version: str + layout: str def __init__(self, - version: str + version: str, + layout: str = "other", ) -> None: self.all_files = [] + self.layout = layout self.version = version @property @@ -68,10 +71,12 @@ def __eq__(self, if not isinstance(other, Kernel): return False return (self.version == other.version - and self.all_files == other.all_files) + and self.all_files == other.all_files + and self.layout == other.layout) def __hash__(self) -> int: - return hash(self.version) + return hash((self.version, self.layout)) def __repr__(self) -> str: - return f'Kernel({repr(self.version)})' + return f'Kernel(version={repr(self.version)}, ' \ + f'layout={repr(self.layout)})' diff --git a/ecleankernel/layout/blspec.py b/ecleankernel/layout/blspec.py index cb5d9a9..9ee7ced 100644 --- a/ecleankernel/layout/blspec.py +++ b/ecleankernel/layout/blspec.py @@ -113,7 +113,7 @@ def find_kernels(self, if dir_path.is_symlink() or not dir_path.is_dir(): continue - k = Kernel(ver) + k = Kernel(ver, layout="bls") for fn in os.listdir(dir_path): if fn.startswith('.'): @@ -141,7 +141,7 @@ def find_kernels(self, kernels[ver] = self.append_kernel_files( KernelFileType.KERNEL, self.ukidir / file, - Kernel(ver), + Kernel(ver, layout="uki"), ver, module_dict, exclusions) diff --git a/test/test_layout_std.py b/test/test_layout_std.py index 7761d34..43dc5d6 100644 --- a/test/test_layout_std.py +++ b/test/test_layout_std.py @@ -330,24 +330,24 @@ def test_main_list_kernels(self, val = (x for x in sout.getvalue().splitlines() if not x.startswith('- last modified:')) self.assertEqual('\n'.join(val), f''' -1.2.4 [None] +other 1.2.4 [None] - config: {td}/boot/config-1.2.4 - modules: {td}/lib/modules/1.2.4 -1.2.3 [1.2.3] +other 1.2.3 [1.2.3] - systemmap: {td}/boot/System.map-1.2.3 - config: {td}/boot/config-1.2.3 - initramfs: {td}/boot/initrd-1.2.3.img - vmlinuz: {td}/boot/vmlinuz-1.2.3 - modules: {td}/lib/modules/1.2.3 - build: {td}/lib/modules/1.2.3/../../../usr/src/linux -1.2.3.old [1.2.3] +other 1.2.3.old [1.2.3] - systemmap: {td}/boot/System.map-1.2.3.old - config: {td}/boot/config-1.2.3.old - initramfs: {td}/boot/initrd-1.2.3.img.old - vmlinuz: {td}/boot/vmlinuz-1.2.3.old - modules: {td}/lib/modules/1.2.3 - build: {td}/lib/modules/1.2.3/../../../usr/src/linux -1.2.2 [1.2.2] +other 1.2.2 [1.2.2] - systemmap: {td}/boot/System.map-1.2.2 - vmlinuz: {td}/boot/vmlinuz-1.2.2 - modules: {td}/lib/modules/1.2.2 From 664464709c153f8994f2a1a156dad495b07a4694 Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Mon, 10 Jul 2023 12:14:51 +0200 Subject: [PATCH 6/7] blsspec.py: support same kernel version in both layouts Signed-off-by: Andrew Ammerlaan --- ecleankernel/layout/blspec.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/ecleankernel/layout/blspec.py b/ecleankernel/layout/blspec.py index 9ee7ced..0926901 100644 --- a/ecleankernel/layout/blspec.py +++ b/ecleankernel/layout/blspec.py @@ -104,7 +104,7 @@ def find_kernels(self, module_directory=self.root / 'lib/modules') # collect from /boot - kernels: typing.Dict[str, Kernel] = {} + kernels: typing.Dict[typing.Tuple[str, str], Kernel] = {} if self.blsdir.is_dir(): for ver in os.listdir(self.blsdir): if ver.startswith('.'): @@ -118,11 +118,12 @@ def find_kernels(self, for fn in os.listdir(dir_path): if fn.startswith('.'): continue - kernels[ver] = self.append_kernel_files( + kernels[(ver, "bls")] = self.append_kernel_files( self.name_map.get(fn, KernelFileType.MISC), dir_path / fn, k, ver, module_dict, exclusions) - kernels[ver].all_files.append(EmptyDirectory(dir_path)) + kernels[(ver, "bls")].all_files.append( + EmptyDirectory(dir_path)) # collect from ESP/Linux if self.ukidir.is_dir(): @@ -134,11 +135,7 @@ def find_kernels(self, ver = file.removeprefix(self.kernel_id + "-").removesuffix(".efi") - if ver in kernels: - raise LayoutNotFound( - "Same kernel version installed in bls and uki layout") - - kernels[ver] = self.append_kernel_files( + kernels[(ver, "uki")] = self.append_kernel_files( KernelFileType.KERNEL, self.ukidir / file, Kernel(ver, layout="uki"), @@ -149,6 +146,19 @@ def find_kernels(self, for mkv, fobjs in module_dict.items(): if any(mkv == k.real_kv for k in kernels.values()): continue - kernels.setdefault(mkv, Kernel(mkv)).all_files.extend(fobjs) + # this mkv does not have a corresponding kernel + # with same real_kv in kernels + match_found = False + for (ver, layout), k in kernels.items(): + if ver == mkv: + # extend existing entry + k.all_files.extend(fobjs) + match_found = True + if not match_found: + # no real_kv for this mkv, no existing entry in kernels + layout = "modules-only" + kernels.setdefault((mkv, layout), + Kernel(mkv, layout=layout) + ).all_files.extend(fobjs) return list(kernels.values()) From b412146733f5583f6774346b498197c3a643b10d Mon Sep 17 00:00:00 2001 From: Andrew Ammerlaan Date: Mon, 10 Jul 2023 13:07:09 +0200 Subject: [PATCH 7/7] test_layout_blsspec.py: version 1.2.5 in both layouts Signed-off-by: Andrew Ammerlaan --- test/test_layout_blspec.py | 73 ++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/test/test_layout_blspec.py b/test/test_layout_blspec.py index ba1734d..e435512 100644 --- a/test/test_layout_blspec.py +++ b/test/test_layout_blspec.py @@ -43,6 +43,8 @@ def create_layout(self, test_spec = [ f"boot/{subdir}/EFI/Linux/{entry}-1.2.6.efi", f"boot/{subdir}/EFI/Linux/{entry}-1.2.5.efi", + f"boot/{subdir}/{entry}/1.2.5/initrd", + f"boot/{subdir}/{entry}/1.2.5/linux", f"boot/{subdir}/{entry}/1.2.4/initrd", f"boot/{subdir}/{entry}/1.2.3/initrd", f"boot/{subdir}/{entry}/1.2.3/linux", @@ -78,6 +80,7 @@ def create_layout(self, f.write(f"{self.entry_token}\n") write_bzImage(bootsub / f"EFI/Linux/{entry}-1.2.6.efi", b'1.2.6 test') write_bzImage(bootsub / f"EFI/Linux/{entry}-1.2.5.efi", b'1.2.5 test') + write_bzImage(bootsub / f"{entry}/1.2.5/linux", b'1.2.5 test') write_bzImage(bootsub / f"{entry}/1.2.3/linux", b'1.2.3 test') write_bzImage(bootsub / f"{entry}/1.2.2/linux", b'1.2.2 test') write_bzImage(bootsub / f"{entry}/1.2.1/linux", b'1.2.1 test') @@ -103,6 +106,9 @@ def assert_kernels(self, files = { f'boot/{subdir}/EFI/Linux/{self.machine_id}-1.2.6.efi': k126, f'boot/{subdir}/EFI/Linux/{self.machine_id}-1.2.5.efi': k125, + f'boot/{subdir}{self.machine_id}/1.2.5/initrd': k125, + f'boot/{subdir}{self.machine_id}/1.2.5/linux': k125, + f'boot/{subdir}{self.machine_id}/1.2.5': k125, f'boot/{subdir}{self.machine_id}/1.2.4/initrd': k124, f'boot/{subdir}{self.machine_id}/1.2.4': k124, f'boot/{subdir}{self.machine_id}/1.2.3/initrd': k123, @@ -184,7 +190,8 @@ def test_find_modules(self) -> None: self.assertEqual( sorted(kernel_paths( - BlSpecLayout(root=path).find_kernels())), + BlSpecLayout(root=path).find_kernels()), + key=lambda ver: ver[0]), [('1.2.1', [EmptyDirectory(blspath / '1.2.1'), GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), @@ -217,6 +224,15 @@ def test_find_modules(self) -> None: ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [EmptyDirectory(blspath / '1.2.5'), + GenericFile(blspath / '1.2.5/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.5/linux'), + ModuleDirectory(modules / '1.2.5'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), ('1.2.5', [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), ModuleDirectory(modules / '1.2.5'), @@ -243,7 +259,8 @@ def test_exclude_misc(self) -> None: self.assertEqual( sorted(kernel_paths( BlSpecLayout(root=path).find_kernels( - exclusions=[KFT.MISC]))), + exclusions=[KFT.MISC])), + key=lambda ver: ver[0]), [('1.2.1', [EmptyDirectory(blspath / '1.2.1'), GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), @@ -275,6 +292,15 @@ def test_exclude_misc(self) -> None: ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [EmptyDirectory(blspath / '1.2.5'), + GenericFile(blspath / '1.2.5/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.5/linux'), + ModuleDirectory(modules / '1.2.5'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), ('1.2.5', [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), ModuleDirectory(modules / '1.2.5'), @@ -301,7 +327,8 @@ def test_exclude_modules(self) -> None: self.assertEqual( sorted(kernel_paths( BlSpecLayout(root=path).find_kernels( - exclusions=[KFT.MODULES]))), + exclusions=[KFT.MODULES])), + key=lambda ver: ver[0]), [('1.2.1', [EmptyDirectory(blspath / '1.2.1'), GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), @@ -330,6 +357,14 @@ def test_exclude_modules(self) -> None: GenericFile(blspath / '1.2.4/initrd', KFT.INITRAMFS), ], None), + ('1.2.5', + [EmptyDirectory(blspath / '1.2.5'), + GenericFile(blspath / '1.2.5/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.5/linux'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), ('1.2.5', [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), GenericFile(modules / '1.2.5/../../../usr/src/linux', @@ -354,7 +389,8 @@ def test_exclude_build(self) -> None: self.assertEqual( sorted(kernel_paths( BlSpecLayout(root=path).find_kernels( - exclusions=[KFT.BUILD]))), + exclusions=[KFT.BUILD])), + key=lambda ver: ver[0]), [('1.2.1', [EmptyDirectory(blspath / '1.2.1'), GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), @@ -383,6 +419,13 @@ def test_exclude_build(self) -> None: ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [EmptyDirectory(blspath / '1.2.5'), + GenericFile(blspath / '1.2.5/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.5/linux'), + ModuleDirectory(modules / '1.2.5'), + ], + '1.2.5'), ('1.2.5', [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), ModuleDirectory(modules / '1.2.5'), @@ -404,7 +447,8 @@ def test_find_modules_EFI(self) -> None: self.assertEqual( sorted(kernel_paths( - BlSpecLayout(root=path).find_kernels())), + BlSpecLayout(root=path).find_kernels()), + key=lambda ver: ver[0]), [('1.2.1', [EmptyDirectory(blspath / '1.2.1'), GenericFile(blspath / '1.2.1/initrd', KFT.INITRAMFS), @@ -437,6 +481,15 @@ def test_find_modules_EFI(self) -> None: ModuleDirectory(modules / '1.2.4'), ], None), + ('1.2.5', + [EmptyDirectory(blspath / '1.2.5'), + GenericFile(blspath / '1.2.5/initrd', KFT.INITRAMFS), + KernelImage(blspath / '1.2.5/linux'), + ModuleDirectory(modules / '1.2.5'), + GenericFile(modules / '1.2.5/../../../usr/src/linux', + KFT.BUILD), + ], + '1.2.5'), ('1.2.5', [KernelImage(ukipath / f"{self.machine_id}-1.2.5.efi"), ModuleDirectory(modules / '1.2.5'), @@ -492,10 +545,10 @@ def test_main_remove_all_pretend(self) -> None: 0) self.assert_kernels(Path(td)) - def test_main_remove_n2(self) -> None: + def test_main_remove_n3(self) -> None: with self.create_layout() as td: self.assertEqual( - main(['--destructive', '-n', '2', + main(['--destructive', '-n', '3', '--root', td, '--debug', '--no-mount']), 0) self.assert_kernels(Path(td), @@ -504,10 +557,10 @@ def test_main_remove_n2(self) -> None: k123=False, k124=False) - def test_main_remove_n2_pretend(self) -> None: + def test_main_remove_n3_pretend(self) -> None: with self.create_layout() as td: self.assertEqual( - main(['--destructive', '-n', '2', '--pretend', + main(['--destructive', '-n', '3', '--pretend', '--root', td, '--debug', '--no-mount']), 0) self.assert_kernels(Path(td)) @@ -515,6 +568,6 @@ def test_main_remove_n2_pretend(self) -> None: def test_wrong_layout_std(self) -> None: with self.create_layout() as td: with self.assertRaises(SystemError): - main(['--destructive', '-n', '2', '--pretend', + main(['--destructive', '-n', '3', '--pretend', '--layout', 'std', '--root', td, '--debug', '--no-mount'])