From 2aec59914031d0f2b59f658243b9b0c49c8e5b13 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Fri, 23 Jun 2023 14:34:40 +0200 Subject: [PATCH 01/32] [tests] Make options_tests faster by calling them on a few plugins only options_tests.py tests -n option which means tens of plugins are redundantly collected. Testing -o option is effectively the same while the sosreport run needs a half time only. The functionality of -n option is tested elsewhere, thus the change does not shrink test coverage. Resolves: #3288 Signed-off-by: Pavel Moravec --- tests/report_tests/options_tests/options_tests.py | 6 +++--- tests/report_tests/options_tests/options_tests_sos.conf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/report_tests/options_tests/options_tests.py b/tests/report_tests/options_tests/options_tests.py index f8e1ee6005..997042ba65 100644 --- a/tests/report_tests/options_tests/options_tests.py +++ b/tests/report_tests/options_tests/options_tests.py @@ -22,8 +22,8 @@ class OptionsFromConfigTest(StageTwoReportTest): def test_case_id_from_config(self): self.assertTrue('8675309' in self.archive) - def test_plugins_skipped_from_config(self): - self.assertPluginNotIncluded(['networking', 'logs']) + def test_plugins_only_from_config(self): + self.assertOnlyPluginsIncluded(['host', 'kernel']) def test_plugopts_logged_from_config(self): self.assertSosLogContains( @@ -41,5 +41,5 @@ def test_plugopts_actually_set(self): def test_effective_options_logged_correctly(self): self.assertSosLogContains( - "effective options now: --batch --case-id 8675309 --plugopts kernel.with-timer=on,kernel.trace=yes --skip-plugins networking,logs" + "effective options now: --batch --case-id 8675309 --only-plugins host,kernel --plugopts kernel.with-timer=on,kernel.trace=yes" ) diff --git a/tests/report_tests/options_tests/options_tests_sos.conf b/tests/report_tests/options_tests/options_tests_sos.conf index c0f641d9f2..fd92d7000a 100644 --- a/tests/report_tests/options_tests/options_tests_sos.conf +++ b/tests/report_tests/options_tests/options_tests_sos.conf @@ -2,7 +2,7 @@ #verbose = 3 [report] -skip-plugins = networking,logs +only-plugins = host,kernel case-id = 8675309 [collect] From 9b5d6ad0883bcd211150b490d28ca6142a0674b5 Mon Sep 17 00:00:00 2001 From: Jose Castillo Date: Thu, 29 Jun 2023 12:34:25 +0100 Subject: [PATCH 02/32] [ceph] Add a new profile for Ceph plugins This patch helps simplify the capture of data from ceph plugins. After the original code was divided in 8 different plugins, it may be a bit confusing which plugins to use when running sos and attempting to capture only ceph data. By selecting the profile, sos will activate only these ceph_* plugins that apply to a specific ceph node. Signed-off-by: Jose Castillo --- sos/report/plugins/ceph_ansible.py | 2 +- sos/report/plugins/ceph_common.py | 2 +- sos/report/plugins/ceph_iscsi.py | 2 +- sos/report/plugins/ceph_mds.py | 2 +- sos/report/plugins/ceph_mgr.py | 2 +- sos/report/plugins/ceph_mon.py | 2 +- sos/report/plugins/ceph_osd.py | 2 +- sos/report/plugins/ceph_rgw.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sos/report/plugins/ceph_ansible.py b/sos/report/plugins/ceph_ansible.py index a84c9c624f..27ff91e549 100644 --- a/sos/report/plugins/ceph_ansible.py +++ b/sos/report/plugins/ceph_ansible.py @@ -14,7 +14,7 @@ class CephAnsible(Plugin, RedHatPlugin, DebianPlugin): short_desc = 'CEPH distributed storage - Ansible installer' plugin_name = 'ceph_ansible' - profiles = ('storage',) + profiles = ('storage', 'ceph') packages = ('ceph-ansible',) diff --git a/sos/report/plugins/ceph_common.py b/sos/report/plugins/ceph_common.py index 105fd8c494..ec5852105c 100644 --- a/sos/report/plugins/ceph_common.py +++ b/sos/report/plugins/ceph_common.py @@ -15,7 +15,7 @@ class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH common' plugin_name = 'ceph_common' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?(mon|rgw|osd).*',) ceph_hostname = gethostname() diff --git a/sos/report/plugins/ceph_iscsi.py b/sos/report/plugins/ceph_iscsi.py index 9614019045..ac6d505c74 100644 --- a/sos/report/plugins/ceph_iscsi.py +++ b/sos/report/plugins/ceph_iscsi.py @@ -14,7 +14,7 @@ class CephISCSI(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = "CEPH iSCSI" plugin_name = "ceph_iscsi" - profiles = ("storage", "virt", "container") + profiles = ("storage", "virt", "container", "ceph") packages = ("ceph-iscsi",) services = ("rbd-target-api", "rbd-target-gw") containers = ("rbd-target-api.*", "rbd-target-gw.*") diff --git a/sos/report/plugins/ceph_mds.py b/sos/report/plugins/ceph_mds.py index da48cb1355..c31e906b2d 100644 --- a/sos/report/plugins/ceph_mds.py +++ b/sos/report/plugins/ceph_mds.py @@ -12,7 +12,7 @@ class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH mds' plugin_name = 'ceph_mds' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?fs.*',) files = ('/var/lib/ceph/mds/',) diff --git a/sos/report/plugins/ceph_mgr.py b/sos/report/plugins/ceph_mgr.py index 6a9aff14a0..f7d4ace332 100644 --- a/sos/report/plugins/ceph_mgr.py +++ b/sos/report/plugins/ceph_mgr.py @@ -35,7 +35,7 @@ class CephMGR(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH mgr' plugin_name = 'ceph_mgr' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') files = ('/var/lib/ceph/mgr/', '/var/lib/ceph/*/mgr*') containers = ('ceph-(.*-)?mgr.*',) diff --git a/sos/report/plugins/ceph_mon.py b/sos/report/plugins/ceph_mon.py index b8dc8b8a4d..d25fe3abbe 100644 --- a/sos/report/plugins/ceph_mon.py +++ b/sos/report/plugins/ceph_mon.py @@ -32,7 +32,7 @@ class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH mon' plugin_name = 'ceph_mon' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') # note: for RHCS 5 / Ceph v16 the containers serve as an enablement trigger # but by default they are not capable of running various ceph commands in # this plugin - the `ceph` binary is functional directly on the host diff --git a/sos/report/plugins/ceph_osd.py b/sos/report/plugins/ceph_osd.py index 4ed4565b63..34a656a333 100644 --- a/sos/report/plugins/ceph_osd.py +++ b/sos/report/plugins/ceph_osd.py @@ -29,7 +29,7 @@ class CephOSD(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH osd' plugin_name = 'ceph_osd' - profiles = ('storage', 'virt', 'container') + profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?osd.*',) files = ('/var/lib/ceph/osd/', '/var/lib/ceph/*/osd*') diff --git a/sos/report/plugins/ceph_rgw.py b/sos/report/plugins/ceph_rgw.py index 79f3c44966..200e30ee33 100644 --- a/sos/report/plugins/ceph_rgw.py +++ b/sos/report/plugins/ceph_rgw.py @@ -14,7 +14,7 @@ class CephRGW(Plugin, RedHatPlugin, UbuntuPlugin): short_desc = 'CEPH rgw' plugin_name = 'ceph_rgw' - profiles = ('storage', 'virt', 'container', 'webserver') + profiles = ('storage', 'virt', 'container', 'webserver', 'ceph') containers = ('ceph-(.*)?rgw.*',) files = ('/var/lib/ceph/radosgw',) From 595497a5e3c31c7a0833877b1e542536dd582261 Mon Sep 17 00:00:00 2001 From: Jose Castillo Date: Thu, 29 Jun 2023 15:50:19 +0100 Subject: [PATCH 03/32] [grub2] Capture /etc/grub2-efi.cfg sos report does not gather the symlink /etc/grub2-efi.cfg when present. If a system is legacy, but has this file, grubby default to using this one first in some cases. Therefore we need to know if it's present or not. Closes: RHBZ#2218563 Signed-off-by: Jose Castillo --- sos/report/plugins/grub2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sos/report/plugins/grub2.py b/sos/report/plugins/grub2.py index d499cdf909..5fb2a96575 100644 --- a/sos/report/plugins/grub2.py +++ b/sos/report/plugins/grub2.py @@ -35,7 +35,8 @@ def setup(self): "/boot/grub2/user.cfg", "/etc/default/grub", "/etc/grub2.cfg", - "/etc/grub.d" + "/etc/grub.d", + "/etc/grub2-efi.cfg" ]) self.add_cmd_output("ls -lanR /boot", tags="ls_boot") From 0345ff4259ea506745ce289e95ecc33aaf2365d9 Mon Sep 17 00:00:00 2001 From: Jake Hunsaker Date: Thu, 29 Jun 2023 15:58:02 -0400 Subject: [PATCH 04/32] [docs] Update maintainer field in man pages Update the maintainer field to be more generic and direct readers to the upstream repo. Closes: #3290 Signed-off-by: Jake Hunsaker --- man/en/sos-clean.1 | 2 +- man/en/sos-collect.1 | 6 +++++- man/en/sos-report.1 | 2 +- man/en/sos.1 | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/man/en/sos-clean.1 b/man/en/sos-clean.1 index e8506eeb0a..c51f327629 100644 --- a/man/en/sos-clean.1 +++ b/man/en/sos-clean.1 @@ -135,7 +135,7 @@ on the system from a previous execution. .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1 index 7f6bd450ec..7ef0571881 100644 --- a/man/en/sos-collect.1 +++ b/man/en/sos-collect.1 @@ -395,4 +395,8 @@ Sosreport option. Override the default compression type. .BR sos.conf (5) .SH MAINTAINER - Jake Hunsaker +.nf +Maintained on GitHub at https://github.com/sosreport/sos +.fi +.SH AUTHORS & CONTRIBUTORS +See \fBAUTHORS\fR file in the package documentation. diff --git a/man/en/sos-report.1 b/man/en/sos-report.1 index c4859b8fea..4aa60a37f5 100644 --- a/man/en/sos-report.1 +++ b/man/en/sos-report.1 @@ -494,7 +494,7 @@ Display usage message. .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. diff --git a/man/en/sos.1 b/man/en/sos.1 index 2d5a972145..b4396fdcb7 100644 --- a/man/en/sos.1 +++ b/man/en/sos.1 @@ -165,7 +165,7 @@ Display usage message. .BR sos.conf (5) .SH MAINTAINER .nf -Jake Hunsaker +Maintained on GitHub at https://github.com/sosreport/sos .fi .SH AUTHORS & CONTRIBUTORS See \fBAUTHORS\fR file in the package documentation. From cca04c1e1fdc97632cdefdc1d1d7a3dea9cb4aea Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Wed, 28 Jun 2023 11:49:56 +0200 Subject: [PATCH 05/32] [clean] Respect permissions of sanitised files When copying files we applied a substitution in, we must replace just original file content (shutil.copyfile) and not also its stat data (shutil.copy). Resolves: #3292 Signed-off-by: Pavel Moravec --- sos/cleaner/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py index feeedf6680..fbcaa9c3c7 100644 --- a/sos/cleaner/__init__.py +++ b/sos/cleaner/__init__.py @@ -778,7 +778,7 @@ def obfuscate_file(self, filename, short_name=None, arc_name=None): % (short_name, err), caller=arc_name) tfile.seek(0) if subs: - shutil.copy(tfile.name, filename) + shutil.copyfile(tfile.name, filename) tfile.close() _ob_short_name = self.obfuscate_string(short_name.split('/')[-1]) From fc1489a621108d3613d3337489a64950e52d77c3 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Thu, 29 Jun 2023 22:57:46 +0200 Subject: [PATCH 06/32] [tests] add test for #3292 Add a test that cleaner keeps permissions of a sanitised file Relevant to: #3292 Signed-off-by: Pavel Moravec --- .../basic_function_tests/report_with_mask.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/cleaner_tests/basic_function_tests/report_with_mask.py b/tests/cleaner_tests/basic_function_tests/report_with_mask.py index 7c4d390594..baee836a21 100644 --- a/tests/cleaner_tests/basic_function_tests/report_with_mask.py +++ b/tests/cleaner_tests/basic_function_tests/report_with_mask.py @@ -9,6 +9,7 @@ from sos_tests import StageOneReportTest, StageTwoReportTest import re +from os import stat class ReportWithMask(StageOneReportTest): @@ -18,6 +19,17 @@ class ReportWithMask(StageOneReportTest): """ sos_cmd = '--mask -o host,networking' + hosts_obfuscated = None + + def pre_sos_setup(self): + # obfuscate a random word from /etc/hosts and ensure the updated + # sanitised file has same permissions (a+r) + try: + self.hosts_obfuscated = open('/etc/hosts').read().strip('#\n').split()[-1] + except (FileNotFoundError, IndexError) as e: + self.warning(f"Unable to process /etc/hosts: {e}") + if self.hosts_obfuscated: + self.sos_cmd += f' --keywords={self.hosts_obfuscated}' def test_mask_was_run(self): self.assertOutputContains('Beginning obfuscation') @@ -53,6 +65,12 @@ def test_mac_addrs_were_obfuscated(self): mac = line.strip().split()[1] assert mac.startswith('53:4f:53'), "Found unobfuscated mac addr %s" % mac + def test_perms_unchanged_on_modified_file(self): + if self.hosts_obfuscated: + imode_orig = stat('/etc/hosts').st_mode + imode_obfuscated = stat(self.get_name_in_archive('etc/hosts')).st_mode + self.assertEqual(imode_orig, imode_obfuscated) + class ReportWithCleanedKeywords(StageOneReportTest): """Testing for obfuscated keywords provided by the user From 24b650731e1a0b5f51938ee51cd6dd7a448f8ca7 Mon Sep 17 00:00:00 2001 From: Daniel Zhou Date: Mon, 3 Jul 2023 11:00:25 -0400 Subject: [PATCH 07/32] [ssh] Add User .ssh Config File Option Resolves issue SUPDEV-137. Adds a plugin option for the ssh module, defining whether it will or will not collect .ssh config files per user Default for new option is True Signed-off-by: Daniel Zhou --- sos/report/plugins/ssh.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sos/report/plugins/ssh.py b/sos/report/plugins/ssh.py index 0fe5dd4e2d..67a022902c 100644 --- a/sos/report/plugins/ssh.py +++ b/sos/report/plugins/ssh.py @@ -8,7 +8,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, IndependentPlugin +from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Ssh(Plugin, IndependentPlugin): @@ -18,6 +18,12 @@ class Ssh(Plugin, IndependentPlugin): plugin_name = 'ssh' profiles = ('services', 'security', 'system', 'identity') + option_list = [ + PluginOpt('userconfs', default=True, val_type=str, + desc=('Changes whether module will ' + 'collect user .ssh configs')) + ] + def setup(self): self.add_file_tags({ @@ -34,7 +40,10 @@ def setup(self): self.add_copy_spec(sshcfgs) self.included_configs(sshcfgs) - self.user_ssh_files_permissions() + + # If userconfs option is set to False, skips this + if self.get_option('userconfs'): + self.user_ssh_files_permissions() def included_configs(self, sshcfgs): # Read configs for any includes and copy those From aade89e49303d69ccb9b636c5e53575e4c0614f4 Mon Sep 17 00:00:00 2001 From: Pablo Acevedo Montserrat Date: Tue, 4 Jul 2023 17:15:15 +0200 Subject: [PATCH 08/32] [microshift] Add initial support for backup info Signed-off-by: Pablo Acevedo Montserrat --- sos/report/plugins/microshift.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sos/report/plugins/microshift.py b/sos/report/plugins/microshift.py index 69981d6314..1b932d648b 100644 --- a/sos/report/plugins/microshift.py +++ b/sos/report/plugins/microshift.py @@ -26,8 +26,10 @@ class Microshift(Plugin, RedHatPlugin): short_desc = 'Microshift' plugin_name = 'microshift' plugin_timeout = 900 - packages = ('microshift', 'microshift-selinux', 'microshift-networking') - services = (plugin_name,) + packages = ('microshift', 'microshift-selinux', 'microshift-networking', + 'microshift-greenboot') + services = (plugin_name, 'greenboot-healthcheck', + 'greenboot-task-runner', 'redboot-task-runner') profiles = (plugin_name,) localhost_kubeconfig = '/var/lib/microshift/resources/kubeadmin/kubeconfig' @@ -146,7 +148,10 @@ def setup(self): Output format for this function is based on `oc adm inspect` command, which is used to retrieve all API resources from the cluster. """ - self.add_forbidden_path('/var/lib/microshift') + if self.path_exists('/var/lib/microshift-backups'): + self.add_copy_spec(['/var/lib/microshift-backups/*/version', + '/var/lib/microshift-backups/*.json']) + self.add_cmd_output([ 'microshift version', 'microshift show-config -m effective' From 59c2660584734af92eca2eae31af3fbf5557f853 Mon Sep 17 00:00:00 2001 From: Jan Jansky Date: Mon, 10 Jul 2023 13:10:22 +0200 Subject: [PATCH 09/32] [clean] Properly obfuscate MAC addresses Some of mac addresses was not properly obfuscated because some collected data contains mac addresses in format 01: and parser was not ready for that. Also added mapper which will obfuscate mac address in case it is in format with _ instead of : as for example 00_50_56_87_5d_01 instead of 00:50:56:87:5d:01 Format with _ is used for example by vmware plugin. Resolves: #3302 Signed-off-by: Jan Jansky --- sos/cleaner/mappings/mac_map.py | 2 +- sos/cleaner/parsers/mac_parser.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sos/cleaner/mappings/mac_map.py b/sos/cleaner/mappings/mac_map.py index 334a668144..4ccba25a40 100644 --- a/sos/cleaner/mappings/mac_map.py +++ b/sos/cleaner/mappings/mac_map.py @@ -75,5 +75,5 @@ def sanitize_item(self, item): if re.match('(([0-9a-fA-F]{4}:){3}([0-9a-fA-F]){4})', item): return self.mac6_quad_template % hextets # match 48-bit IPv4 MAC addresses - if re.match('([0-9a-fA-F]:?){12}', item): + if re.match('([0-9a-fA-F][:_]?){12}', item): return self.mac_template % hextets diff --git a/sos/cleaner/parsers/mac_parser.py b/sos/cleaner/parsers/mac_parser.py index 88b0ac2e98..4e790018e9 100644 --- a/sos/cleaner/parsers/mac_parser.py +++ b/sos/cleaner/parsers/mac_parser.py @@ -25,8 +25,8 @@ ) # aa:bb:cc:dd:ee:ff avoiding ipv6 substring matches IPV4_REG = ( - r'((? Date: Tue, 27 Jun 2023 17:49:03 +0100 Subject: [PATCH 10/32] [gh_worklow] Build and push snap on release Adding workdlow to publish the released sos to the latest/candidate channel when the a new release is created via the tag. This ensures less mannual intervention for future releases. Update the snap creation for anyone building manually, and need python3-venv as a build dependancy. Signed-off-by: Arif Ali --- .github/workflows/snap.yaml | 13 ++++++++++++- snap/snapcraft.yaml | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/snap.yaml b/.github/workflows/snap.yaml index e68908355c..d7d8935232 100644 --- a/.github/workflows/snap.yaml +++ b/.github/workflows/snap.yaml @@ -3,10 +3,13 @@ on: push: branches: - main + release: + types: + - published jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 concurrency: group: snap-build cancel-in-progress: true @@ -25,8 +28,16 @@ jobs: - run: | sudo sos help - uses: snapcore/action-publish@v1 + if: ${{ github.event_name == 'push' }} env: SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} with: snap: ${{ steps.build-snap.outputs.snap }} release: "latest/edge" + - uses: snapcore/action-publish@v1 + if: ${{ github.event_name == 'release' }} + env: + SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} + with: + snap: ${{ steps.build-snap.outputs.snap }} + release: "latest/candidate" diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 94f427c2a1..dc3b43e2dc 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -26,6 +26,7 @@ parts: - python3 - snapcraft - gettext + - python3-venv stage-packages: - python3.10-minimal - libpython3.10-minimal From 984abd5cf5b4379508b0fc7d959f17e56577964f Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Wed, 31 May 2023 13:37:17 +0100 Subject: [PATCH 11/32] [maas] remove the collection of apache2 logs MAAS no longer uses apache2 for it web hosting, so no longer required for this plugin Signed-off-by: Arif Ali --- sos/report/plugins/maas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sos/report/plugins/maas.py b/sos/report/plugins/maas.py index ca819511a4..d79201121f 100644 --- a/sos/report/plugins/maas.py +++ b/sos/report/plugins/maas.py @@ -90,7 +90,6 @@ def setup(self): "/var/lib/maas/http/*.conf", "/var/lib/maas/*.conf", "/var/lib/maas/rsyslog", - "/var/log/apache2*", "/var/log/maas*", "/var/log/upstart/maas-*", ]) From 5bcdb79cb59edf3169d99227777bf700f04a036e Mon Sep 17 00:00:00 2001 From: Nikhil Kshirsagar Date: Wed, 28 Jun 2023 07:10:57 +0000 Subject: [PATCH 12/32] [plugins/ceph][ubuntu] Add microceph collections Collect the microceph data in the ceph plugins. Signed-off-by: Nikhil Kshirsagar --- sos/report/plugins/ceph_common.py | 70 +++++++++-------- sos/report/plugins/ceph_mon.py | 59 +++++++++----- sos/report/plugins/ceph_osd.py | 123 +++++++++++++++++------------- 3 files changed, 146 insertions(+), 106 deletions(-) diff --git a/sos/report/plugins/ceph_common.py b/sos/report/plugins/ceph_common.py index ec5852105c..12c5d171d1 100644 --- a/sos/report/plugins/ceph_common.py +++ b/sos/report/plugins/ceph_common.py @@ -41,45 +41,53 @@ class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin): # This check will enable the plugin regardless of being # containerized or not - files = ('/etc/ceph/ceph.conf',) + files = ('/etc/ceph/ceph.conf', + '/var/snap/microceph/*',) def setup(self): all_logs = self.get_option("all_logs") - self.add_file_tags({ - '.*/ceph.conf': 'ceph_conf', - '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log', - }) - - if not all_logs: - self.add_copy_spec("/var/log/calamari/*.log",) + microceph_pkg = self.policy.package_manager.pkg_by_name('microceph') + if not microceph_pkg: + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log', + }) + + if not all_logs: + self.add_copy_spec("/var/log/calamari/*.log",) + else: + self.add_copy_spec("/var/log/calamari",) + + self.add_copy_spec([ + "/var/log/ceph/**/ceph.log", + "/var/log/ceph/**/ceph.audit.log*", + "/var/log/calamari/*.log", + "/etc/ceph/", + "/etc/calamari/", + "/var/lib/ceph/tmp/", + ]) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/*keyring*", + "/var/lib/ceph/*/*keyring*", + "/var/lib/ceph/*/*/*keyring*", + "/var/lib/ceph/osd", + "/var/lib/ceph/mon", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) else: - self.add_copy_spec("/var/log/calamari",) - - self.add_copy_spec([ - "/var/log/ceph/**/ceph.log", - "/var/log/ceph/**/ceph.audit.log*", - "/var/log/calamari/*.log", - "/etc/ceph/", - "/etc/calamari/", - "/var/lib/ceph/tmp/", - ]) + self.add_copy_spec([ + "/var/snap/microceph/common/logs/ceph.log", + "/var/snap/microceph/common/logs/ceph.audit.log", + ]) self.add_cmd_output([ "ceph -v", ]) - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/*keyring*", - "/var/lib/ceph/*/*keyring*", - "/var/lib/ceph/*/*/*keyring*", - "/var/lib/ceph/osd", - "/var/lib/ceph/mon", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - # vim: set et ts=4 sw=4 : diff --git a/sos/report/plugins/ceph_mon.py b/sos/report/plugins/ceph_mon.py index d25fe3abbe..6d36f73481 100644 --- a/sos/report/plugins/ceph_mon.py +++ b/sos/report/plugins/ceph_mon.py @@ -14,8 +14,8 @@ class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): """ This plugin serves to collect information on monitor nodes within a Ceph - cluster. It is designed to collect from several versions of Ceph, including - those versions that serve as the basis for RHCS 4 and RHCS 5. + or microceph cluster. It is designed to collect from several versions of + Ceph, including versions that serve as the basis for RHCS 4 and RHCS 5. Older versions of Ceph will have collections from locations such as /var/log/ceph, whereas newer versions (as of this plugin's latest update) @@ -37,32 +37,49 @@ class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): # but by default they are not capable of running various ceph commands in # this plugin - the `ceph` binary is functional directly on the host containers = ('ceph-(.*-)?mon.*',) - files = ('/var/lib/ceph/mon/', '/var/lib/ceph/*/mon*') + files = ('/var/lib/ceph/mon/', '/var/lib/ceph/*/mon*', + '/var/snap/microceph/common/data/mon/*') ceph_version = 0 def setup(self): self.ceph_version = self.get_ceph_version() - self.add_file_tags({ - '.*/ceph.conf': 'ceph_conf', - "/var/log/ceph/(.*/)?ceph-.*mon.*.log": 'ceph_mon_log' - }) - - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/**/*keyring*", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/**/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) + microceph_pkg = self.policy.package_manager.pkg_by_name('microceph') + if not microceph_pkg: + self.add_file_tags({ + '.*/ceph.conf': 'ceph_conf', + "/var/log/ceph/(.*/)?ceph-.*mon.*.log": 'ceph_mon_log' + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + self.add_copy_spec([ + "/run/ceph/**/ceph-mon*", + "/var/lib/ceph/**/kv_backend", + "/var/log/ceph/**/*ceph-mon*.log" + ]) - self.add_copy_spec([ - "/run/ceph/**/ceph-mon*", - "/var/lib/ceph/**/kv_backend", - "/var/log/ceph/**/*ceph-mon*.log" - ]) + else: + self.add_forbidden_path([ + "/var/snap/microceph/common/**/*keyring*", + "/var/snap/microceph/current/**/*keyring*", + "/var/snap/microceph/common/data/mon/*/store.db", + "/var/snap/microceph/common/state/*", + ]) + + self.add_copy_spec([ + "/var/snap/microceph/common/data/mon/*", + "/var/snap/microceph/common/logs/*ceph-mon*.log", + "/var/snap/microceph/current/conf/*", + ]) self.add_cmd_output("ceph report", tags="ceph_report") self.add_cmd_output([ diff --git a/sos/report/plugins/ceph_osd.py b/sos/report/plugins/ceph_osd.py index 34a656a333..9dc851e0a1 100644 --- a/sos/report/plugins/ceph_osd.py +++ b/sos/report/plugins/ceph_osd.py @@ -31,62 +31,77 @@ class CephOSD(Plugin, RedHatPlugin, UbuntuPlugin): plugin_name = 'ceph_osd' profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?osd.*',) - files = ('/var/lib/ceph/osd/', '/var/lib/ceph/*/osd*') + files = ('/var/lib/ceph/osd/', '/var/lib/ceph/*/osd*', + '/var/snap/microceph/common/data/osd/*') def setup(self): - - self.add_file_tags({ - "/var/log/ceph/(.*/)?ceph-(.*-)?osd.*.log": 'ceph_osd_log', - }) - - self.add_forbidden_path([ - "/etc/ceph/*keyring*", - "/var/lib/ceph/**/*keyring*", - # Excludes temporary ceph-osd mount location like - # /var/lib/ceph/tmp/mnt.XXXX from sos collection. - "/var/lib/ceph/**/tmp/*mnt*", - "/etc/ceph/*bindpass*" - ]) - - # Only collect OSD specific files - self.add_copy_spec([ - "/run/ceph/**/ceph-osd*", - "/var/lib/ceph/**/kv_backend", - "/var/log/ceph/**/ceph-osd*.log", - "/var/log/ceph/**/ceph-volume*.log", - ]) - - self.add_cmd_output([ - "ceph-disk list", - "ceph-volume lvm list" - ]) - - cmds = [ - "bluestore bluefs available", - "config diff", - "config show", - "dump_blacklist", - "dump_blocked_ops", - "dump_historic_ops_by_duration", - "dump_historic_slow_ops", - "dump_mempools", - "dump_ops_in_flight", - "dump_op_pq_state", - "dump_osd_network", - "dump_reservations", - "dump_watchers", - "log dump", - "perf dump", - "perf histogram dump", - "objecter_requests", - "ops", - "status", - "version", - ] - - self.add_cmd_output( - [f"ceph daemon {i} {c}" for i in self.get_socks() for c in cmds] - ) + microceph_pkg = self.policy.package_manager.pkg_by_name('microceph') + if not microceph_pkg: + self.add_file_tags({ + "/var/log/ceph/(.*/)?ceph-(.*-)?osd.*.log": 'ceph_osd_log', + }) + + self.add_forbidden_path([ + "/etc/ceph/*keyring*", + "/var/lib/ceph/**/*keyring*", + # Excludes temporary ceph-osd mount location like + # /var/lib/ceph/tmp/mnt.XXXX from sos collection. + "/var/lib/ceph/**/tmp/*mnt*", + "/etc/ceph/*bindpass*" + ]) + + # Only collect OSD specific files + self.add_copy_spec([ + "/run/ceph/**/ceph-osd*", + "/var/lib/ceph/**/kv_backend", + "/var/log/ceph/**/ceph-osd*.log", + "/var/log/ceph/**/ceph-volume*.log", + ]) + + self.add_cmd_output([ + "ceph-disk list", + "ceph-volume lvm list" + ]) + + cmds = [ + "bluestore bluefs available", + "config diff", + "config show", + "dump_blacklist", + "dump_blocked_ops", + "dump_historic_ops_by_duration", + "dump_historic_slow_ops", + "dump_mempools", + "dump_ops_in_flight", + "dump_op_pq_state", + "dump_osd_network", + "dump_reservations", + "dump_watchers", + "log dump", + "perf dump", + "perf histogram dump", + "objecter_requests", + "ops", + "status", + "version", + ] + + self.add_cmd_output( + [f"ceph daemon {i} {c}" for i in self.get_socks() for c in cmds] + ) + + else: + # Only collect microceph files, don't run any commands + self.add_forbidden_path([ + "/var/snap/microceph/common/**/*keyring*", + "/var/snap/microceph/current/**/*keyring*", + "/var/snap/microceph/common/state/*", + ]) + + self.add_copy_spec([ + "/var/snap/microceph/common/data/osd/*", + "/var/snap/microceph/common/logs/*ceph-osd*.log", + ]) def get_socks(self): """ From d2a14422ef5771fd704a4c761f26768d4bd9bb7f Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Fri, 14 Jul 2023 12:50:58 +0100 Subject: [PATCH 13/32] [plugins] py311 fix for re.M Add a common regex for re.MULTILINE so that doesn't need to be defined and remove the definitions where `(?m)` is being defined Closes: #3301 Signed-off-by: Arif Ali --- sos/report/plugins/__init__.py | 5 +++-- sos/report/plugins/collectd.py | 2 +- sos/report/plugins/grafana.py | 2 +- sos/report/plugins/juju.py | 2 +- sos/report/plugins/openssl.py | 2 +- sos/report/plugins/openstack_ansible.py | 2 +- sos/report/plugins/openstack_aodh.py | 4 ++-- sos/report/plugins/openstack_barbican.py | 4 ++-- sos/report/plugins/openstack_ceilometer.py | 4 ++-- sos/report/plugins/openstack_cinder.py | 4 ++-- sos/report/plugins/openstack_designate.py | 2 +- sos/report/plugins/openstack_glance.py | 4 ++-- sos/report/plugins/openstack_heat.py | 4 ++-- sos/report/plugins/openstack_horizon.py | 2 +- sos/report/plugins/openstack_instack.py | 4 ++-- sos/report/plugins/openstack_ironic.py | 4 ++-- sos/report/plugins/openstack_keystone.py | 6 +++--- sos/report/plugins/openstack_manila.py | 4 ++-- sos/report/plugins/openstack_neutron.py | 4 ++-- sos/report/plugins/openstack_nova.py | 5 +++-- sos/report/plugins/openstack_octavia.py | 2 +- sos/report/plugins/openstack_placement.py | 4 ++-- sos/report/plugins/openstack_sahara.py | 4 ++-- sos/report/plugins/openstack_swift.py | 4 ++-- sos/report/plugins/openstack_trove.py | 4 ++-- sos/report/plugins/ovirt.py | 2 +- sos/report/plugins/salt.py | 2 +- sos/report/plugins/saltmaster.py | 2 +- 28 files changed, 48 insertions(+), 46 deletions(-) diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py index e8aedca813..a4dfbb64aa 100644 --- a/sos/report/plugins/__init__.py +++ b/sos/report/plugins/__init__.py @@ -1274,12 +1274,13 @@ def do_file_sub(self, srcpath, regexp, subst): """ try: path = self._get_dest_for_srcpath(srcpath) + common_flags = re.IGNORECASE | re.MULTILINE if hasattr(regexp, "pattern"): pattern = regexp.pattern - flags = regexp.flags | re.IGNORECASE + flags = regexp.flags | common_flags else: pattern = regexp - flags = re.IGNORECASE + flags = common_flags self._log_debug("substituting scrpath '%s'" % srcpath) self._log_debug("substituting '%s' for '%s' in '%s'" % (subst, pattern, path)) diff --git a/sos/report/plugins/collectd.py b/sos/report/plugins/collectd.py index 818f1568df..653555430f 100644 --- a/sos/report/plugins/collectd.py +++ b/sos/report/plugins/collectd.py @@ -51,7 +51,7 @@ def postproc(self): "Password", "User", "[<]*URL", "Address" ] - regexp = r"((?m)^[#]*\s*(%s)\s* \s*)(.*)" % "|".join(protect_keys) + regexp = r"(^[#]*\s*(%s)\s* \s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub( "/etc/collectd.d/*.conf", regexp, r'\1"*********"' diff --git a/sos/report/plugins/grafana.py b/sos/report/plugins/grafana.py index f58f18710a..4aa5668c21 100644 --- a/sos/report/plugins/grafana.py +++ b/sos/report/plugins/grafana.py @@ -42,6 +42,6 @@ def postproc(self): "admin_password", "secret_key" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub("/etc/grafana/grafana.ini", regexp, r"\1*********") diff --git a/sos/report/plugins/juju.py b/sos/report/plugins/juju.py index 3a947b54a6..b0208042a3 100644 --- a/sos/report/plugins/juju.py +++ b/sos/report/plugins/juju.py @@ -63,7 +63,7 @@ def postproc(self): ] # Redact simple yaml style "key: value". - keys_regex = r"((?m)^\s*(%s)\s*:\s*)(.*)" % "|".join(protect_keys) + keys_regex = r"(^\s*(%s)\s*:\s*)(.*)" % "|".join(protect_keys) sub_regex = r"\1*********" self.do_path_regex_sub(agents_path, keys_regex, sub_regex) # Redact certificates diff --git a/sos/report/plugins/openssl.py b/sos/report/plugins/openssl.py index b9c77f4d0c..1003f99da0 100644 --- a/sos/report/plugins/openssl.py +++ b/sos/report/plugins/openssl.py @@ -27,7 +27,7 @@ def postproc(self): "challengePassword" ] - regexp = r"(?m)^(\s*#?\s*(%s).*=)(.*)" % "|".join(protect_keys) + regexp = r"^(\s*#?\s*(%s).*=)(.*)" % "|".join(protect_keys) self.do_file_sub( '/etc/ssl/openssl.cnf', diff --git a/sos/report/plugins/openstack_ansible.py b/sos/report/plugins/openstack_ansible.py index c7488e7643..1d260147a6 100644 --- a/sos/report/plugins/openstack_ansible.py +++ b/sos/report/plugins/openstack_ansible.py @@ -34,7 +34,7 @@ def postproc(self): "/etc/openstack_deploy/user_secrets.yml", "/etc/rpc_deploy/user_secrets.yml" ] - regexp = r"(?m)^\s*#*([\w_]*:\s*).*" + regexp = r"^\s*#*([\w_]*:\s*).*" for secrets_file in secrets_files: self.do_path_regex_sub( secrets_file, diff --git a/sos/report/plugins/openstack_aodh.py b/sos/report/plugins/openstack_aodh.py index 3688706bc8..a53e15e305 100644 --- a/sos/report/plugins/openstack_aodh.py +++ b/sos/report/plugins/openstack_aodh.py @@ -80,11 +80,11 @@ def postproc(self): connection_keys = ["connection", "backend_url", "transport_url"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_barbican.py b/sos/report/plugins/openstack_barbican.py index aaa6c0fb24..14a75f1dd1 100644 --- a/sos/report/plugins/openstack_barbican.py +++ b/sos/report/plugins/openstack_barbican.py @@ -42,7 +42,7 @@ def postproc(self): ] self.do_file_sub( "/etc/barbican/barbican.conf", - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1********" ) @@ -50,7 +50,7 @@ def postproc(self): self.do_path_regex_sub( "/etc/barbican/barbican.conf", - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6") diff --git a/sos/report/plugins/openstack_ceilometer.py b/sos/report/plugins/openstack_ceilometer.py index 0031afd251..7b313f5444 100644 --- a/sos/report/plugins/openstack_ceilometer.py +++ b/sos/report/plugins/openstack_ceilometer.py @@ -59,11 +59,11 @@ def postproc(self): connection_keys = ["connection", "backend_url", "transport_url"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_cinder.py b/sos/report/plugins/openstack_cinder.py index 23483a8d05..56f65cce6b 100644 --- a/sos/report/plugins/openstack_cinder.py +++ b/sos/report/plugins/openstack_cinder.py @@ -94,11 +94,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_designate.py b/sos/report/plugins/openstack_designate.py index a2ea37ab7c..616bb0c158 100644 --- a/sos/report/plugins/openstack_designate.py +++ b/sos/report/plugins/openstack_designate.py @@ -81,7 +81,7 @@ def postproc(self): "ssl_key_password", "ssl_client_key_password", "memcache_secret_key" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub("/etc/designate/*", regexp, r"\1*********") self.do_path_regex_sub( diff --git a/sos/report/plugins/openstack_glance.py b/sos/report/plugins/openstack_glance.py index ec0feee680..c8ec8fff94 100644 --- a/sos/report/plugins/openstack_glance.py +++ b/sos/report/plugins/openstack_glance.py @@ -96,11 +96,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_heat.py b/sos/report/plugins/openstack_heat.py index 28e20ad213..3d0460393a 100644 --- a/sos/report/plugins/openstack_heat.py +++ b/sos/report/plugins/openstack_heat.py @@ -109,11 +109,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_horizon.py b/sos/report/plugins/openstack_horizon.py index 167b377a28..f7952df9be 100644 --- a/sos/report/plugins/openstack_horizon.py +++ b/sos/report/plugins/openstack_horizon.py @@ -50,7 +50,7 @@ def postproc(self): "SECRET_KEY", "EMAIL_HOST_PASSWORD" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) for regpath in [r"/etc/openstack-dashboard/.*\.json", "/etc/openstack-dashboard/local_settings$"]: self.do_path_regex_sub(regpath, regexp, r"\1*********") diff --git a/sos/report/plugins/openstack_instack.py b/sos/report/plugins/openstack_instack.py index af99b0d172..1ca70fb02a 100644 --- a/sos/report/plugins/openstack_instack.py +++ b/sos/report/plugins/openstack_instack.py @@ -133,13 +133,13 @@ def postproc(self): "undercloud_swift_password", "undercloud_tuskar_password", ] - regexp = r"((?m)(%s)=)(.*)" % "|".join(protected_keys) + regexp = r"((%s)=)(.*)" % "|".join(protected_keys) self.do_file_sub("/home/stack/.instack/install-undercloud.log", regexp, r"\1*********") self.do_file_sub(UNDERCLOUD_CONF_PATH, regexp, r"\1*********") protected_json_keys = ["pm_password", "ssh-key", "password"] - json_regexp = r'((?m)"(%s)": )(".*?")' % "|".join(protected_json_keys) + json_regexp = r'("(%s)": )(".*?")' % "|".join(protected_json_keys) self.do_file_sub("/home/stack/instackenv.json", json_regexp, r"\1*********") self.do_file_sub('/home/stack/.tripleo/history', diff --git a/sos/report/plugins/openstack_ironic.py b/sos/report/plugins/openstack_ironic.py index 6646f45bd9..05985e53a6 100644 --- a/sos/report/plugins/openstack_ironic.py +++ b/sos/report/plugins/openstack_ironic.py @@ -145,11 +145,11 @@ def postproc(self): connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_keystone.py b/sos/report/plugins/openstack_keystone.py index 378d359cdf..0fa5f1d539 100644 --- a/sos/report/plugins/openstack_keystone.py +++ b/sos/report/plugins/openstack_keystone.py @@ -98,11 +98,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) @@ -110,7 +110,7 @@ def postproc(self): # obfuscate LDAP plaintext passwords in domain config dir self.do_path_regex_sub( self.domain_config_dir, - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1********" ) diff --git a/sos/report/plugins/openstack_manila.py b/sos/report/plugins/openstack_manila.py index 3c0e49c2e4..4c03e50f8d 100644 --- a/sos/report/plugins/openstack_manila.py +++ b/sos/report/plugins/openstack_manila.py @@ -64,11 +64,11 @@ def postproc(self): connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_neutron.py b/sos/report/plugins/openstack_neutron.py index a9f2e637d5..f24dfebef4 100644 --- a/sos/report/plugins/openstack_neutron.py +++ b/sos/report/plugins/openstack_neutron.py @@ -89,11 +89,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_nova.py b/sos/report/plugins/openstack_nova.py index 8bdcb1c6d1..15e3ecc129 100644 --- a/sos/report/plugins/openstack_nova.py +++ b/sos/report/plugins/openstack_nova.py @@ -109,6 +109,7 @@ def setup(self): self.add_copy_spec(self.path_join(novadir, novalog)) self.add_copy_spec([ "/var/log/{}*/nova*.log".format(self.apachepkg), + "/var/log/{}*/placement*.log".format(self.apachepkg), ]) pp = ['', '_libvirt', '_metadata', '_placement'] @@ -154,11 +155,11 @@ def postproc(self): connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_octavia.py b/sos/report/plugins/openstack_octavia.py index 45d686356e..75657d3192 100644 --- a/sos/report/plugins/openstack_octavia.py +++ b/sos/report/plugins/openstack_octavia.py @@ -117,7 +117,7 @@ def postproc(self): "connection", "transport_url", "server_certs_key_passphrase", "memcache_secret_key" ] - regexp = r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub("/etc/octavia/*", regexp, r"\1*********") self.do_path_regex_sub( diff --git a/sos/report/plugins/openstack_placement.py b/sos/report/plugins/openstack_placement.py index 637b95fdc7..6d63cb3584 100644 --- a/sos/report/plugins/openstack_placement.py +++ b/sos/report/plugins/openstack_placement.py @@ -75,11 +75,11 @@ def postproc(self): connection_keys = ["database_connection", "slave_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_sahara.py b/sos/report/plugins/openstack_sahara.py index 99361d7d13..0a7fcf04c2 100644 --- a/sos/report/plugins/openstack_sahara.py +++ b/sos/report/plugins/openstack_sahara.py @@ -52,11 +52,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_swift.py b/sos/report/plugins/openstack_swift.py index 232a65edd3..4f4324c230 100644 --- a/sos/report/plugins/openstack_swift.py +++ b/sos/report/plugins/openstack_swift.py @@ -62,11 +62,11 @@ def postproc(self): connection_keys = ["connection", "sql_connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/openstack_trove.py b/sos/report/plugins/openstack_trove.py index f617072bb4..de8fe77c57 100644 --- a/sos/report/plugins/openstack_trove.py +++ b/sos/report/plugins/openstack_trove.py @@ -51,11 +51,11 @@ def postproc(self): connection_keys = ["connection"] self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), + r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(protect_keys), r"\1*********" ) self.apply_regex_sub( - r"((?m)^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % + r"(^\s*(%s)\s*=\s*(.*)://(\w*):)(.*)(@(.*))" % "|".join(connection_keys), r"\1*********\6" ) diff --git a/sos/report/plugins/ovirt.py b/sos/report/plugins/ovirt.py index 19375deb10..d5b6d77252 100644 --- a/sos/report/plugins/ovirt.py +++ b/sos/report/plugins/ovirt.py @@ -263,7 +263,7 @@ def postproc(self): "pool.default.ssl.truststore.password", "config.datasource.dbpassword" ] - regexp = r"((?m)^\s*#*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) + regexp = r"(^\s*#*(%s)\s*=\s*)(.*)" % "|".join(protect_keys) self.do_path_regex_sub(r"/etc/ovirt-engine/aaa/.*\.properties", regexp, r"\1*********") diff --git a/sos/report/plugins/salt.py b/sos/report/plugins/salt.py index c2a830b00d..bad5e3a647 100644 --- a/sos/report/plugins/salt.py +++ b/sos/report/plugins/salt.py @@ -30,7 +30,7 @@ def setup(self): self.add_forbidden_path("/etc/salt/pki/*/*.pem") def postproc(self): - regexp = r'((?m)^\s+.*(pass|secret|(? Date: Tue, 18 Jul 2023 08:58:52 +0200 Subject: [PATCH 14/32] [options] compatibility fix for python 3.12 Addresses ConfigParser.readfp removal in 3.12. Fixes #3308 Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=2223526 Signed-off-by: Sandro Bonazzola --- sos/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sos/options.py b/sos/options.py index 2d5a513597..deb65f6590 100644 --- a/sos/options.py +++ b/sos/options.py @@ -226,7 +226,7 @@ def _update_from_section(section, config): try: try: with open(config_file) as f: - config.readfp(f) + config.read_file(f, config_file) except DuplicateOptionError as err: raise exit("Duplicate option '%s' in section '%s' in file %s" % (err.option, err.section, config_file)) From 221f587c4af9dace771b5695ecbd9d00e2c62cf3 Mon Sep 17 00:00:00 2001 From: Nikhil Kshirsagar Date: Wed, 19 Jul 2023 09:56:28 +0000 Subject: [PATCH 15/32] [utilities][ceph] wildcard support for path in files, fix ceph paths This lets files triggers use * and also updates the ceph plugins to trigger only if folders are non empty Closes: #3311 Signed-off-by: Nikhil Kshirsagar --- sos/report/plugins/ceph_mds.py | 2 +- sos/report/plugins/ceph_mgr.py | 2 +- sos/report/plugins/ceph_mon.py | 2 +- sos/report/plugins/ceph_osd.py | 2 +- sos/report/plugins/ceph_rgw.py | 2 +- sos/utilities.py | 2 ++ 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sos/report/plugins/ceph_mds.py b/sos/report/plugins/ceph_mds.py index c31e906b2d..c2155f3409 100644 --- a/sos/report/plugins/ceph_mds.py +++ b/sos/report/plugins/ceph_mds.py @@ -14,7 +14,7 @@ class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin): plugin_name = 'ceph_mds' profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?fs.*',) - files = ('/var/lib/ceph/mds/',) + files = ('/var/lib/ceph/mds/*',) def setup(self): self.add_file_tags({ diff --git a/sos/report/plugins/ceph_mgr.py b/sos/report/plugins/ceph_mgr.py index f7d4ace332..d07631568c 100644 --- a/sos/report/plugins/ceph_mgr.py +++ b/sos/report/plugins/ceph_mgr.py @@ -36,7 +36,7 @@ class CephMGR(Plugin, RedHatPlugin, UbuntuPlugin): plugin_name = 'ceph_mgr' profiles = ('storage', 'virt', 'container', 'ceph') - files = ('/var/lib/ceph/mgr/', '/var/lib/ceph/*/mgr*') + files = ('/var/lib/ceph/mgr/*', '/var/lib/ceph/*/mgr*') containers = ('ceph-(.*-)?mgr.*',) def setup(self): diff --git a/sos/report/plugins/ceph_mon.py b/sos/report/plugins/ceph_mon.py index 6d36f73481..7541ae9d95 100644 --- a/sos/report/plugins/ceph_mon.py +++ b/sos/report/plugins/ceph_mon.py @@ -37,7 +37,7 @@ class CephMON(Plugin, RedHatPlugin, UbuntuPlugin): # but by default they are not capable of running various ceph commands in # this plugin - the `ceph` binary is functional directly on the host containers = ('ceph-(.*-)?mon.*',) - files = ('/var/lib/ceph/mon/', '/var/lib/ceph/*/mon*', + files = ('/var/lib/ceph/mon/*', '/var/lib/ceph/*/mon*', '/var/snap/microceph/common/data/mon/*') ceph_version = 0 diff --git a/sos/report/plugins/ceph_osd.py b/sos/report/plugins/ceph_osd.py index 9dc851e0a1..41bde4c910 100644 --- a/sos/report/plugins/ceph_osd.py +++ b/sos/report/plugins/ceph_osd.py @@ -31,7 +31,7 @@ class CephOSD(Plugin, RedHatPlugin, UbuntuPlugin): plugin_name = 'ceph_osd' profiles = ('storage', 'virt', 'container', 'ceph') containers = ('ceph-(.*-)?osd.*',) - files = ('/var/lib/ceph/osd/', '/var/lib/ceph/*/osd*', + files = ('/var/lib/ceph/osd/*', '/var/lib/ceph/*/osd*', '/var/snap/microceph/common/data/osd/*') def setup(self): diff --git a/sos/report/plugins/ceph_rgw.py b/sos/report/plugins/ceph_rgw.py index 200e30ee33..9688d528fa 100644 --- a/sos/report/plugins/ceph_rgw.py +++ b/sos/report/plugins/ceph_rgw.py @@ -16,7 +16,7 @@ class CephRGW(Plugin, RedHatPlugin, UbuntuPlugin): plugin_name = 'ceph_rgw' profiles = ('storage', 'virt', 'container', 'webserver', 'ceph') containers = ('ceph-(.*)?rgw.*',) - files = ('/var/lib/ceph/radosgw',) + files = ('/var/lib/ceph/radosgw/*',) def setup(self): self.add_copy_spec('/var/log/ceph/ceph-client.rgw*.log', diff --git a/sos/utilities.py b/sos/utilities.py index a09c9d2c1d..5e7b2315f5 100644 --- a/sos/utilities.py +++ b/sos/utilities.py @@ -319,6 +319,8 @@ def _os_wrapper(path, sysroot, method, module=os.path): def path_exists(path, sysroot): + if '*' in path: + return _os_wrapper(path, sysroot, 'glob', module=glob) return _os_wrapper(path, sysroot, 'exists') From fc67659d1b18ebab71f758b89e5bea40d3e3595e Mon Sep 17 00:00:00 2001 From: Nikhil Kshirsagar Date: Thu, 20 Jul 2023 11:51:53 +0530 Subject: [PATCH 16/32] [plugins/samba] collect output of 'net ads info' Signed-off-by: Nikhil Kshirsagar --- sos/report/plugins/samba.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sos/report/plugins/samba.py b/sos/report/plugins/samba.py index ed68bfe96e..6c57cf9320 100644 --- a/sos/report/plugins/samba.py +++ b/sos/report/plugins/samba.py @@ -39,6 +39,7 @@ def setup(self): "wbinfo --domain='.' --domain-groups", "wbinfo --trusted-domains --verbose", "net primarytrust dumpinfo", + "net ads info", ]) From cdf78fe26d3cdeb94dc45484fa66bf58d886500d Mon Sep 17 00:00:00 2001 From: Nathaniel Clark Date: Fri, 14 Jul 2023 09:26:02 -0400 Subject: [PATCH 17/32] [lustre] Add more debug params This prevents mb_groups from being collected as it is large, and not useful for debug. This also collects more state of the filesystem. Signed-off-by: Nathaniel Clark --- sos/report/plugins/lustre.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/sos/report/plugins/lustre.py b/sos/report/plugins/lustre.py index 222c6d5656..2f7eb96967 100644 --- a/sos/report/plugins/lustre.py +++ b/sos/report/plugins/lustre.py @@ -35,25 +35,44 @@ def setup(self): "lnetctl net show -v" ]) - self.get_params("basic", ["version", "health_check", "debug"]) + self.get_params( + "basic", + ["version", "health_check", "debug", "timeout"] + ) self.get_params("lnet", ["peers", "routes", "routers", "nis"]) + self.get_params( + "ldlm-lru", + ["ldlm.namespaces.*.lru_max_age", "ldlm.namespaces.*.lru_size"] + ) self.get_params("ldlm-states", ["*.*.state"]) self.get_params("jobid", ["jobid_name", "jobid_var"]) self.get_params("job-stats", ["*.*.job_stats"]) - self.get_params("exports", ["*.*.exports.*.*"]) + self.get_params("server_uuids", ["*.*.*server_uuid"]) + self.get_params("mgc_irstate", ["mgc.*.ir_state"]) # Client Specific self.add_cmd_output([ "lfs df", "lfs df -i" ]) + self.get_params("osc_client", [ + "osc.*.max_dirty_mb", + "osc.*.max_pages_per_rpc", + "osc.*.checksums", + "osc.*.max_rpcs_in_flight" + ]) # Server Specific self.get_params("osd", ["osd-*.*.{mntdev,files*," + "kbytes*,blocksize,brw_stats}"]) self.get_params("quota", ["osd-*.*.quota_slave." + "{info,limit_*,acct_*}"]) + self.get_params("mgs", ["mgs.MGS.ir_timeout", "mgs.MGS.live.*"]) + self.get_params("exports", ["*.*.exports.*.*"]) + self.get_params("mntdev", ["osd*.*.mntdev"]) + # mb_groups can be VERY large, and provide minimal debug usefulness + self.add_forbidden_path("*/mb_groups") self.add_copy_spec([ "/sys/fs/ldiskfs", "/proc/fs/ldiskfs", From a660c4f160f020c32f8eb2ee5412c54404528ca7 Mon Sep 17 00:00:00 2001 From: Jake Hunsaker Date: Thu, 20 Jul 2023 17:38:49 -0400 Subject: [PATCH 18/32] [build] Bump version to 4.5.6 This commit marks the release of sos-4.5.6. Signed-off-by: Jake Hunsaker --- docs/conf.py | 4 ++-- sos.spec | 5 ++++- sos/__init__.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c501ececee..926dd91960 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,9 +59,9 @@ # built documents. # # The short X.Y version. -version = '4.5.5' +version = '4.5.6' # The full version, including alpha/beta/rc tags. -release = '4.5.5' +release = '4.5.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/sos.spec b/sos.spec index 9ad95535b6..8c84b6e2f3 100644 --- a/sos.spec +++ b/sos.spec @@ -1,6 +1,6 @@ Summary: A set of tools to gather troubleshooting information from a system Name: sos -Version: 4.5.5 +Version: 4.5.6 Release: 1%{?dist} Source0: https://github.com/sosreport/sos/archive/%{name}-%{version}.tar.gz License: GPL-2.0-or-later @@ -86,6 +86,9 @@ rm -rf %{buildroot}/usr/config/ %config(noreplace) %{_sysconfdir}/sos/sos.conf %changelog +* Thu Jul 20 2023 Jake Hunsaker = 4.5.6 +- New upstream release + * Fri Jun 23 2023 Jake Hunsaker = 4.5.5 - New upstream release diff --git a/sos/__init__.py b/sos/__init__.py index c5011365ec..148cc9e754 100644 --- a/sos/__init__.py +++ b/sos/__init__.py @@ -14,7 +14,7 @@ This module houses the i18n setup and message function. The default is to use gettext to internationalize messages. """ -__version__ = "4.5.5" +__version__ = "4.5.6" import os import sys From 0af74a8b24ec9dab0ca0089d5b834ab9908173ac Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy Date: Fri, 21 Jul 2023 15:03:01 +0530 Subject: [PATCH 19/32] Collect db files for ovn interconnect environment This updates openshift_ovn plugin to collect ovn db files when it is running with interconnect configuration. Signed-off-by: Periyasamy Palanisamy --- sos/report/plugins/openshift_ovn.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sos/report/plugins/openshift_ovn.py b/sos/report/plugins/openshift_ovn.py index 98522b1ed6..d81fc97aae 100644 --- a/sos/report/plugins/openshift_ovn.py +++ b/sos/report/plugins/openshift_ovn.py @@ -27,7 +27,13 @@ def setup(self): "/var/log/openvswitch/libreswan.log", "/var/log/openvswitch/ovs-monitor-ipsec.log" ]) + # Collect ovn interconnect specific files if exists. + self.add_copy_spec([ + "/var/lib/ovn-ic/etc/ovnnb_db.db", + "/var/lib/ovn-ic/etc/ovnsb_db.db" + ]) + # The ovn cluster/status is not valid anymore for interconnect setup. self.add_cmd_output([ 'ovn-appctl -t /var/run/ovn/ovnnb_db.ctl ' + 'cluster/status OVN_Northbound', @@ -38,6 +44,10 @@ def setup(self): 'ovs-appctl -t /var/run/ovn/ovn-controller.*.ctl ' + 'ct-zone-list'], container='ovnkube-node') + # Collect ovs ct-zone-list directly on host for interconnect setup. + self.add_cmd_output([ + 'ovs-appctl -t /var/run/ovn-ic/ovn-controller.*.ctl ' + + 'ct-zone-list']) self.add_cmd_output([ 'ovs-appctl -t ovs-monitor-ipsec tunnels/show', 'ipsec status', From 473f4958e95fc138bda3fd55942109ec7d29639a Mon Sep 17 00:00:00 2001 From: Jake Hunsaker Date: Wed, 19 Jul 2023 13:09:28 -0400 Subject: [PATCH 20/32] [report] Don't use sysroot for network device enumeration If sos is being used in a live environment to diagnose an issue, using sysroot can cause the network device enumeration via /sys/class/net crawling to fail. This will be the case for systems that do not use `nmcli`. When in a live environment, network devices will not be under `/$sysroot/sys/class/net` but the "regular" path for the booted environment. Similarly, if sos is being run in a container that is properly configured, network devices will appear under `/sys/class/net` and not (necessarily) under the sysroot path that mounts the host's filesystem. As such, disregard a configured sysroot when enumerating network devices by crawling `/sys/class/net`, and trap any exceptions that may percolate up from this in edge case environments. Related: #3307 Signed-off-by: Jake Hunsaker --- sos/report/__init__.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sos/report/__init__.py b/sos/report/__init__.py index 9c51b3a6db..eb985cd072 100644 --- a/sos/report/__init__.py +++ b/sos/report/__init__.py @@ -612,10 +612,18 @@ def _get_eth_devs(self, namespace=None): filt_devs = ['bonding_masters'] _eth_devs = [] if not namespace: - _eth_devs = [ - dev for dev in listdir('/sys/class/net', self.opts.sysroot) - if dev not in filt_devs - ] + try: + # Override checking sysroot here, as network devices will not + # be under the sysroot in live environments or in containers + # that are correctly setup to collect from the host + _eth_devs = [ + dev for dev in listdir('/sys/class/net', None) + if dev not in filt_devs + ] + except Exception as err: + self.soslog.warning( + f'Failed to manually determine network devices: {err}' + ) else: try: _nscmd = "ip netns exec %s ls /sys/class/net" % namespace From d53384f3ec9d6ac0f73f1e6800daa123cc9b6c97 Mon Sep 17 00:00:00 2001 From: Kevin Traynor Date: Mon, 24 Jul 2023 13:17:47 +0100 Subject: [PATCH 21/32] [openvswitch] Add shorter pmd rxq usage periods Since OVS 3.1 'ovs-appctl dpif-netdev/pmd-rxq-show' also supports showing the pmd rxq % usage over shorter timeframes than the default 1 minute, down to a granularity of 5 secs. A user testing may send traffic and have a steady state for less than 1 min, so it is useful to also see pmd rxq % usage over some shorter time periods. Add pmd-rxq-show stats for 5 and 30 secs. Signed-off-by: Kevin Traynor --- sos/report/plugins/openvswitch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sos/report/plugins/openvswitch.py b/sos/report/plugins/openvswitch.py index 5b32b2104a..98b08cb341 100644 --- a/sos/report/plugins/openvswitch.py +++ b/sos/report/plugins/openvswitch.py @@ -129,6 +129,8 @@ def setup(self): # Capture OVS datapath list "ovs-vsctl -t 5 list datapath", # Capture DPDK queue to pmd mapping + "ovs-appctl dpif-netdev/pmd-rxq-show -secs 5", + "ovs-appctl dpif-netdev/pmd-rxq-show -secs 30", "ovs-appctl dpif-netdev/pmd-rxq-show", # Capture DPDK pmd stats "ovs-appctl dpif-netdev/pmd-stats-show", From f62413b555c8f6d82801fa710410e31c826cf9ce Mon Sep 17 00:00:00 2001 From: Nagoor Shaik Date: Tue, 1 Aug 2023 12:07:06 +0530 Subject: [PATCH 22/32] Resolves: #3323 fetch id in foreman_auth_table Signed-off-by: Nagoor Shaik Signed-off-by: Nagoor Shaik --- sos/report/plugins/foreman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sos/report/plugins/foreman.py b/sos/report/plugins/foreman.py index 3ac3d3df47..4b0e95d38f 100644 --- a/sos/report/plugins/foreman.py +++ b/sos/report/plugins/foreman.py @@ -188,7 +188,7 @@ def setup(self): ) authcmd = ( - 'select type,name,host,port,account,base_dn,attr_login,' + 'select id,type,name,host,port,account,base_dn,attr_login,' 'onthefly_register,tls from auth_sources' ) From aad58d7c8366adfd9269687284aff975dcc79f5b Mon Sep 17 00:00:00 2001 From: illuminatus Date: Mon, 31 Jul 2023 12:27:59 -0700 Subject: [PATCH 23/32] [containerd] include containerd.io in packages set Signed-off-by: Trevor Benson --- sos/report/plugins/containerd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sos/report/plugins/containerd.py b/sos/report/plugins/containerd.py index 1977fe22fe..33231da1ab 100644 --- a/sos/report/plugins/containerd.py +++ b/sos/report/plugins/containerd.py @@ -14,7 +14,7 @@ class Containerd(Plugin, RedHatPlugin, UbuntuPlugin, CosPlugin): short_desc = 'Containerd containers' plugin_name = 'containerd' profiles = ('container',) - packages = ('containerd',) + packages = ('containerd', 'containerd.io',) def setup(self): self.add_copy_spec([ From 6d2334f1a942561f79eb4c855337156f2466185e Mon Sep 17 00:00:00 2001 From: Nikhil Kshirsagar Date: Tue, 25 Jul 2023 12:33:46 +0530 Subject: [PATCH 24/32] [plugins/ceph] Don't trigger ceph_common on cinder nodes ceph_common is triggered due to librados2 package. Remove it. Signed-off-by: Nikhil Kshirsagar --- sos/report/plugins/ceph_common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sos/report/plugins/ceph_common.py b/sos/report/plugins/ceph_common.py index 12c5d171d1..444e0f52ba 100644 --- a/sos/report/plugins/ceph_common.py +++ b/sos/report/plugins/ceph_common.py @@ -27,7 +27,6 @@ class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin): 'libcephfs1', 'ceph-fs-common', 'calamari-server', - 'librados2' ) services = ( From 5caabe01086478c9f4fc175f02e2e2c7f89d080e Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Wed, 2 Aug 2023 08:53:30 +0200 Subject: [PATCH 25/32] [discovery] Enable the plugin by containers Enable the plugin by running containers and collect some further system logs. Resolves: #3324 Signed-off-by: Pavel Moravec --- sos/report/plugins/discovery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sos/report/plugins/discovery.py b/sos/report/plugins/discovery.py index d593b4f756..e54132988b 100644 --- a/sos/report/plugins/discovery.py +++ b/sos/report/plugins/discovery.py @@ -16,12 +16,15 @@ class Discovery(Plugin, RedHatPlugin): short_desc = 'Discovery inspection and reporting tool' plugin_name = 'discovery' packages = ('discovery', 'discovery-tools',) + containers = ('dsc-db', 'discovery',) def setup(self): self.add_copy_spec([ "/root/discovery/db/volume/data/userdata/pg_log/", "/root/discovery/server/volumes/log/app.log", - "/root/discovery/server/volumes/log/discovery-server.log" + "/root/discovery/server/volumes/log/discovery-server.log", + "/var/lib/containers/storage/volumes/dsc-data/_data/userdata/log/", + "/var/discovery/server/volumes/log/", ]) self.add_container_logs([ From 2d3fc8f60389cab01b4b16a16a5824a85da1d1cf Mon Sep 17 00:00:00 2001 From: Nikhil Kshirsagar Date: Fri, 4 Aug 2023 18:22:53 +0530 Subject: [PATCH 26/32] [plugins/ubuntu] Do not load tls module if not loaded Adds a predicate to avoid loading the tls module if it is not loaded. Resolves: #3326 Signed-off-by: Nikhil Kshirsagar --- sos/report/plugins/ubuntu.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sos/report/plugins/ubuntu.py b/sos/report/plugins/ubuntu.py index 9fc3c3a18a..a6b1d6d37e 100644 --- a/sos/report/plugins/ubuntu.py +++ b/sos/report/plugins/ubuntu.py @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, UbuntuPlugin +from sos.report.plugins import Plugin, UbuntuPlugin, SoSPredicate from sos.utilities import is_executable @@ -30,8 +30,11 @@ def setup(self): ua_tools_status = 'pro status' else: ua_tools_status = 'ubuntu-advantage status' - self.add_cmd_output(ua_tools_status) - self.add_cmd_output("%s --format json" % ua_tools_status) + ua_pred = SoSPredicate(self, kmods=['tls']) + self.add_cmd_output(ua_tools_status, + pred=ua_pred, changes=True) + self.add_cmd_output("%s --format json" % ua_tools_status, + pred=ua_pred, changes=True) if not self.get_option("all_logs"): self.add_copy_spec([ From 46794f44814cb8ccaa9b9330a7ca1028a36db5d4 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Mon, 7 Aug 2023 10:55:17 +0200 Subject: [PATCH 27/32] [ultrapath] Add new plugin for Huawei UltraPath Relevant: rhbz2187407 Resolves: #3328 Author: Nitin U. Yewale Signed-off-by: Pavel Moravec --- sos/report/plugins/ultrapath.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 sos/report/plugins/ultrapath.py diff --git a/sos/report/plugins/ultrapath.py b/sos/report/plugins/ultrapath.py new file mode 100644 index 0000000000..2fd8735c91 --- /dev/null +++ b/sos/report/plugins/ultrapath.py @@ -0,0 +1,40 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, RedHatPlugin + + +class UltraPath(Plugin, RedHatPlugin): + + short_desc = 'HUAWEI UltraPath' + + plugin_name = 'ultrapath' + profiles = ('storage', 'hardware') + packages = ('UltraPath',) + kernel_mods = ('nxup', 'nxupext_a') + + def setup(self): + """ Huawei UltraPath specific information - commands + """ + self.add_cmd_output([ + "upadm show version", + "upadm show connectarray", + "upadm show option", + "upadm show upconfig", + "upadm show diskarray", + "upadmin show vlun", + ]) + + result = self.collect_cmd_output('upadm show path') + if result['status'] == 0: + for line in result['output'].splitlines(): + if line.startswith("Array ID :"): + self.add_cmd_output("upadm show lun array=%s" % + line.split(':')[1].strip()) + +# vim: set et ts=4 sw=4 : From 7545f7dbec5273cc36d09cae184e663dc0e84404 Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Mon, 7 Aug 2023 13:28:58 +0100 Subject: [PATCH 28/32] [slurm] Add slurm plugin Slurm is a workload manager in the HPC space, this is a start on this, and there may be further additions in the future. Signed-off-by: Arif Ali --- sos/report/plugins/slurm.py | 128 ++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 sos/report/plugins/slurm.py diff --git a/sos/report/plugins/slurm.py b/sos/report/plugins/slurm.py new file mode 100644 index 0000000000..f74bc9892b --- /dev/null +++ b/sos/report/plugins/slurm.py @@ -0,0 +1,128 @@ +# This file is part of the sos project: https://github.com/sosreport/sos +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# version 2 of the GNU General Public License. +# +# See the LICENSE file in the source distribution for further information. + +from sos.report.plugins import Plugin, UbuntuPlugin, RedHatPlugin +from sos.utilities import is_executable + + +class Slurm(Plugin, UbuntuPlugin, RedHatPlugin): + + short_desc = "Slurm Workload Manager" + + plugin_name = 'slurm' + profiles = ('hpc',) + packages = ( + # Ubuntu + 'slurm-wlm', + 'slurmd', + 'slurmdbd', + 'slurmctld', + # EL + 'slurm', + 'slurm-slurmctld', + 'slurm-slurmd', + 'slurm-slurmdbd', + ) + services = ( + 'slurmd', + 'slurmdbd', + 'slurmctld', + ) + + def setup(self): + """ Slurm Workload Manager + """ + + self.add_copy_spec('/etc/slurm/*.conf') + + if is_executable('sinfo'): + self.add_cmd_output([ + 'sinfo --all --list-reasons --long', + 'sinfo --all --long', + ]) + + if is_executable('squeue'): + self.add_cmd_output([ + 'squeue --all --long', + ]) + + scontrol_cmds = [ + 'aliases', + 'assoc_mgr', + 'bbstat', + 'burstBuffer', + 'config', + 'daemons', + 'dwstat', + 'federation', + 'frontend', + 'job', + 'licenses', + 'node', + 'partition', + 'reservation', + 'slurmd', + 'step', + 'topology', + ] + + if is_executable('scontrol'): + self.add_cmd_output( + [f"scontrol show {i}" for i in scontrol_cmds] + ) + + config_file = '/etc/slurm/slurm.conf' + + slurmd_log_file = '/var/log/slurmd.log' + slurmctld_log_file = '/var/log/slurmctld.log' + + try: + with open(config_file, 'r') as cf: + for line in cf.read().splitlines(): + if not line: + continue + words = line.split('=') + if words[0].strip() == 'SlurmdLogFile': + slurmd_log_file = words[1].strip() + if words[0].strip() == 'SlurmctldLogFile': + slurmctld_log_file = words[1].strip() + except IOError as error: + self._log_error('Could not open conf file %s: %s' % + (config_file, error)) + + if not self.get_option("all_logs"): + self.add_copy_spec([ + slurmd_log_file, + slurmctld_log_file, + ]) + else: + self.add_copy_spec([ + f"{slurmd_log_file}*", + f"{slurmctld_log_file}*", + ]) + + def postproc(self): + conf_paths = "/etc/slurm" + + slurm_keys = [ + 'AccountingStoragePass', + 'JobCompPass', + ] + slurm_keys_regex = r"(^\s*(%s)\s*=\s*)(.*)" % "|".join(slurm_keys) + slurmdbd_key_regex = r'(^\s*(StoragePass)\s*=\s*)(.*)' + + sub = r'\1********' + + self.do_file_sub( + f"{conf_paths}/slurm.conf", + slurm_keys_regex, sub + ) + self.do_file_sub( + f"{conf_paths}/slurmdbd.conf", + slurmdbd_key_regex, sub + ) From cac0cca511f82b861b32917bc72033afdc403f95 Mon Sep 17 00:00:00 2001 From: Jose Castillo Date: Sat, 12 Aug 2023 16:43:04 +0100 Subject: [PATCH 29/32] [flake 8] Fix two new style alerts Two new alerts: sos/report/plugins/release.py:9:52: E231 missing whitespace after ',' sos/report/plugins/__init__.py:456:21: E721 do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance() Signed-off-by: Jose Castillo --- sos/report/plugins/__init__.py | 2 +- sos/report/plugins/release.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sos/report/plugins/__init__.py b/sos/report/plugins/__init__.py index a4dfbb64aa..fd3818e553 100644 --- a/sos/report/plugins/__init__.py +++ b/sos/report/plugins/__init__.py @@ -453,7 +453,7 @@ def set_value(self, val): if type('') in self.val_type: self.value = str(val) return - if not any([type(val) == _t for _t in self.val_type]): + if not any([type(val) is _t for _t in self.val_type]): valid = [] for t in self.val_type: if t is None: diff --git a/sos/report/plugins/release.py b/sos/report/plugins/release.py index a57f37f1c0..7086ba779a 100644 --- a/sos/report/plugins/release.py +++ b/sos/report/plugins/release.py @@ -6,7 +6,7 @@ # # See the LICENSE file in the source distribution for further information. -from sos.report.plugins import Plugin, RedHatPlugin,\ +from sos.report.plugins import Plugin, RedHatPlugin, \ DebianPlugin, UbuntuPlugin, CosPlugin From abb4466841ebc2bce07217bd74bdf1f30c25bb66 Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Thu, 10 Aug 2023 00:56:32 +0100 Subject: [PATCH 30/32] [readme] Update for snap status and installation Signed-off-by: Arif Ali --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b1ec811dd..70db0b6394 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=main)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=main)](https://sos.readthedocs.io/en/main/?badge=main) +[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=main)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=main)](https://sos.readthedocs.io/en/main/?badge=main) [![sosreport](https://snapcraft.io/sosreport/badge.svg)](https://snapcraft.io/sosreport) + # SoS @@ -147,6 +148,12 @@ Ubuntu (14.04 LTS and above) users install via apt: ``` # sudo apt install sosreport +``` + +### Snap Installation + +``` +# snap install sosreport --classic ``` [0]: https://github.com/sosreport/sos/wiki/Contribution-Guidelines From 3ed37adb11cd7ddccecbcb72fc393ca8b6646715 Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Thu, 10 Aug 2023 10:57:10 +0100 Subject: [PATCH 31/32] [cirrus] Update gce images Signed-off-by: Arif Ali --- .cirrus.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 0b99935113..93103c7e38 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -22,15 +22,15 @@ env: UBUNTU_PROJECT: "ubuntu-os-cloud" # Images exist on GCP already - CENTOS_9_IMAGE_NAME: "centos-stream-9-v20221102" - CENTOS_8_IMAGE_NAME: "centos-stream-8-v20230306" - DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230306" + CENTOS_9_IMAGE_NAME: "centos-stream-9-v20230809" + CENTOS_8_IMAGE_NAME: "centos-stream-8-v20230809" + DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230809" FEDORA_IMAGE_NAME: "fedora-cloud-base-gcp-38-1-6-x86-64" FEDORA_PRIOR_IMAGE_NAME: "fedora-cloud-base-gcp-37-1-7-x86-64" - UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230302" - UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230302" - UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230324" - UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230302" + UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230727" + UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230724" + UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230605" + UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230727" # Curl-command prefix for downloading task artifacts, simply add the # the url-encoded task name, artifact name, and path as a suffix. From d71e6b508b1350b0ea56ef07ec2e872ae61d8ef1 Mon Sep 17 00:00:00 2001 From: Pavel Moravec Date: Tue, 8 Aug 2023 22:48:48 +0200 Subject: [PATCH 32/32] [cleaner] Use fully_trusted filter for extraction Since Python 3.12, archive.extractall would require setting an extraction filter otherwise an exception is raised. See PEP 706 for more. Since Python 3.10 and 3.11 can raise false alarms as exceptions (see #3330 for examples), we must use the legacy fully_trusted filter. Relevant: #3319 Closes: #3330 Signed-off-by: Pavel Moravec --- sos/cleaner/archives/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sos/cleaner/archives/__init__.py b/sos/cleaner/archives/__init__.py index 70e168d8b9..a729862d57 100644 --- a/sos/cleaner/archives/__init__.py +++ b/sos/cleaner/archives/__init__.py @@ -25,6 +25,12 @@ def extract_archive(archive_path, tmpdir): archive = tarfile.open(archive_path) path = os.path.join(tmpdir, 'cleaner') + # set extract filter since python 3.12 (see PEP-706 for more) + # Because python 3.10 and 3.11 raises false alarms as exceptions + # (see #3330 for examples), we can't use data filter but must + # fully trust the archive (legacy behaviour) + archive.extraction_filter = getattr(tarfile, 'fully_trusted_filter', + (lambda member, path: member)) archive.extractall(path) archive.close() return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0])