[grafana] generic logic applied to collect

Closes: #3278

Signed-off-by: Pravesh Gaire <[email protected]>

.cirrus.yml

@@ -22,15 +22,15 @@ env:
     UBUNTU_PROJECT: "ubuntu-os-cloud"
 
     # Images exist on GCP already
-    CENTOS_9_IMAGE_NAME: "centos-stream-9-v20221102"
-    CENTOS_8_IMAGE_NAME: "centos-stream-8-v20230306"
-    DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230306"
+    CENTOS_9_IMAGE_NAME: "centos-stream-9-v20230809"
+    CENTOS_8_IMAGE_NAME: "centos-stream-8-v20230809"
+    DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230809"
     FEDORA_IMAGE_NAME: "fedora-cloud-base-gcp-38-1-6-x86-64"
     FEDORA_PRIOR_IMAGE_NAME: "fedora-cloud-base-gcp-37-1-7-x86-64"
-    UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230302"
-    UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230302"
-    UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230324"
-    UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230302"
+    UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230727"
+    UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230724"
+    UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230605"
+    UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230727"
 
     # Curl-command prefix for downloading task artifacts, simply add the
     # the url-encoded task name, artifact name, and path as a suffix.

.github/workflows/snap.yaml

@@ -3,10 +3,13 @@ on:
   push:
     branches:
       - main
+  release:
+    types:
+      - published
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     concurrency:
       group: snap-build
       cancel-in-progress: true
@@ -25,8 +28,16 @@ jobs:
     - run: |
         sudo sos help
     - uses: snapcore/action-publish@v1
+      if: ${{ github.event_name == 'push' }}
       env:
         SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
       with:
         snap: ${{ steps.build-snap.outputs.snap }}
         release: "latest/edge"
+    - uses: snapcore/action-publish@v1
+      if: ${{ github.event_name == 'release' }}
+      env:
+        SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
+      with:
+        snap: ${{ steps.build-snap.outputs.snap }}
+        release: "latest/candidate"

README.md

@@ -1,4 +1,5 @@
-[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=main)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=main)](https://sos.readthedocs.io/en/main/?badge=main)
+[![Build Status](https://api.cirrus-ci.com/github/sosreport/sos.svg?branch=main)](https://cirrus-ci.com/github/sosreport/sos) [![Documentation Status](https://readthedocs.org/projects/sos/badge/?version=main)](https://sos.readthedocs.io/en/main/?badge=main) [![sosreport](https://snapcraft.io/sosreport/badge.svg)](https://snapcraft.io/sosreport)
+
 
 # SoS
 
@@ -147,6 +148,12 @@ Ubuntu (14.04 LTS and above) users install via apt:
 
 ```
 # sudo apt install sosreport
+```
+
+### Snap Installation
+
+```
+# snap install sosreport --classic
 ```
 
  [0]: https://github.com/sosreport/sos/wiki/Contribution-Guidelines

docs/conf.py

@@ -59,9 +59,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '4.5.5'
+version = '4.5.6'
 # The full version, including alpha/beta/rc tags.
-release = '4.5.5'
+release = '4.5.6'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

man/en/sos-clean.1

@@ -135,7 +135,7 @@ on the system from a previous execution.
 
 .SH MAINTAINER
 .nf
-Jake Hunsaker <jhunsake@redhat.com>
+Maintained on GitHub at https://github.com/sosreport/sos
 .fi
 .SH AUTHORS & CONTRIBUTORS
 See \fBAUTHORS\fR file in the package documentation.

man/en/sos-collect.1

@@ -395,4 +395,8 @@ Sosreport option. Override the default compression type.
 .BR sos.conf (5)
 
 .SH MAINTAINER
-    Jake Hunsaker <[email protected]>
+.nf
+Maintained on GitHub at https://github.com/sosreport/sos
+.fi
+.SH AUTHORS & CONTRIBUTORS
+See \fBAUTHORS\fR file in the package documentation.

man/en/sos-report.1

@@ -494,7 +494,7 @@ Display usage message.
 
 .SH MAINTAINER
 .nf
-Jake Hunsaker <jhunsake@redhat.com>
+Maintained on GitHub at https://github.com/sosreport/sos
 .fi
 .SH AUTHORS & CONTRIBUTORS
 See \fBAUTHORS\fR file in the package documentation.

man/en/sos.1

@@ -165,7 +165,7 @@ Display usage message.
 .BR sos.conf (5)
 .SH MAINTAINER
 .nf
-Jake Hunsaker <jhunsake@redhat.com>
+Maintained on GitHub at https://github.com/sosreport/sos
 .fi
 .SH AUTHORS & CONTRIBUTORS
 See \fBAUTHORS\fR file in the package documentation.

snap/snapcraft.yaml

@@ -26,6 +26,7 @@ parts:
       - python3
       - snapcraft
       - gettext
+      - python3-venv
     stage-packages:
       - python3.10-minimal
       - libpython3.10-minimal

sos.spec

@@ -1,6 +1,6 @@
 Summary: A set of tools to gather troubleshooting information from a system
 Name: sos
-Version: 4.5.5
+Version: 4.5.6
 Release: 1%{?dist}
 Source0: https://github.com/sosreport/sos/archive/%{name}-%{version}.tar.gz
 License: GPL-2.0-or-later
@@ -86,6 +86,9 @@ rm -rf %{buildroot}/usr/config/
 %config(noreplace) %{_sysconfdir}/sos/sos.conf
 
 %changelog
+* Thu Jul 20 2023 Jake Hunsaker <[email protected]> = 4.5.6
+- New upstream release
+
 * Fri Jun 23 2023 Jake Hunsaker <[email protected]> = 4.5.5
 - New upstream release
 

sos/__init__.py

@@ -14,7 +14,7 @@
 This module houses the i18n setup and message function. The default is to use
 gettext to internationalize messages.
 """
-__version__ = "4.5.5"
+__version__ = "4.5.6"
 
 import os
 import sys

sos/cleaner/__init__.py

@@ -778,7 +778,7 @@ def obfuscate_file(self, filename, short_name=None, arc_name=None):
                                        % (short_name, err), caller=arc_name)
             tfile.seek(0)
             if subs:
-                shutil.copy(tfile.name, filename)
+                shutil.copyfile(tfile.name, filename)
             tfile.close()
 
         _ob_short_name = self.obfuscate_string(short_name.split('/')[-1])

sos/cleaner/archives/__init__.py

@@ -25,6 +25,12 @@
 def extract_archive(archive_path, tmpdir):
     archive = tarfile.open(archive_path)
     path = os.path.join(tmpdir, 'cleaner')
+    # set extract filter since python 3.12 (see PEP-706 for more)
+    # Because python 3.10 and 3.11 raises false alarms as exceptions
+    # (see #3330 for examples), we can't use data filter but must
+    # fully trust the archive (legacy behaviour)
+    archive.extraction_filter = getattr(tarfile, 'fully_trusted_filter',
+                                        (lambda member, path: member))
     archive.extractall(path)
     archive.close()
     return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0])

sos/cleaner/mappings/mac_map.py

@@ -75,5 +75,5 @@ def sanitize_item(self, item):
         if re.match('(([0-9a-fA-F]{4}:){3}([0-9a-fA-F]){4})', item):
             return self.mac6_quad_template % hextets
         # match 48-bit IPv4 MAC addresses
-        if re.match('([0-9a-fA-F]:?){12}', item):
+        if re.match('([0-9a-fA-F][:_]?){12}', item):
             return self.mac_template % hextets

sos/cleaner/parsers/mac_parser.py

@@ -25,8 +25,8 @@
 )
 # aa:bb:cc:dd:ee:ff avoiding ipv6 substring matches
 IPV4_REG = (
-    r'((?<!([0-9a-fA-F\'\"]:)|::)(([^:\-])?([0-9a-fA-F]{2}([:-])){5}'
-    r'([0-9a-fA-F]){2}(\'|\")?(\/|\,|\-|\.|\s|$)))'
+    r'((?<!([0-9a-fA-F\'\"]:)|::)'
+    r'(([^:\-])?(([0-9a-fA-F]{2}([:\-\_])){5,6}([0-9a-fA-F]{2}))))'
 )
 
 

sos/options.py

@@ -226,7 +226,7 @@ def _update_from_section(section, config):
         try:
             try:
                 with open(config_file) as f:
-                    config.readfp(f)
+                    config.read_file(f, config_file)
             except DuplicateOptionError as err:
                 raise exit("Duplicate option '%s' in section '%s' in file %s"
                            % (err.option, err.section, config_file))

sos/report/__init__.py

@@ -612,10 +612,18 @@ def _get_eth_devs(self, namespace=None):
         filt_devs = ['bonding_masters']
         _eth_devs = []
         if not namespace:
-            _eth_devs = [
-                dev for dev in listdir('/sys/class/net', self.opts.sysroot)
-                if dev not in filt_devs
-            ]
+            try:
+                # Override checking sysroot here, as network devices will not
+                # be under the sysroot in live environments or in containers
+                # that are correctly setup to collect from the host
+                _eth_devs = [
+                    dev for dev in listdir('/sys/class/net', None)
+                    if dev not in filt_devs
+                ]
+            except Exception as err:
+                self.soslog.warning(
+                    f'Failed to manually determine network devices: {err}'
+                )
         else:
             try:
                 _nscmd = "ip netns exec %s ls /sys/class/net" % namespace

sos/report/plugins/__init__.py

@@ -453,7 +453,7 @@ def set_value(self, val):
         if type('') in self.val_type:
             self.value = str(val)
             return
-        if not any([type(val) == _t for _t in self.val_type]):
+        if not any([type(val) is _t for _t in self.val_type]):
             valid = []
             for t in self.val_type:
                 if t is None:
@@ -1274,12 +1274,13 @@ def do_file_sub(self, srcpath, regexp, subst):
         """
         try:
             path = self._get_dest_for_srcpath(srcpath)
+            common_flags = re.IGNORECASE | re.MULTILINE
             if hasattr(regexp, "pattern"):
                 pattern = regexp.pattern
-                flags = regexp.flags | re.IGNORECASE
+                flags = regexp.flags | common_flags
             else:
                 pattern = regexp
-                flags = re.IGNORECASE
+                flags = common_flags
             self._log_debug("substituting scrpath '%s'" % srcpath)
             self._log_debug("substituting '%s' for '%s' in '%s'"
                             % (subst, pattern, path))

sos/report/plugins/ceph_ansible.py

@@ -14,7 +14,7 @@ class CephAnsible(Plugin, RedHatPlugin, DebianPlugin):
     short_desc = 'CEPH distributed storage - Ansible installer'
 
     plugin_name = 'ceph_ansible'
-    profiles = ('storage',)
+    profiles = ('storage', 'ceph')
 
     packages = ('ceph-ansible',)
 

sos/report/plugins/ceph_common.py

@@ -15,7 +15,7 @@ class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin):
     short_desc = 'CEPH common'
 
     plugin_name = 'ceph_common'
-    profiles = ('storage', 'virt', 'container')
+    profiles = ('storage', 'virt', 'container', 'ceph')
 
     containers = ('ceph-(.*-)?(mon|rgw|osd).*',)
     ceph_hostname = gethostname()
@@ -27,7 +27,6 @@ class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin):
         'libcephfs1',
         'ceph-fs-common',
         'calamari-server',
-        'librados2'
     )
 
     services = (
@@ -41,45 +40,53 @@ class Ceph_Common(Plugin, RedHatPlugin, UbuntuPlugin):
 
     # This check will enable the plugin regardless of being
     # containerized or not
-    files = ('/etc/ceph/ceph.conf',)
+    files = ('/etc/ceph/ceph.conf',
+             '/var/snap/microceph/*',)
 
     def setup(self):
         all_logs = self.get_option("all_logs")
 
-        self.add_file_tags({
-            '.*/ceph.conf': 'ceph_conf',
-            '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log',
-        })
-
-        if not all_logs:
-            self.add_copy_spec("/var/log/calamari/*.log",)
+        microceph_pkg = self.policy.package_manager.pkg_by_name('microceph')
+        if not microceph_pkg:
+            self.add_file_tags({
+                '.*/ceph.conf': 'ceph_conf',
+                '/var/log/ceph(.*)?/ceph.log.*': 'ceph_log',
+            })
+
+            if not all_logs:
+                self.add_copy_spec("/var/log/calamari/*.log",)
+            else:
+                self.add_copy_spec("/var/log/calamari",)
+
+            self.add_copy_spec([
+                "/var/log/ceph/**/ceph.log",
+                "/var/log/ceph/**/ceph.audit.log*",
+                "/var/log/calamari/*.log",
+                "/etc/ceph/",
+                "/etc/calamari/",
+                "/var/lib/ceph/tmp/",
+            ])
+
+            self.add_forbidden_path([
+                "/etc/ceph/*keyring*",
+                "/var/lib/ceph/*keyring*",
+                "/var/lib/ceph/*/*keyring*",
+                "/var/lib/ceph/*/*/*keyring*",
+                "/var/lib/ceph/osd",
+                "/var/lib/ceph/mon",
+                # Excludes temporary ceph-osd mount location like
+                # /var/lib/ceph/tmp/mnt.XXXX from sos collection.
+                "/var/lib/ceph/tmp/*mnt*",
+                "/etc/ceph/*bindpass*"
+            ])
         else:
-            self.add_copy_spec("/var/log/calamari",)
-
-        self.add_copy_spec([
-            "/var/log/ceph/**/ceph.log",
-            "/var/log/ceph/**/ceph.audit.log*",
-            "/var/log/calamari/*.log",
-            "/etc/ceph/",
-            "/etc/calamari/",
-            "/var/lib/ceph/tmp/",
-        ])
+            self.add_copy_spec([
+                "/var/snap/microceph/common/logs/ceph.log",
+                "/var/snap/microceph/common/logs/ceph.audit.log",
+            ])
 
         self.add_cmd_output([
             "ceph -v",
         ])
 
-        self.add_forbidden_path([
-            "/etc/ceph/*keyring*",
-            "/var/lib/ceph/*keyring*",
-            "/var/lib/ceph/*/*keyring*",
-            "/var/lib/ceph/*/*/*keyring*",
-            "/var/lib/ceph/osd",
-            "/var/lib/ceph/mon",
-            # Excludes temporary ceph-osd mount location like
-            # /var/lib/ceph/tmp/mnt.XXXX from sos collection.
-            "/var/lib/ceph/tmp/*mnt*",
-            "/etc/ceph/*bindpass*"
-        ])
-
 # vim: set et ts=4 sw=4 :

sos/report/plugins/ceph_iscsi.py

@@ -14,7 +14,7 @@ class CephISCSI(Plugin, RedHatPlugin, UbuntuPlugin):
     short_desc = "CEPH iSCSI"
 
     plugin_name = "ceph_iscsi"
-    profiles = ("storage", "virt", "container")
+    profiles = ("storage", "virt", "container", "ceph")
     packages = ("ceph-iscsi",)
     services = ("rbd-target-api", "rbd-target-gw")
     containers = ("rbd-target-api.*", "rbd-target-gw.*")

sos/report/plugins/ceph_mds.py

@@ -12,9 +12,9 @@
 class CephMDS(Plugin, RedHatPlugin, UbuntuPlugin):
     short_desc = 'CEPH mds'
     plugin_name = 'ceph_mds'
-    profiles = ('storage', 'virt', 'container')
+    profiles = ('storage', 'virt', 'container', 'ceph')
     containers = ('ceph-(.*-)?fs.*',)
-    files = ('/var/lib/ceph/mds/',)
+    files = ('/var/lib/ceph/mds/*',)
 
     def setup(self):
         self.add_file_tags({

sos/report/plugins/ceph_mgr.py

@@ -35,8 +35,8 @@ class CephMGR(Plugin, RedHatPlugin, UbuntuPlugin):
     short_desc = 'CEPH mgr'
 
     plugin_name = 'ceph_mgr'
-    profiles = ('storage', 'virt', 'container')
-    files = ('/var/lib/ceph/mgr/', '/var/lib/ceph/*/mgr*')
+    profiles = ('storage', 'virt', 'container', 'ceph')
+    files = ('/var/lib/ceph/mgr/*', '/var/lib/ceph/*/mgr*')
     containers = ('ceph-(.*-)?mgr.*',)
 
     def setup(self):