Add support for Pebble service manager

hotsos/core/host_helpers/__init__.py

@@ -13,13 +13,15 @@
     DockerImageHelper,
     SnapPackageHelper,
 )
+from .pebble import (  # noqa: F403,F401
+    PebbleHelper,
+)
 from .ssl import (  # noqa: F403,F401
     SSLCertificate,
     SSLCertificatesHelper,
 )
 from .systemd import (  # noqa: F403,F401
     SystemdHelper,
-    SVC_EXPR_TEMPLATES,
 )
 from .uptime import (  # noqa: F403,F401
     UptimeHelper,

hotsos/core/host_helpers/cli.py

@@ -513,6 +513,7 @@ def __call__(self, *args, **kwargs):
             return NullSource()()
 
         # binary sources only apply if data_root is system root
+        bin_out = None
         for bsource in [s for s in self.sources if s.TYPE == "BIN"]:
             cache = False
             # NOTE: we currently only support caching commands with no
@@ -524,18 +525,21 @@ def __call__(self, *args, **kwargs):
                     return out
 
             try:
-                out = bsource(*args, **kwargs)
+                bin_out = bsource(*args, **kwargs)
+                if cache and bin_out is not None:
+                    try:
+                        self.cache.save(self.cmdkey, bin_out)
+                    except pickle.PicklingError as exc:
+                        log.info("unable to cache command '%s' output: %s",
+                                 self.cmdkey, exc)
+
+                # if command executed but returned nothing that still counts
+                # as success.
+                break
             except CLIExecError as exc:
-                return exc.return_value
-
-            if cache and out is not None:
-                try:
-                    self.cache.save(self.cmdkey, out)
-                except pickle.PicklingError as exc:
-                    log.info("unable to cache command '%s' output: %s",
-                             self.cmdkey, exc)
+                bin_out = exc.return_value
 
-            return out
+        return bin_out
 
 
 class CLICacheWrapper(object):
@@ -837,6 +841,14 @@ def command_catalog(self):
             'pacemaker_crm_status':
                 [BinCmd('crm status'),
                  FileCmd('sos_commands/pacemaker/crm_status')],
+            'pebble_services':
+                [BinCmd('pebble services'),
+                 # This is how operator charms run it
+                 BinCmd('/charm/bin/pebble services'),
+                 # The following does not exist in sosreport yet but adding
+                 # since it is useful for testing and will hopefully be
+                 # supported in sos at some point.
+                 FileCmd('sos_commands/pebble/pebble_services')],
             'ps':
                 [BinCmd('ps auxwww'),
                  FileCmd('ps')],

hotsos/core/host_helpers/common.py

@@ -1,8 +1,10 @@
 import abc
+import re
 
 from searchkit.utils import MPCache
 
 from hotsos.core.config import HotSOSConfig
+from hotsos.core.log import log
 
 
 class HostHelpersBase(abc.ABC):
@@ -38,3 +40,61 @@ def cache_load(self):
     @abc.abstractmethod
     def cache_save(self):
         """ Save contents to cache. """
+
+
+class ServiceManagerBase(abc.ABC):
+    PS_CMD_EXPR_TEMPLATES = {
+        'absolute': r".+\S+bin/({})(?:\s+.+|$)",
+        'snap': r".+\S+\d+/({})(?:\s+.+|$)",
+        'relative': r".+\s({})(?:\s+.+|$)",
+    }
+
+    def __init__(self, service_exprs, ps_allow_relative=True):
+        """
+        @param service_exprs: list of python.re expressions used to match
+                              service names.
+        @param ps_allow_relative: whether to allow commands to be identified
+                                  from ps as run using an relative binary
+                                  path e.g. mycmd as opposed to /bin/mycmd.
+        """
+        self._ps_allow_relative = ps_allow_relative
+        self._service_exprs = set(service_exprs)
+
+    def get_cmd_from_ps_line(self, line, expr):
+        """
+        Match a command in ps output line.
+
+        @param line: line from ps output
+        @param expr: regex to match a command. See PS_CMD_EXPR_TEMPLATES.
+        @param return: matched command name.
+        """
+        for expr_type, expr_tmplt in self.PS_CMD_EXPR_TEMPLATES.items():
+            if expr_type == 'relative' and not self._ps_allow_relative:
+                continue
+
+            ret = re.compile(expr_tmplt.format(expr)).match(line)
+            if ret:
+                cmd = ret.group(1)
+                log.debug("matched command '%s' with expr type '%s'", cmd,
+                          expr_type)
+                return cmd
+
+    @property
+    @abc.abstractmethod
+    def services(self):
+        """ Return a dictionary of identified services and their state. """
+
+    @property
+    @abc.abstractmethod
+    def processes(self):
+        """
+        Return a dictionary of processes associated with identified
+        services.
+        """
+
+    @property
+    @abc.abstractmethod
+    def summary(self):
+        """ Return a dictionary summary of this class i.e. services,
+        their state and associated processes.
+        """

hotsos/core/host_helpers/pebble.py

@@ -0,0 +1,110 @@
+import re
+
+from hotsos.core.log import log
+from hotsos.core.factory import FactoryBase
+from hotsos.core.host_helpers import CLIHelper
+from hotsos.core.host_helpers.common import ServiceManagerBase
+from hotsos.core.utils import cached_property, sorted_dict
+
+
+class PebbleService(object):
+
+    def __init__(self, name, state):
+        self.name = name
+        self.state = state
+
+    def __repr__(self):
+        return "name={}, state={}".format(self.name, self.state)
+
+
+class PebbleHelper(ServiceManagerBase):
+    """ Helper class used to query pebble services. """
+
+    @cached_property
+    def services(self):  # pylint: disable=W0236
+        """ Return a dict of identified pebble services and their state. """
+        _services = {}
+        for line in CLIHelper().pebble_services():
+            for svc_name_expr in self._service_exprs:
+                _expr = r"({})\s+\S+\s+(\S+)\s+.*".format(svc_name_expr)
+                ret = re.compile(_expr).match(line)
+                if not ret:
+                    continue
+
+                name = ret.group(1)
+                _services[name] = PebbleService(name=name, state=ret.group(2))
+
+        return _services
+
+    @cached_property
+    def processes(self):  # pylint: disable=W0236
+        """
+        Identify running processes from ps that are associated with resolved
+        pebble services.  The search pattern used to identify a service is also
+        used to match the process binary name.
+
+        Returns a dictionary of process names along with the number of each.
+        """
+        _proc_info = {}
+        for line in CLIHelper().ps():
+            for expr in self._service_exprs:
+                """
+                look for running process with this name.
+                We need to account for different types of process binary e.g.
+
+                /snap/<name>/1830/<svc>
+                /usr/bin/<svc>
+
+                and filter e.g.
+
+                /var/lib/<svc> and /var/log/<svc>
+                """
+                cmd = self.get_cmd_from_ps_line(line, expr)
+                if cmd:
+                    if cmd in _proc_info:
+                        _proc_info[cmd] += 1
+                    else:
+                        _proc_info[cmd] = 1
+
+        return _proc_info
+
+    @property
+    def _service_info(self):
+        """Return a dictionary of pebble services grouped by state. """
+        info = {}
+        for svc, obj in sorted_dict(self.services).items():
+            state = obj.state
+            if state not in info:
+                info[state] = []
+
+            info[state].append(svc)
+
+        return info
+
+    @property
+    def _process_info(self):
+        """Return a list of processes associated with services. """
+        return ["{} ({})".format(name, count)
+                for name, count in sorted_dict(self.processes).items()]
+
+    @property
+    def summary(self):
+        """
+        Output a dict summary of this class i.e. services, their state and any
+        processes run by them.
+        """
+        return {'pebble': self._service_info,
+                'ps': self._process_info}
+
+
+class ServiceFactory(FactoryBase):
+    """
+    Factory to dynamically create PebbleService objects for given services.
+
+    Service objects are returned when a getattr() is done on this object using
+    the name of the service as the attr name.
+    """
+
+    def __getattr__(self, svc):
+        log.debug("creating service object for %s", svc)
+        return PebbleHelper([svc]).services.get(svc)

hotsos/core/host_helpers/systemd.py

@@ -8,14 +8,9 @@
 from hotsos.core.config import HotSOSConfig
 from hotsos.core.factory import FactoryBase
 from hotsos.core.host_helpers import CLIHelper
+from hotsos.core.host_helpers.common import ServiceManagerBase
 from hotsos.core.utils import cached_property, sorted_dict
 
-SVC_EXPR_TEMPLATES = {
-    "absolute": r".+\S+bin/({})(?:\s+.+|$)",
-    "snap": r".+\S+\d+/({})(?:\s+.+|$)",
-    "relative": r".+\s({})(?:\s+.+|$)",
-    }
-
 
 class SystemdService(object):
 
@@ -64,19 +59,11 @@ def __repr__(self):
                        self.has_instances))
 
 
-class SystemdHelper(object):
+class SystemdHelper(ServiceManagerBase):
     """ Helper class used to query systemd services. """
 
-    def __init__(self, service_exprs, ps_allow_relative=True):
-        """
-        @param service_exprs: list of python.re expressions used to match
-                              service names.
-        @param ps_allow_relative: whether to allow commands to be identified
-                                  from ps as run using an relative binary
-                                  path e.g. mycmd as opposed to /bin/mycmd.
-        """
-        self._ps_allow_relative = ps_allow_relative
-        self._service_exprs = set(service_exprs)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self._cached_unit_files_exprs = {}
 
     @cached_property
@@ -125,7 +112,7 @@ def _unit_files_expr(self, svc_name_expr):
         return self._cached_unit_files_exprs[svc_name_expr]
 
     @cached_property
-    def services(self):
+    def services(self):  # pylint: disable=W0236
         """
         Return a dict of identified systemd services and their state.
 
@@ -190,18 +177,6 @@ def masked_services(self):
 
         return self._service_info.get('masked', [])
 
-    def get_process_cmd_from_line(self, line, expr):
-        for expr_type, expr_tmplt in SVC_EXPR_TEMPLATES.items():
-            if expr_type == 'relative' and not self._ps_allow_relative:
-                continue
-
-            ret = re.compile(expr_tmplt.format(expr)).match(line)
-            if ret:
-                svc = ret.group(1)
-                log.debug("matched process %s with %s expr", svc,
-                          expr_type)
-                return svc
-
     def get_services_expanded(self, name):
         _expanded = []
         for line in self._systemctl_list_units:
@@ -256,11 +231,11 @@ def _service_filtered_ps(self):
         return ps_filtered
 
     @cached_property
-    def processes(self):
+    def processes(self):  # pylint: disable=W0236
         """
         Identify running processes from ps that are associated with resolved
-        systemd services. The same search pattern used for identifying systemd
-        services is to match the process binary name.
+        systemd services. The search pattern used to identify a service is also
+        used to match the process binary name.
 
         Returns a dictionary of process names along with the number of each.
         """
@@ -278,12 +253,14 @@ def processes(self):
 
                 /var/lib/<svc> and /var/log/<svc>
                 """
-                cmd = self.get_process_cmd_from_line(line, expr)
-                if cmd:
-                    if cmd in _proc_info:
-                        _proc_info[cmd] += 1
-                    else:
-                        _proc_info[cmd] = 1
+                cmd = self.get_cmd_from_ps_line(line, expr)
+                if not cmd:
+                    continue
+
+                if cmd in _proc_info:
+                    _proc_info[cmd] += 1
+                else:
+                    _proc_info[cmd] = 1
 
         return _proc_info
 

hotsos/core/plugins/juju/common.py

@@ -1,7 +1,7 @@
 import os
 
 from hotsos.core import plugintools
-from hotsos.core.host_helpers import SystemdHelper
+from hotsos.core.host_helpers import PebbleHelper, SystemdHelper
 from hotsos.core.plugins.juju.resources import JujuBase
 
 # matches date and time at start if log lines
@@ -19,6 +19,7 @@ class JujuChecksBase(plugintools.PluginPartBase, JujuBase):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        self.pebble = PebbleHelper(service_exprs=JUJU_SVC_EXPRS)
         self.systemd = SystemdHelper(service_exprs=JUJU_SVC_EXPRS)
         # this is needed for juju scenarios
         self.systemd_processes = self.systemd.processes

hotsos/core/plugins/kubernetes.py

@@ -5,6 +5,7 @@
 from hotsos.core.host_helpers import (
     APTPackageHelper,
     HostNetworkingHelper,
+    PebbleHelper,
     SnapPackageHelper,
     SystemdHelper,
 )
@@ -107,6 +108,7 @@ def __init__(self, *args, **kwargs):
         snap_deps = deps + K8S_PACKAGE_DEPS_SNAP
         self.snaps = SnapPackageHelper(core_snaps=K8S_PACKAGES,
                                        other_snaps=snap_deps)
+        self.pebble = PebbleHelper(service_exprs=SERVICES)
         self.systemd = SystemdHelper(service_exprs=SERVICES)
 
     @property