From 44cfa7646f2f7f9b6b2b36495fb6b7b308ba7733 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Wed, 26 Apr 2023 11:23:41 +0100 Subject: [PATCH] Add support for Pebble service manager Also extends CLIHelper to support trying more than one binary command. Note that pebble info is not yet supported in sosreport so this functionality is only available when running against a host. Adapts openstack plugin to fallback to pebble if systemd services not detected. Resolves: #584 --- hotsos/core/host_helpers/__init__.py | 4 +- hotsos/core/host_helpers/cli.py | 32 +++-- hotsos/core/host_helpers/common.py | 60 ++++++++++ hotsos/core/host_helpers/pebble.py | 110 ++++++++++++++++++ hotsos/core/host_helpers/systemd.py | 55 +++------ hotsos/core/plugins/juju/common.py | 3 +- hotsos/core/plugins/kubernetes.py | 2 + hotsos/core/plugins/mysql.py | 2 + hotsos/core/plugins/openstack/common.py | 6 +- hotsos/core/plugins/openstack/openstack.py | 6 + hotsos/core/plugins/openvswitch/common.py | 7 +- hotsos/core/plugins/rabbitmq/common.py | 7 +- hotsos/core/plugins/storage/ceph.py | 5 +- hotsos/core/plugins/vault.py | 7 +- .../engine/properties/requires/__init__.py | 1 + .../engine/properties/requires/common.py | 48 ++++++++ .../properties/requires/types/systemd.py | 47 +------- hotsos/defs/README.md | 33 ++++++ .../openstack/octavia/hm_port_health.yaml | 6 +- .../storage/ceph/ceph-mon/autoscaler_bug.yaml | 1 + .../storage/ceph/ceph-mon/ceph-mon.yaml | 6 +- hotsos/plugin_extensions/juju/summary.py | 2 + .../plugin_extensions/kubernetes/summary.py | 5 +- hotsos/plugin_extensions/mysql/summary.py | 2 + hotsos/plugin_extensions/openstack/summary.py | 2 + .../plugin_extensions/openvswitch/summary.py | 2 + hotsos/plugin_extensions/rabbitmq/summary.py | 2 + .../plugin_extensions/storage/ceph_summary.py | 2 + hotsos/plugin_extensions/vault/summary.py | 2 + tests/unit/test_host_helpers.py | 32 +++++ 30 files changed, 392 insertions(+), 107 deletions(-) create mode 100644 hotsos/core/host_helpers/pebble.py diff --git a/hotsos/core/host_helpers/__init__.py b/hotsos/core/host_helpers/__init__.py index bb6b8e894..cf3dc3671 100644 --- a/hotsos/core/host_helpers/__init__.py +++ b/hotsos/core/host_helpers/__init__.py @@ -13,13 +13,15 @@ DockerImageHelper, SnapPackageHelper, ) +from .pebble import ( # noqa: F403,F401 + PebbleHelper, +) from .ssl import ( # noqa: F403,F401 SSLCertificate, SSLCertificatesHelper, ) from .systemd import ( # noqa: F403,F401 SystemdHelper, - SVC_EXPR_TEMPLATES, ) from .uptime import ( # noqa: F403,F401 UptimeHelper, diff --git a/hotsos/core/host_helpers/cli.py b/hotsos/core/host_helpers/cli.py index cb672be7d..2b29802e5 100644 --- a/hotsos/core/host_helpers/cli.py +++ b/hotsos/core/host_helpers/cli.py @@ -513,6 +513,7 @@ def __call__(self, *args, **kwargs): return NullSource()() # binary sources only apply if data_root is system root + bin_out = None for bsource in [s for s in self.sources if s.TYPE == "BIN"]: cache = False # NOTE: we currently only support caching commands with no @@ -524,18 +525,21 @@ def __call__(self, *args, **kwargs): return out try: - out = bsource(*args, **kwargs) + bin_out = bsource(*args, **kwargs) + if cache and bin_out is not None: + try: + self.cache.save(self.cmdkey, bin_out) + except pickle.PicklingError as exc: + log.info("unable to cache command '%s' output: %s", + self.cmdkey, exc) + + # if command executed but returned nothing that still counts + # as success. + break except CLIExecError as exc: - return exc.return_value - - if cache and out is not None: - try: - self.cache.save(self.cmdkey, out) - except pickle.PicklingError as exc: - log.info("unable to cache command '%s' output: %s", - self.cmdkey, exc) + bin_out = exc.return_value - return out + return bin_out class CLICacheWrapper(object): @@ -837,6 +841,14 @@ def command_catalog(self): 'pacemaker_crm_status': [BinCmd('crm status'), FileCmd('sos_commands/pacemaker/crm_status')], + 'pebble_services': + [BinCmd('pebble services'), + # This is how operator charms run it + BinCmd('/charm/bin/pebble services'), + # The following does not exist in sosreport yet but adding + # since it is useful for testing and will hopefully be + # supported in sos at some point. + FileCmd('sos_commands/pebble/pebble_services')], 'ps': [BinCmd('ps auxwww'), FileCmd('ps')], diff --git a/hotsos/core/host_helpers/common.py b/hotsos/core/host_helpers/common.py index 627348026..63c24b5af 100644 --- a/hotsos/core/host_helpers/common.py +++ b/hotsos/core/host_helpers/common.py @@ -1,8 +1,10 @@ import abc +import re from searchkit.utils import MPCache from hotsos.core.config import HotSOSConfig +from hotsos.core.log import log class HostHelpersBase(abc.ABC): @@ -38,3 +40,61 @@ def cache_load(self): @abc.abstractmethod def cache_save(self): """ Save contents to cache. """ + + +class ServiceManagerBase(abc.ABC): + PS_CMD_EXPR_TEMPLATES = { + 'absolute': r".+\S+bin/({})(?:\s+.+|$)", + 'snap': r".+\S+\d+/({})(?:\s+.+|$)", + 'relative': r".+\s({})(?:\s+.+|$)", + } + + def __init__(self, service_exprs, ps_allow_relative=True): + """ + @param service_exprs: list of python.re expressions used to match + service names. + @param ps_allow_relative: whether to allow commands to be identified + from ps as run using an relative binary + path e.g. mycmd as opposed to /bin/mycmd. + """ + self._ps_allow_relative = ps_allow_relative + self._service_exprs = set(service_exprs) + + def get_cmd_from_ps_line(self, line, expr): + """ + Match a command in ps output line. + + @param line: line from ps output + @param expr: regex to match a command. See PS_CMD_EXPR_TEMPLATES. + @param return: matched command name. + """ + for expr_type, expr_tmplt in self.PS_CMD_EXPR_TEMPLATES.items(): + if expr_type == 'relative' and not self._ps_allow_relative: + continue + + ret = re.compile(expr_tmplt.format(expr)).match(line) + if ret: + cmd = ret.group(1) + log.debug("matched command '%s' with expr type '%s'", cmd, + expr_type) + return cmd + + @property + @abc.abstractmethod + def services(self): + """ Return a dictionary of identified services and their state. """ + + @property + @abc.abstractmethod + def processes(self): + """ + Return a dictionary of processes associated with identified + services. + """ + + @property + @abc.abstractmethod + def summary(self): + """ Return a dictionary summary of this class i.e. services, + their state and associated processes. + """ diff --git a/hotsos/core/host_helpers/pebble.py b/hotsos/core/host_helpers/pebble.py new file mode 100644 index 000000000..d705f664b --- /dev/null +++ b/hotsos/core/host_helpers/pebble.py @@ -0,0 +1,110 @@ +import re + +from hotsos.core.log import log +from hotsos.core.factory import FactoryBase +from hotsos.core.host_helpers import CLIHelper +from hotsos.core.host_helpers.common import ServiceManagerBase +from hotsos.core.utils import cached_property, sorted_dict + + +class PebbleService(object): + + def __init__(self, name, state): + self.name = name + self.state = state + + def __repr__(self): + return "name={}, state={}".format(self.name, self.state) + + +class PebbleHelper(ServiceManagerBase): + """ Helper class used to query pebble services. """ + + @cached_property + def services(self): # pylint: disable=W0236 + """ Return a dict of identified pebble services and their state. """ + _services = {} + for line in CLIHelper().pebble_services(): + for svc_name_expr in self._service_exprs: + _expr = r"({})\s+\S+\s+(\S+)\s+.*".format(svc_name_expr) + ret = re.compile(_expr).match(line) + if not ret: + continue + + name = ret.group(1) + _services[name] = PebbleService(name=name, state=ret.group(2)) + + return _services + + @cached_property + def processes(self): # pylint: disable=W0236 + """ + Identify running processes from ps that are associated with resolved + pebble services. The search pattern used to identify a service is also + used to match the process binary name. + + Returns a dictionary of process names along with the number of each. + """ + _proc_info = {} + for line in CLIHelper().ps(): + for expr in self._service_exprs: + """ + look for running process with this name. + We need to account for different types of process binary e.g. + + /snap//1830/ + /usr/bin/ + + and filter e.g. + + /var/lib/ and /var/log/ + """ + cmd = self.get_cmd_from_ps_line(line, expr) + if cmd: + if cmd in _proc_info: + _proc_info[cmd] += 1 + else: + _proc_info[cmd] = 1 + + return _proc_info + + @property + def _service_info(self): + """Return a dictionary of pebble services grouped by state. """ + info = {} + for svc, obj in sorted_dict(self.services).items(): + state = obj.state + if state not in info: + info[state] = [] + + info[state].append(svc) + + return info + + @property + def _process_info(self): + """Return a list of processes associated with services. """ + return ["{} ({})".format(name, count) + for name, count in sorted_dict(self.processes).items()] + + @property + def summary(self): + """ + Output a dict summary of this class i.e. services, their state and any + processes run by them. + """ + return {'pebble': self._service_info, + 'ps': self._process_info} + + +class ServiceFactory(FactoryBase): + """ + Factory to dynamically create PebbleService objects for given services. + + Service objects are returned when a getattr() is done on this object using + the name of the service as the attr name. + """ + + def __getattr__(self, svc): + log.debug("creating service object for %s", svc) + return PebbleHelper([svc]).services.get(svc) diff --git a/hotsos/core/host_helpers/systemd.py b/hotsos/core/host_helpers/systemd.py index 8bc7eb095..c200ab039 100644 --- a/hotsos/core/host_helpers/systemd.py +++ b/hotsos/core/host_helpers/systemd.py @@ -8,14 +8,9 @@ from hotsos.core.config import HotSOSConfig from hotsos.core.factory import FactoryBase from hotsos.core.host_helpers import CLIHelper +from hotsos.core.host_helpers.common import ServiceManagerBase from hotsos.core.utils import cached_property, sorted_dict -SVC_EXPR_TEMPLATES = { - "absolute": r".+\S+bin/({})(?:\s+.+|$)", - "snap": r".+\S+\d+/({})(?:\s+.+|$)", - "relative": r".+\s({})(?:\s+.+|$)", - } - class SystemdService(object): @@ -64,19 +59,11 @@ def __repr__(self): self.has_instances)) -class SystemdHelper(object): +class SystemdHelper(ServiceManagerBase): """ Helper class used to query systemd services. """ - def __init__(self, service_exprs, ps_allow_relative=True): - """ - @param service_exprs: list of python.re expressions used to match - service names. - @param ps_allow_relative: whether to allow commands to be identified - from ps as run using an relative binary - path e.g. mycmd as opposed to /bin/mycmd. - """ - self._ps_allow_relative = ps_allow_relative - self._service_exprs = set(service_exprs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._cached_unit_files_exprs = {} @cached_property @@ -125,7 +112,7 @@ def _unit_files_expr(self, svc_name_expr): return self._cached_unit_files_exprs[svc_name_expr] @cached_property - def services(self): + def services(self): # pylint: disable=W0236 """ Return a dict of identified systemd services and their state. @@ -190,18 +177,6 @@ def masked_services(self): return self._service_info.get('masked', []) - def get_process_cmd_from_line(self, line, expr): - for expr_type, expr_tmplt in SVC_EXPR_TEMPLATES.items(): - if expr_type == 'relative' and not self._ps_allow_relative: - continue - - ret = re.compile(expr_tmplt.format(expr)).match(line) - if ret: - svc = ret.group(1) - log.debug("matched process %s with %s expr", svc, - expr_type) - return svc - def get_services_expanded(self, name): _expanded = [] for line in self._systemctl_list_units: @@ -256,11 +231,11 @@ def _service_filtered_ps(self): return ps_filtered @cached_property - def processes(self): + def processes(self): # pylint: disable=W0236 """ Identify running processes from ps that are associated with resolved - systemd services. The same search pattern used for identifying systemd - services is to match the process binary name. + systemd services. The search pattern used to identify a service is also + used to match the process binary name. Returns a dictionary of process names along with the number of each. """ @@ -278,12 +253,14 @@ def processes(self): /var/lib/ and /var/log/ """ - cmd = self.get_process_cmd_from_line(line, expr) - if cmd: - if cmd in _proc_info: - _proc_info[cmd] += 1 - else: - _proc_info[cmd] = 1 + cmd = self.get_cmd_from_ps_line(line, expr) + if not cmd: + continue + + if cmd in _proc_info: + _proc_info[cmd] += 1 + else: + _proc_info[cmd] = 1 return _proc_info diff --git a/hotsos/core/plugins/juju/common.py b/hotsos/core/plugins/juju/common.py index 40873a206..f42065049 100644 --- a/hotsos/core/plugins/juju/common.py +++ b/hotsos/core/plugins/juju/common.py @@ -1,7 +1,7 @@ import os from hotsos.core import plugintools -from hotsos.core.host_helpers import SystemdHelper +from hotsos.core.host_helpers import PebbleHelper, SystemdHelper from hotsos.core.plugins.juju.resources import JujuBase # matches date and time at start if log lines @@ -19,6 +19,7 @@ class JujuChecksBase(plugintools.PluginPartBase, JujuBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.pebble = PebbleHelper(service_exprs=JUJU_SVC_EXPRS) self.systemd = SystemdHelper(service_exprs=JUJU_SVC_EXPRS) # this is needed for juju scenarios self.systemd_processes = self.systemd.processes diff --git a/hotsos/core/plugins/kubernetes.py b/hotsos/core/plugins/kubernetes.py index 0e2e09f5a..90e0a52ec 100644 --- a/hotsos/core/plugins/kubernetes.py +++ b/hotsos/core/plugins/kubernetes.py @@ -5,6 +5,7 @@ from hotsos.core.host_helpers import ( APTPackageHelper, HostNetworkingHelper, + PebbleHelper, SnapPackageHelper, SystemdHelper, ) @@ -107,6 +108,7 @@ def __init__(self, *args, **kwargs): snap_deps = deps + K8S_PACKAGE_DEPS_SNAP self.snaps = SnapPackageHelper(core_snaps=K8S_PACKAGES, other_snaps=snap_deps) + self.pebble = PebbleHelper(service_exprs=SERVICES) self.systemd = SystemdHelper(service_exprs=SERVICES) @property diff --git a/hotsos/core/plugins/mysql.py b/hotsos/core/plugins/mysql.py index 251dc442a..65a4b3b84 100644 --- a/hotsos/core/plugins/mysql.py +++ b/hotsos/core/plugins/mysql.py @@ -2,6 +2,7 @@ import glob from hotsos.core.host_helpers import ( APTPackageHelper, + PebbleHelper, SystemdHelper, ) from hotsos.core import ( @@ -20,6 +21,7 @@ class MySQLChecksBase(plugintools.PluginPartBase): def __init__(self, *args, **kwargs): super().__init__() self.apt_info = APTPackageHelper(core_pkgs=CORE_APT) + self.pebble = PebbleHelper(service_exprs=MYSQL_SVC_EXPRS) self.systemd = SystemdHelper(service_exprs=MYSQL_SVC_EXPRS) @property diff --git a/hotsos/core/plugins/openstack/common.py b/hotsos/core/plugins/openstack/common.py index 5e9f6af50..09cc712c9 100644 --- a/hotsos/core/plugins/openstack/common.py +++ b/hotsos/core/plugins/openstack/common.py @@ -20,6 +20,7 @@ APTPackageHelper, DockerImageHelper, DPKGVersionCompare, + PebbleHelper, SystemdHelper, SSLCertificate, SSLCertificatesHelper, @@ -93,8 +94,9 @@ class OpenstackChecksBase(OpenstackBase, plugintools.PluginPartBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.ost_projects = OSTProjectCatalog() - self.systemd = SystemdHelper( - service_exprs=self.ost_projects.service_exprs) + service_exprs = self.ost_projects.service_exprs + self.pebble = PebbleHelper(service_exprs=service_exprs) + self.systemd = SystemdHelper(service_exprs=service_exprs) self.apt = APTPackageHelper( core_pkgs=self.ost_projects.packages_core_exprs, other_pkgs=self.ost_projects.packages_dep_exprs) diff --git a/hotsos/core/plugins/openstack/openstack.py b/hotsos/core/plugins/openstack/openstack.py index 0536053c3..7f999894a 100644 --- a/hotsos/core/plugins/openstack/openstack.py +++ b/hotsos/core/plugins/openstack/openstack.py @@ -2,6 +2,7 @@ import os from hotsos.core import host_helpers +from hotsos.core.log import log from hotsos.core.config import HotSOSConfig from hotsos.core.plugins.openstack.exceptions import ( EXCEPTIONS_COMMON, @@ -308,6 +309,11 @@ def services_expr(self): def services(self): exprs = self.services_expr info = host_helpers.SystemdHelper(service_exprs=exprs) + if not info.services: + log.debug("no systemd services found for '%s' - trying pebble", + self.name) + info = host_helpers.PebbleHelper(service_exprs=exprs) + return info.services def log_paths(self, include_deprecated_services=True): diff --git a/hotsos/core/plugins/openvswitch/common.py b/hotsos/core/plugins/openvswitch/common.py index 13a45a3af..72ee1635f 100644 --- a/hotsos/core/plugins/openvswitch/common.py +++ b/hotsos/core/plugins/openvswitch/common.py @@ -1,5 +1,9 @@ from hotsos.core import plugintools -from hotsos.core.host_helpers import APTPackageHelper, SystemdHelper +from hotsos.core.host_helpers import ( + APTPackageHelper, + PebbleHelper, + SystemdHelper, +) from hotsos.core.ycheck.events import YEventCheckerBase from hotsos.core.utils import cached_property, sorted_dict @@ -30,6 +34,7 @@ def __init__(self, *args, **kwargs): p_deps = OVS_PKGS_DEPS + [PY_CLIENT_PREFIX.format(p) for p in OVS_PKGS_DEPS] self.apt = APTPackageHelper(core_pkgs=p_core, other_pkgs=p_deps) + self.pebble = PebbleHelper(service_exprs=OVS_SERVICES_EXPRS) self.systemd = SystemdHelper(service_exprs=OVS_SERVICES_EXPRS) @cached_property diff --git a/hotsos/core/plugins/rabbitmq/common.py b/hotsos/core/plugins/rabbitmq/common.py index e34e5e545..7964a6a54 100644 --- a/hotsos/core/plugins/rabbitmq/common.py +++ b/hotsos/core/plugins/rabbitmq/common.py @@ -1,5 +1,9 @@ from hotsos.core import plugintools -from hotsos.core.host_helpers import APTPackageHelper, SystemdHelper +from hotsos.core.host_helpers import ( + APTPackageHelper, + PebbleHelper, + SystemdHelper, +) from hotsos.core.plugins.rabbitmq.report import RabbitMQReport RMQ_SERVICES_EXPRS = [ @@ -24,6 +28,7 @@ class RabbitMQChecksBase(RabbitMQBase, plugintools.PluginPartBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.apt = APTPackageHelper(core_pkgs=RMQ_PACKAGES) + self.pebble = PebbleHelper(service_exprs=RMQ_SERVICES_EXPRS) self.systemd = SystemdHelper(service_exprs=RMQ_SERVICES_EXPRS) @property diff --git a/hotsos/core/plugins/storage/ceph.py b/hotsos/core/plugins/storage/ceph.py index 176bf87e2..ec43009e6 100644 --- a/hotsos/core/plugins/storage/ceph.py +++ b/hotsos/core/plugins/storage/ceph.py @@ -20,9 +20,9 @@ CLIHelper, DPKGVersionCompare, HostNetworkingHelper, + PebbleHelper, SystemdHelper, SectionalConfigBase, - SVC_EXPR_TEMPLATES, ) from hotsos.core.host_helpers.cli import get_ps_axo_flags_available from hotsos.core.plugins.storage import StorageBase @@ -808,7 +808,7 @@ def etime(self): if not ret: continue - expt_tmplt = SVC_EXPR_TEMPLATES["absolute"] + expt_tmplt = SystemdHelper.PS_CMD_EXPR_TEMPLATES['absolute'] ret = re.compile(expt_tmplt.format(daemon)).search(line) if ret: ps_info.append(ret.group(0)) @@ -896,6 +896,7 @@ def __init__(self, *args, **kwargs): self.bcache = BcacheBase() self.apt = APTPackageHelper(core_pkgs=CEPH_PKGS_CORE, other_pkgs=CEPH_PKGS_OTHER) + self.pebble = PebbleHelper(service_exprs=CEPH_SERVICES_EXPRS) self.systemd = SystemdHelper(service_exprs=CEPH_SERVICES_EXPRS) self.cluster = CephCluster() self.cli = CLIHelper() diff --git a/hotsos/core/plugins/vault.py b/hotsos/core/plugins/vault.py index 8566d6114..4260a0d36 100644 --- a/hotsos/core/plugins/vault.py +++ b/hotsos/core/plugins/vault.py @@ -1,5 +1,9 @@ from hotsos.core.plugintools import PluginPartBase -from hotsos.core.host_helpers import SystemdHelper, SnapPackageHelper +from hotsos.core.host_helpers import ( + PebbleHelper, + SnapPackageHelper, + SystemdHelper, +) from hotsos.core.utils import cached_property CORE_SNAPS = ['vault'] @@ -12,6 +16,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.snaps = SnapPackageHelper(core_snaps=CORE_SNAPS) self.systemd = SystemdHelper(service_exprs=SERVICE_EXPRS) + self.pebble = PebbleHelper(service_exprs=SERVICE_EXPRS) @cached_property def vault_installed(self): diff --git a/hotsos/core/ycheck/engine/properties/requires/__init__.py b/hotsos/core/ycheck/engine/properties/requires/__init__.py index 596f47e03..31641f5b0 100644 --- a/hotsos/core/ycheck/engine/properties/requires/__init__.py +++ b/hotsos/core/ycheck/engine/properties/requires/__init__.py @@ -5,4 +5,5 @@ YRequirementTypeBase, YRequirementTypeWithOpsBase, PackageCheckItemsBase, + ServiceCheckItemsBase, ) diff --git a/hotsos/core/ycheck/engine/properties/requires/common.py b/hotsos/core/ycheck/engine/properties/requires/common.py index b7cc67835..d127389a7 100644 --- a/hotsos/core/ycheck/engine/properties/requires/common.py +++ b/hotsos/core/ycheck/engine/properties/requires/common.py @@ -193,3 +193,51 @@ def ops(self): return self.default_ops return self.content.get('ops', self.default_ops) + + +class ServiceCheckItemsBase(CheckItemsBase): + + @cached_property + def _started_after_services(self): + svcs = [] + for _, settings in self: + if type(settings) != dict: + continue + + svc = settings.get('started-after') + if svc: + svcs.append(svc) + + return svcs + + @cached_property + def _services_to_check(self): + return [item[0] for item in self] + + @property + def _svcs_all(self): + """ + We include started-after services since if a check has specified one it + is expected to exist for that check to pass. + """ + return self._services_to_check + self._started_after_services + + @property + @abc.abstractmethod + def _svcs_info(self): + """ ServiceManagerBase implementation with _svcs_all as input. """ + + @cached_property + def not_installed(self): + _installed = self.installed.keys() + return set(_installed).symmetric_difference(self._svcs_all) + + @cached_property + def installed(self): + return self._svcs_info.services + + def processes_running(self, processes): + """ Check any processes provided. """ + a = set(processes) + b = set(self._svcs_info.processes.keys()) + return a.issubset(b) diff --git a/hotsos/core/ycheck/engine/properties/requires/types/systemd.py b/hotsos/core/ycheck/engine/properties/requires/types/systemd.py index 8fc9e47cf..d537a7ead 100644 --- a/hotsos/core/ycheck/engine/properties/requires/types/systemd.py +++ b/hotsos/core/ycheck/engine/properties/requires/types/systemd.py @@ -4,58 +4,17 @@ from hotsos.core.host_helpers import SystemdHelper from hotsos.core.ycheck.engine.properties.requires import ( intercept_exception, - CheckItemsBase, + ServiceCheckItemsBase, YRequirementTypeBase, ) -from hotsos.core.utils import cached_property -class ServiceCheckItems(CheckItemsBase): - - @cached_property - def _started_after_services(self): - svcs = [] - for _, settings in self: - if type(settings) != dict: - continue - - svc = settings.get('started-after') - if svc: - svcs.append(svc) - - return svcs - - @cached_property - def _services_to_check(self): - return [item[0] for item in self] +class SystemdServiceCheckItems(ServiceCheckItemsBase): @property - def _svcs_all(self): - """ - We include started-after services since if a check has specified one it - is expected to exist for that check to pass. - """ - return self._services_to_check + self._started_after_services - - @cached_property def _svcs_info(self): return SystemdHelper(self._svcs_all) - @cached_property - def not_installed(self): - _installed = self.installed.keys() - return set(_installed).symmetric_difference(self._svcs_all) - - @cached_property - def installed(self): - return self._svcs_info.services - - def processes_running(self, processes): - """ Check any processes provided. """ - a = set(processes) - b = set(self._svcs_info.processes.keys()) - return a.issubset(b) - class YRequirementTypeSystemd(YRequirementTypeBase): """ Provides logic to perform checks on systemd resources. """ @@ -118,7 +77,7 @@ def _result(self): default_op = 'eq' _result = True - items = ServiceCheckItems(self.content) + items = SystemdServiceCheckItems(self.content) if not items.not_installed: for svc, settings in items: svc_obj = items.installed[svc] diff --git a/hotsos/defs/README.md b/hotsos/defs/README.md index 5344706ad..c05f5b42e 100644 --- a/hotsos/defs/README.md +++ b/hotsos/defs/README.md @@ -663,6 +663,39 @@ CACHE_KEYS revision ``` +#### pebble +Takes a pebble service and optionally some parameters to check. +Returns True if service exists and, if provided, parameters match. +Short and long forms are supported as follows. + +format + +``` +pebble: (state not checked here) + +or + +pebble: [svc1, svc2 ...] (state not checked here) + +or + +pebble: SVCS + +where SVCS is a dict of one or more services e.g. + +pebble: + service_name: + state: + op: (optional. default is 'eq') + processes: list of processes we expect to be running (optional) + ... +``` + +``` +CACHE_KEYS + services +``` + #### systemd Takes a systemd service and optionally some parameters to check. Returns True if service exists and, if provided, parameters match. diff --git a/hotsos/defs/scenarios/openstack/octavia/hm_port_health.yaml b/hotsos/defs/scenarios/openstack/octavia/hm_port_health.yaml index 582fc51a8..47c0d81bc 100644 --- a/hotsos/defs/scenarios/openstack/octavia/hm_port_health.yaml +++ b/hotsos/defs/scenarios/openstack/octavia/hm_port_health.yaml @@ -1,7 +1,9 @@ checks: octavia_worker_installed: - - property: hotsos.core.plugins.openstack.octavia.OctaviaBase.installed - - systemd: octavia-worker + property: hotsos.core.plugins.openstack.octavia.OctaviaBase.installed + or: + - systemd: octavia-worker + - pebble: octavia-worker hm_port_has_no_packet_drops_or_errors: property: hotsos.core.plugins.openstack.octavia.OctaviaBase.hm_port_healthy hm_port_address_check: diff --git a/hotsos/defs/scenarios/storage/ceph/ceph-mon/autoscaler_bug.yaml b/hotsos/defs/scenarios/storage/ceph/ceph-mon/autoscaler_bug.yaml index c9bdf44d2..e5fb03ac3 100644 --- a/hotsos/defs/scenarios/storage/ceph/ceph-mon/autoscaler_bug.yaml +++ b/hotsos/defs/scenarios/storage/ceph/ceph-mon/autoscaler_bug.yaml @@ -31,6 +31,7 @@ checks: systemd: ceph-mon: state: enabled + pebble: ceph-mon autoscaler_enabled: varops: [[$autoscaler_enabled_pools], [length_hint]] autoscaler_disabled: diff --git a/hotsos/defs/scenarios/storage/ceph/ceph-mon/ceph-mon.yaml b/hotsos/defs/scenarios/storage/ceph/ceph-mon/ceph-mon.yaml index 0efe48b7f..770b7ab70 100644 --- a/hotsos/defs/scenarios/storage/ceph/ceph-mon/ceph-mon.yaml +++ b/hotsos/defs/scenarios/storage/ceph/ceph-mon/ceph-mon.yaml @@ -9,5 +9,7 @@ # if necessary. requires: # ensures that a ceph-mon unit is enabled but does not assert that it is active - systemd: - ceph-mon: enabled + or: + systemd: + ceph-mon: enabled + pebble: ceph-mon diff --git a/hotsos/plugin_extensions/juju/summary.py b/hotsos/plugin_extensions/juju/summary.py index 1cb3ed990..cec9811c2 100644 --- a/hotsos/plugin_extensions/juju/summary.py +++ b/hotsos/plugin_extensions/juju/summary.py @@ -107,6 +107,8 @@ class JujuSummary(JujuChecksBase): def __summary_services(self): if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(1) def __summary_version(self): diff --git a/hotsos/plugin_extensions/kubernetes/summary.py b/hotsos/plugin_extensions/kubernetes/summary.py index e19e9fbca..f89929dd3 100644 --- a/hotsos/plugin_extensions/kubernetes/summary.py +++ b/hotsos/plugin_extensions/kubernetes/summary.py @@ -6,7 +6,10 @@ class KubernetesSummary(KubernetesChecksBase): @idx(0) def __summary_services(self): - return self.systemd.summary + if self.systemd.services: + return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(1) def __summary_snaps(self): diff --git a/hotsos/plugin_extensions/mysql/summary.py b/hotsos/plugin_extensions/mysql/summary.py index 3de0a18a9..ef970c183 100644 --- a/hotsos/plugin_extensions/mysql/summary.py +++ b/hotsos/plugin_extensions/mysql/summary.py @@ -8,6 +8,8 @@ class MySQLSummary(MySQLChecksBase): def __summary_services(self): if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(1) def __summary_dpkg(self): diff --git a/hotsos/plugin_extensions/openstack/summary.py b/hotsos/plugin_extensions/openstack/summary.py index 33864af6e..dd4ddbc36 100644 --- a/hotsos/plugin_extensions/openstack/summary.py +++ b/hotsos/plugin_extensions/openstack/summary.py @@ -15,6 +15,8 @@ def __summary_services(self): """Get string info for running services.""" if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(2) def __summary_dpkg(self): diff --git a/hotsos/plugin_extensions/openvswitch/summary.py b/hotsos/plugin_extensions/openvswitch/summary.py index 9e0e1c47e..4e4958d2d 100644 --- a/hotsos/plugin_extensions/openvswitch/summary.py +++ b/hotsos/plugin_extensions/openvswitch/summary.py @@ -17,6 +17,8 @@ def __summary_services(self): """Get string info for running daemons.""" if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(1) def __summary_dpkg(self): diff --git a/hotsos/plugin_extensions/rabbitmq/summary.py b/hotsos/plugin_extensions/rabbitmq/summary.py index 292ba6091..d63f4b6ba 100644 --- a/hotsos/plugin_extensions/rabbitmq/summary.py +++ b/hotsos/plugin_extensions/rabbitmq/summary.py @@ -8,6 +8,8 @@ class RabbitMQSummary(RabbitMQChecksBase): def __summary_services(self): if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(1) def __summary_dpkg(self): diff --git a/hotsos/plugin_extensions/storage/ceph_summary.py b/hotsos/plugin_extensions/storage/ceph_summary.py index 0deae0268..8fbcc30a2 100644 --- a/hotsos/plugin_extensions/storage/ceph_summary.py +++ b/hotsos/plugin_extensions/storage/ceph_summary.py @@ -15,6 +15,8 @@ def __summary_services(self): """Get string info for running services.""" if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(2) def __summary_dpkg(self): diff --git a/hotsos/plugin_extensions/vault/summary.py b/hotsos/plugin_extensions/vault/summary.py index 40ea1ad4a..fe57bf669 100644 --- a/hotsos/plugin_extensions/vault/summary.py +++ b/hotsos/plugin_extensions/vault/summary.py @@ -8,6 +8,8 @@ class VaultSummary(VaultChecksBase): def __summary_services(self): if self.systemd.services: return self.systemd.summary + elif self.pebble.services: + return self.pebble.summary @idx(1) def __summary_snaps(self): diff --git a/tests/unit/test_host_helpers.py b/tests/unit/test_host_helpers.py index 8f837505f..4c03e1697 100644 --- a/tests/unit/test_host_helpers.py +++ b/tests/unit/test_host_helpers.py @@ -16,6 +16,18 @@ c-key = 2-8,10-31 """ +PEBBLE_SERVICES = """Service Startup Current Since +nova-conductor enabled backoff today at 10:25 UTC +""" + +# pylint: disable=C0301 +PEBBLE_PS = """USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND +root 1 0.0 0.0 717708 10516 ? Ssl 08:43 0:01 /charm/bin/pebble run --create-dirs --hold --http :38814 --verbose +root 3048 0.0 0.0 2620 600 pts/0 Ss 10:14 0:00 sh -c bash +root 3055 0.0 0.0 7372 4036 pts/0 S 10:14 0:00 bash +root 3225 0.0 0.2 80748 65780 ? R 10:42 0:00 /usr/bin/python3 /usr/bin/nova-conductor +""" # noqa + class TestHostNetworkingHelper(utils.BaseTestCase): @@ -309,6 +321,26 @@ def test_systemd_helper(self): self.assertEqual(s.summary, expected) +class TestPebbleHelper(utils.BaseTestCase): + + @utils.create_data_root({'sos_commands/pebble/pebble_services': + PEBBLE_SERVICES}) + def test_service_factory(self): + svc = getattr(host_helpers.pebble.ServiceFactory(), 'nova-conductor') + self.assertEqual(svc.state, 'backoff') + + self.assertEqual(host_helpers.pebble.ServiceFactory().noexist, None) + + @utils.create_data_root({'sos_commands/pebble/pebble_services': + PEBBLE_SERVICES, + 'ps': PEBBLE_PS}) + def test_pebble_helper(self): + expected = {'ps': ['nova-conductor (1)'], + 'pebble': {'backoff': ['nova-conductor']}} + s = host_helpers.pebble.PebbleHelper([r'nova\S+']) + self.assertEqual(s.summary, expected) + + class TestFileStatHelper(utils.BaseTestCase): @utils.create_data_root({'foo': 'bar'})