Infra and tests for ODF CLI (red-hat-storage#10277)

Signed-off-by: Aviadp <[email protected]>

ocs_ci/helpers/odf_cli.py

@@ -0,0 +1,118 @@
+import os
+
+from stat import S_IEXEC
+from logging import getLogger
+from typing import Union
+
+from ocs_ci.ocs.exceptions import NotSupportedException
+from ocs_ci.utility.version import get_semantic_ocs_version_from_config, VERSION_4_15
+from ocs_ci.utility.utils import exec_cmd
+from ocs_ci.framework import config
+from ocs_ci.deployment.ocp import download_pull_secret
+from ocs_ci.ocs.constants import ODF_CLI_DEV_IMAGE
+
+
+log = getLogger(__name__)
+
+
+class ODFCLIRetriever:
+    def __init__(self):
+        self.semantic_version = get_semantic_ocs_version_from_config()
+        self.local_cli_path = os.path.join(config.RUN["bin_dir"], "odf-cli")
+
+    def retrieve_odf_cli_binary(self):
+        """
+        Download and set up the ODF-CLI binary.
+
+        Raises:
+            NotSupportedException: If ODF CLI is not supported on the current version or deployment.
+            AssertionError: If the CLI binary is not found or not executable.
+        """
+        self._validate_odf_cli_support()
+
+        image = self._get_odf_cli_image()
+        self._extract_cli_binary(image)
+        self._set_executable_permissions()
+        self._verify_cli_binary()
+        self.add_cli_to_path()
+
+    def _validate_odf_cli_support(self):
+        if self.semantic_version < VERSION_4_15:
+            raise NotSupportedException(
+                f"ODF CLI tool not supported on ODF {self.semantic_version}"
+            )
+
+    def _get_odf_cli_image(self, build_no: str = None):
+        if build_no:
+            return f"{ODF_CLI_DEV_IMAGE}:{build_no}"
+        else:
+            return f"{ODF_CLI_DEV_IMAGE}:latest-{self.semantic_version}"
+
+    def _extract_cli_binary(self, image):
+        pull_secret_path = download_pull_secret()
+        local_cli_dir = os.path.dirname(self.local_cli_path)
+
+        exec_cmd(
+            f"oc image extract --registry-config {pull_secret_path} "
+            f"{image} --confirm "
+            f"--path {local_cli_dir}:{local_cli_dir}"
+        )
+
+    def _set_executable_permissions(self):
+        current_permissions = os.stat(self.local_cli_path).st_mode
+        os.chmod(self.local_cli_path, current_permissions | S_IEXEC)
+
+    def _verify_cli_binary(self):
+        assert os.path.isfile(
+            self.local_cli_path
+        ), f"ODF CLI file not found at {self.local_cli_path}"
+        assert os.access(
+            self.local_cli_path, os.X_OK
+        ), "The ODF CLI binary does not have execution permissions"
+
+    def add_cli_to_path(self):
+        """
+        Add the directory containing the ODF CLI binary to the system PATH.
+        """
+        cli_dir = os.path.dirname(self.local_cli_path)
+        current_path = os.environ.get("PATH", "")
+        if cli_dir not in current_path:
+            os.environ["PATH"] = f"{cli_dir}:{current_path}"
+        log.info(f"Added {cli_dir} to PATH")
+
+
+class ODFCliRunner:
+    def __init__(self) -> None:
+        self.binary_name = "odf-cli"
+
+    def run_command(self, command_args: Union[str, list]) -> str:
+        if isinstance(command_args, str):
+            full_command = str(self.binary_name + command_args)
+
+        elif isinstance(command_args, list):
+            full_command = " ".join(command_args)
+        exec_cmd(full_command)
+
+    def run_help(self):
+        return self.run_command(" help")
+
+    def run_get_health(self):
+        return self.run_command(" get health")
+
+    def run_get_recovery_profile(self):
+        return self.run_command(" get recovery-profile")
+
+    def run_get_mon_endpoint(self):
+        return self.run_command(" get mon-endpoints")
+
+    def run_rook_restart(self):
+        return self.run_command(" operator rook restart")
+
+    def run_rook_set_log_level(self, log_level: str):
+        assert log_level in (
+            "DEBUG",
+            "INFO",
+            "WARNING",
+            "ERROR",
+        ), f"log level {log_level} is not supported"
+        return self.run_command(f" operator rook set ROOK_LOG_LEVEL {log_level}")

tests/functional/odf-cli/test_get_commands.py

@@ -0,0 +1,142 @@
+import logging
+import re
+
+from ocs_ci.ocs.resources.pod import get_mon_pods
+from ocs_ci.helpers.odf_cli import ODFCLIRetriever, ODFCliRunner
+from ocs_ci.framework.testlib import tier1, brown_squad, polarion_id
+
+logger = logging.getLogger(__name__)
+
+
+@tier1
+@brown_squad
+class TestGetCommands:
+    def test_download_binary(self):
+        self.odf_cli_retriever = ODFCLIRetriever()
+        self.odf_cli_retriever.retrieve_odf_cli_binary()
+        self.odf_cli_runner = ODFCliRunner()
+
+    @polarion_id("OCS-6237")
+    def test_get_health(self):
+
+        output = self.odf_cli_runner.run_get_health()
+        self.validate_mon_pods(output)
+        self.validate_mon_quorum_and_health(output)
+        self.validate_osd_pods(output)
+        self.validate_running_pods(output)
+        self.validate_pg_status(output)
+        self.validate_mgr_pods(output)
+
+    @polarion_id("OCS-6238")
+    def test_get_mon_endpoint(self):
+        output = self.odf_cli_runner.run_get_mon_endpoint()
+        assert output, "Mon endpoint not found in output"
+        # Validate the format of the mon endpoint output
+        endpoint_pattern = r"^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+,?)+$"
+        assert re.match(
+            endpoint_pattern, output.strip()
+        ), f"Invalid mon endpoint format: {output}"
+
+        # Get the number of monitor pods
+        mon_pods = get_mon_pods()
+        expected_mon_count = len(mon_pods)
+
+        # Check that we have the correct number of endpoints
+        endpoints = output.strip().split(",")
+        assert (
+            len(endpoints) == expected_mon_count
+        ), f"Expected {expected_mon_count} mon endpoints, but found {len(endpoints)}"
+
+        # Validate each endpoint
+        for endpoint in endpoints:
+            ip, port = endpoint.split(":")
+            assert (
+                1 <= int(port) <= 65535
+            ), f"Invalid port number in endpoint: {endpoint}"
+            octets = ip.split(".")
+            assert len(octets) == 4, f"Invalid IP address in endpoint: {endpoint}"
+            assert all(
+                0 <= int(octet) <= 255 for octet in octets
+            ), f"Invalid IP address in endpoint: {endpoint}"
+
+    def validate_mon_pods(self, output):
+        mon_section = re.search(
+            r"Info: Checking if at least three mon pods are running on different nodes\n(.*?)\n\n",
+            output,
+            re.DOTALL,
+        )
+        assert mon_section, "Mon pods section not found in output"
+        mon_pods = mon_section.group(1).split("\n")
+        assert (
+            len(mon_pods) >= 3
+        ), f"Expected at least 3 mon pods, found {len(mon_pods)}"
+        nodes = set()
+        for pod in mon_pods:
+            assert "Running" in pod, f"Mon pod not in Running state: {pod}"
+            node = pod.split()[-1]
+            nodes.add(node)
+        assert (
+            len(nodes) >= 3
+        ), f"Mon pods should be on at least 3 different nodes, found {len(nodes)}"
+
+    def validate_mon_quorum_and_health(self, output):
+        health_ok = "Info: HEALTH_OK" in output
+        assert health_ok, "Ceph health is not OK"
+
+    def validate_osd_pods(self, output):
+        osd_section = re.search(
+            r"Info: Checking if at least three osd pods are running on different nodes\n(.*?)\n\n",
+            output,
+            re.DOTALL,
+        )
+        assert osd_section, "OSD pods section not found in output"
+        osd_pods = osd_section.group(1).split("\n")
+        assert (
+            len(osd_pods) >= 3
+        ), f"Expected at least 3 OSD pods, found {len(osd_pods)}"
+        nodes = set()
+        for pod in osd_pods:
+            assert "Running" in pod, f"OSD pod not in Running state: {pod}"
+            node = pod.split()[-1]
+            nodes.add(node)
+        assert (
+            len(nodes) >= 3
+        ), f"OSD pods should be on at least 3 different nodes, found {len(nodes)}"
+
+    def validate_running_pods(self, output):
+        running_pods_section = re.search(
+            r"Info: Pods that are in 'Running' or `Succeeded` status\n(.*?)\n\nWarning:",
+            output,
+            re.DOTALL,
+        )
+        assert running_pods_section, "Running pods section not found in output"
+        running_pods = running_pods_section.group(1).split("\n")
+        assert len(running_pods) > 0, "No running pods found"
+        for pod in running_pods:
+            assert (
+                "Running" in pod or "Succeeded" in pod
+            ), f"Pod not in Running or Succeeded state: {pod}"
+
+    def validate_pg_status(self, output):
+        pg_status = re.search(
+            r"Info: Checking placement group status\nInfo:\s+PgState: (.*?), PgCount: (\d+)",
+            output,
+        )
+        assert pg_status, "Placement group status not found in output"
+        pg_state, pg_count = pg_status.groups()
+        assert (
+            pg_state == "active+clean"
+        ), f"Expected PG state to be 'active+clean', found '{pg_state}'"
+        assert int(pg_count) > 0, f"Expected positive PG count, found {pg_count}"
+
+    def validate_mgr_pods(self, output):
+        mgr_section = re.search(
+            r"Info: Checking if at least one mgr pod is running\n(.*?)$",
+            output,
+            re.DOTALL,
+        )
+        assert mgr_section, "MGR pods section not found in output"
+        mgr_pods = mgr_section.group(1).split("\n")
+        assert len(mgr_pods) >= 1, f"Expected at least 1 MGR pod, found {len(mgr_pods)}"
+        for pod in mgr_pods:
+            assert "Running" in pod, f"MGR pod not in Running state: {pod}"

tests/functional/odf-cli/test_operator_restart.py

@@ -0,0 +1,36 @@
+import logging
+
+from ocs_ci.ocs.resources.pod import (
+    get_operator_pods,
+    validate_pods_are_respinned_and_running_state,
+)
+from ocs_ci.utility.utils import TimeoutSampler
+from ocs_ci.helpers.odf_cli import ODFCLIRetriever, ODFCliRunner
+from ocs_ci.framework.testlib import tier1, brown_squad, polarion_id
+
+logger = logging.getLogger(__name__)
+
+
+@tier1
+@brown_squad
+@polarion_id("OCS-6235")
+class TestOperatorRestart:
+    def check_operator_status(self):
+        operator_pods = get_operator_pods()
+        return validate_pods_are_respinned_and_running_state(operator_pods)
+
+    def verify_operator_restart(self):
+        logger.info("Verifying operator restart...")
+        sampler = TimeoutSampler(timeout=300, sleep=10, func=self.check_operator_status)
+        if not sampler.wait_for_func_status(result=True):
+            raise AssertionError(
+                "Operator did not restart successfully within the expected time"
+            )
+        logger.info("Operator restart verified successfully")
+
+    def test_operator_restart(self):
+        self.odf_cli_retriever = ODFCLIRetriever()
+        self.odf_cli_retriever.retrieve_odf_cli_binary()
+        self.odf_cli_runner = ODFCliRunner()
+        self.odf_cli_runner.run_rook_restart()
+        self.verify_operator_restart()

tests/functional/z_cluster/test_rook_ceph_operator_log_type.py

@@ -10,6 +10,7 @@
     get_last_log_time_date,
     check_osd_log_exist_on_rook_ceph_operator_pod,
 )
+from ocs_ci.helpers.odf_cli import ODFCLIRetriever, ODFCliRunner
 from ocs_ci.framework.pytest_customization.marks import brown_squad
 from ocs_ci.framework.testlib import (
     ManageTest,
@@ -52,13 +53,34 @@ def teardown(self):
         set_configmap_log_level_rook_ceph_operator(value="INFO")
         ceph_health_check()
 
-    def test_rook_ceph_operator_log_type(self):
+    def set_rook_ceph_operator_log_level(self, value, method):
+        """
+        Set the log level for the rook-ceph operator using either the configmap method
+        or the ODF CLI method, depending on the OCS version.
+        """
+        if method == "odf_cli":
+            # Use ODF CLI method
+            odf_cli_retriever = ODFCLIRetriever()
+            odf_cli_retriever.retrieve_odf_cli_binary()
+            odf_cli_runner = ODFCliRunner()
+            odf_cli_runner.run_rook_set_log_level(value)
+        else:
+            # Use existing configmap method
+            set_configmap_log_level_rook_ceph_operator(value=value)
+
+    @pytest.mark.parametrize(
+        "method",
+        [
+            pytest.param("configmap"),
+            pytest.param("odf_cli"),
+        ],
+    )
+    def test_rook_ceph_operator_log_type(self, method):
         """
         Test the ability to change the log level in rook-ceph operator dynamically
         without rook-ceph operator pod restart.
-
         """
-        set_configmap_log_level_rook_ceph_operator(value="DEBUG")
+        self.set_rook_ceph_operator_log_level("DEBUG", method=method)
         last_log_date_time_obj = get_last_log_time_date()
 
         log.info("Respin OSD pod")
@@ -76,7 +98,7 @@ def test_rook_ceph_operator_log_type(self):
         if not sample.wait_for_func_status(result=True):
             raise ValueError("OSD DEBUG Log does not exist")
 
-        set_configmap_log_level_rook_ceph_operator(value="INFO")
+        self.set_rook_ceph_operator_log_level("INFO", method=method)
         last_log_date_time_obj = get_last_log_time_date()
 
         log.info("Respin OSD pod")