Creation of command line tools[cmf artifact list, cmf executions list…

…, cmf pipeline list] (#207)

* Created cmd tool for list of pipelines

* Created list of executions cmd tool

* Created cmd tool for list of artifacts

* Removed unnecessary import statement and initialized variables

* Added list as a parameter to artifact command and created seprate files for executions and pipelines list

* Updated code

* Added new cmf commands[cmf artifact list, cmf executions list, cmf pipeline list]]

* Updated copyright version

cmflib/cli/__init__.py

@@ -14,8 +14,6 @@
 # limitations under the License.
 ###
 
-import sys
-
 
 class CmfParserError(Exception):
     """Base class for CLI parser errors."""

cmflib/cli/parser.py

@@ -16,15 +16,12 @@
 
 """Main parser for the cmf cli"""
 import argparse
-import logging
-import os
-import sys
 
-from cmflib.commands import artifact, metadata, init
+from cmflib.commands import artifact, metadata, init, execution, pipeline
 
 from cmflib.cli import CmfParserError
 
-COMMANDS = [artifact, metadata, init]
+COMMANDS = [artifact, metadata, init, execution, pipeline]
 
 
 def _find_parser(parser, cmd_cls):

cmflib/cli/utils.py

@@ -16,7 +16,6 @@
 
 import subprocess
 import os
-import sys
 
 def fix_subparsers(subparsers):
     subparsers.required = True

cmflib/cmf.py

@@ -68,6 +68,9 @@
     _init_amazonS3,
     _init_sshremote,
     _init_osdfremote,
+    _artifact_list,
+    _pipeline_list,
+    _execution_list,
 )
 
 class Cmf:
@@ -2300,3 +2303,63 @@ def non_related_args(type : str, args : dict):
             non_related_args=list(set(available_args)-set(dict_repository_args[repo]))
     return non_related_args
 
+
+def pipeline_list(filepath = "./mlmd"):
+    """ Display list of pipline for current mlmd.
+
+    Example:
+    ```python
+         result = _pipeline_list("./mlmd_directory")
+    ```
+
+    Args:
+        filepath: File path to store the MLMD file. 
+    Returns:
+        Output from the _pipeline_list function.
+    """
+
+    # Optional arguments: filepath( path to store the MLMD file)
+    output = _pipeline_list(filepath)
+    return output
+
+
+def execution_list(pipeline_name: str, filepath = "./mlmd", execution_id: str = "", long = True):
+    """ Display list of execution for given pipeline.
+    Example: 
+    ```python 
+        result = _execution_list("example_pipeline", "./mlmd_directory", "example_execution_id", "long") 
+    ```
+    Args: 
+       pipeline_name: Name of the pipeline. 
+       filepath: Path to store the mlmd file. 
+       execution_id: Executions for particular execution id. 
+       long: Detailed summary regarding execution.
+    Returns:
+       Output from the _execution_list function. 
+    """
+
+    # Required arguments: pipeline_name
+    # Optional arguments: filepath( path to store mlmd file), execution_id, long
+    output = _execution_list(pipeline_name, filepath, execution_id, long)
+    return output
+
+
+def artifact_list(pipeline_name: str, filepath = "./mlmd", artifact_name: str = "", long = True):
+    """ Display list of artifact for given pipeline.
+    Example: 
+    ```python 
+        result = _artifact_list("example_pipeline", "./mlmd_directory", "example_artifact_name", "long") 
+    ```
+    Args: 
+       pipeline_name: Name of the pipeline. 
+       filepath: Path to store the mlmd file. 
+       artifact_name: Artifacts for particular artifact name. 
+       long: Detailed summary regarding artifact.
+    Returns:
+       Output from the _artifact_list function. 
+    """
+
+    # Required arguments: pipeline_name
+    # Optional arguments: filepath( path to store mlmd file), artifact_name, long
+    output = _artifact_list(pipeline_name, filepath, artifact_name, long)
+    return output

cmflib/cmf_commands_wrapper.py

@@ -282,3 +282,57 @@ def _init_osdfremote(path, key_id, key_path, key_issuer, git_remote_url, cmf_ser
     print(msg)
     return msg
 
+def _artifact_list(pipeline_name, file_name, artifact_name, long):
+    cli_args = cli.parse_args(
+            [
+               "artifact",
+               "list",
+               "-p",
+               pipeline_name,
+               "-f",
+               file_name,
+               "-a",
+               artifact_name,
+               "-l",
+               long
+            ]
+           )
+    cmd = cli_args.func(cli_args)
+    msg = cmd.do_run()
+    print(msg)
+    return msg
+
+def _pipeline_list(file_name):
+    cli_args = cli.parse_args(
+            [
+               "pipeline",
+               "list",
+               "-f",
+               file_name
+            ]
+           )
+    cmd = cli_args.func(cli_args)
+    msg = cmd.do_run()
+    print(msg)
+    return msg
+
+def _execution_list(pipeline_name, file_name, execution_id, long):
+    cli_args = cli.parse_args(
+            [
+               "execution",
+               "list",
+               "-p",
+               pipeline_name,
+               "-f",
+               file_name,
+               "-e",
+               execution_id,
+               "-l",
+               long
+            ]
+           )
+    cmd = cli_args.func(cli_args)
+    msg = cmd.do_run()
+    print(msg)
+    return msg
+

cmflib/commands/artifact/__init__.py

@@ -16,19 +16,19 @@
 
 import argparse
 
-from cmflib.commands.artifact import pull, push
+from cmflib.commands.artifact import pull, push, list
 from cmflib.cli.utils import *
 
-SUB_COMMANDS = [pull, push]
+SUB_COMMANDS = [pull, push, list]
 
 # This parser adds positional arguments to the main parser
 def add_parser(subparsers, parent_parser):
-    ARTIFACT_HELP = "Command for artifact pull/push."
+    ARTIFACT_HELP = "Command for artifact pull/push/list."
 
     artifact_parser = subparsers.add_parser(
         "artifact",
         parents=[parent_parser],
-        description="Pull or Push artifacts as per current cmf configuration.",
+        description="Pull, Push or List artifacts as per current cmf configuration.",
         help=ARTIFACT_HELP,
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

cmflib/commands/artifact/list.py

@@ -0,0 +1,115 @@
+###
+# Copyright (2024) Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+
+import argparse
+import os
+import pandas as pd
+
+from cmflib.cli.command import CmdBase
+from cmflib import cmfquery
+
+class CmdArtifactsList(CmdBase):
+    def update_dataframe(self, df):
+        for c in df.columns:
+            if c.startswith('custom_properties_'):
+                df.rename(columns = {c:c.replace('custom_properties_','')}, inplace = True)
+            else:
+                df = df.drop(c, axis = 1)
+        return df
+
+    def search_artifact(self, df):
+        for index, row in df.iterrows():
+            name = row['name'].split(":")[0]
+            file_name = name.split('/')[-1]
+            if file_name == self.args.artifact_name:
+                return row['id']
+        return -1
+
+    def run(self):
+        current_directory = os.getcwd()
+        # default path for mlmd file name
+        mlmd_file_name = "./mlmd"
+        if self.args.file_name:
+            mlmd_file_name = self.args.file_name
+            if mlmd_file_name == "mlmd":
+                mlmd_file_name = "./mlmd"
+            current_directory = os.path.dirname(mlmd_file_name)
+        if not os.path.exists(mlmd_file_name):
+            return f"ERROR: {mlmd_file_name} doesn't exists in {current_directory} directory."
+
+        # Creating cmfquery object
+        query = cmfquery.CmfQuery(mlmd_file_name)
+
+        df = query.get_all_artifacts_by_context(self.args.pipeline_name)
+
+        if not df.empty:
+            if self.args.artifact_name:
+                artifact_id = self.search_artifact(df)
+                if(artifact_id != -1):
+                    df = df.query(f'id == {int(artifact_id)}')
+                else:
+                    df = "Artifact name does not exist.."
+        else:
+            df = "Pipeline does not exist..."
+
+        if not isinstance(df, str):
+            if self.args.long:
+                pd.set_option('display.max_rows', None)  # Set to None to display all rows
+                pd.set_option('display.max_columns', None)  # Set to None to display all columns
+            else:
+                df = self.update_dataframe(df)
+        return df
+
+def add_parser(subparsers, parent_parser):
+    ARTIFACT_LIST_HELP = "Display list of artifact as present in current mlmd"
+
+    parser = subparsers.add_parser(
+        "list",
+        parents=[parent_parser],
+        description="Display artifact list",
+        help=ARTIFACT_LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    required_argumets = parser.add_argument_group("required arguments")
+
+    required_argumets.add_argument(
+        "-p", 
+        "--pipeline_name", 
+        required=True,
+        help="Specify pipeline name.", 
+        metavar="<pipeline_name>", 
+    )
+
+    parser.add_argument(
+        "-f", "--file_name", help="Specify mlmd file name.", metavar="<file_name>",
+    )
+
+    parser.add_argument(
+        "-a", 
+        "--artifact_name", 
+        help="Specify artifact name.", 
+        metavar="<artifact_name>",
+    )
+
+    parser.add_argument(
+        "-l", 
+        "--long", 
+        action='store_true',
+        help="Display detailed summary of artifact",
+    )
+
+    parser.set_defaults(func=CmdArtifactsList)

cmflib/commands/artifact/pull.py

@@ -159,7 +159,6 @@ def search_artifact(self, input_dict):
 
     def run(self):
         # check whether the mlmd file exist or not in current directory
-        pipeline_name = self.args.pipeline_name
         current_directory = os.getcwd()
         mlmd_file_name = "./mlmd"
         if self.args.file_name:

cmflib/commands/artifact/push.py

@@ -17,8 +17,6 @@
 #!/usr/bin/env python3
 import argparse
 import os
-import subprocess
-import time
 
 from cmflib import cmfquery
 from cmflib.cli.command import CmdBase

cmflib/commands/execution/__init__.py

@@ -0,0 +1,43 @@
+###
+# Copyright (2024) Hewlett Packard Enterprise Development LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+
+import argparse
+
+from cmflib.commands.execution import list
+from cmflib.cli.utils import *
+
+SUB_COMMANDS = [list]
+
+# This parser adds positional argumets to the main parser
+def add_parser(subparsers, parent_parser):
+    LIST_HELP = "Command to list executions."
+
+    list_parser = subparsers.add_parser(
+        "executions", 
+        parents=[parent_parser],
+        description="Display list of executions as present in current mlmd",
+        help=LIST_HELP,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    list_subparsers = list_parser.add_subparsers(
+        dest="cmd", help="Use `cmf execution CMD --help` for " "command-specific help."
+    )
+
+    fix_subparsers(list_subparsers)
+    for cmd in SUB_COMMANDS:
+        cmd.add_parser(list_subparsers, parent_parser)
+