From d6a8bc491213712b6568b7adbc00afcbfe532031 Mon Sep 17 00:00:00 2001 From: AyeshaSanadi <167299982+AyeshaSanadi@users.noreply.github.com> Date: Tue, 15 Oct 2024 04:24:04 +0530 Subject: [PATCH] Creation of command line tools[cmf artifact list, cmf executions list, cmf pipeline list] (#207) * Created cmd tool for list of pipelines * Created list of executions cmd tool * Created cmd tool for list of artifacts * Removed unnecessary import statement and initialized variables * Added list as a parameter to artifact command and created seprate files for executions and pipelines list * Updated code * Added new cmf commands[cmf artifact list, cmf executions list, cmf pipeline list]] * Updated copyright version --- cmflib/cli/__init__.py | 2 - cmflib/cli/parser.py | 7 +- cmflib/cli/utils.py | 1 - cmflib/cmf.py | 63 ++++++++++++++ cmflib/cmf_commands_wrapper.py | 54 ++++++++++++ cmflib/commands/artifact/__init__.py | 8 +- cmflib/commands/artifact/list.py | 115 +++++++++++++++++++++++++ cmflib/commands/artifact/pull.py | 1 - cmflib/commands/artifact/push.py | 2 - cmflib/commands/execution/__init__.py | 43 ++++++++++ cmflib/commands/execution/list.py | 116 ++++++++++++++++++++++++++ cmflib/commands/init/amazonS3.py | 3 - cmflib/commands/init/local.py | 4 - cmflib/commands/init/minioS3.py | 3 - cmflib/commands/init/osdfremote.py | 4 - cmflib/commands/init/show.py | 2 - cmflib/commands/init/sshremote.py | 3 - cmflib/commands/metadata/export.py | 2 - cmflib/commands/metadata/pull.py | 5 -- cmflib/commands/pipeline/__init__.py | 43 ++++++++++ cmflib/commands/pipeline/list.py | 56 +++++++++++++ docs/cmf_client/cmf_client.md | 101 ++++++++++++++++++---- 22 files changed, 580 insertions(+), 58 deletions(-) create mode 100644 cmflib/commands/artifact/list.py create mode 100644 cmflib/commands/execution/__init__.py create mode 100644 cmflib/commands/execution/list.py create mode 100644 cmflib/commands/pipeline/__init__.py create mode 100644 cmflib/commands/pipeline/list.py diff --git a/cmflib/cli/__init__.py b/cmflib/cli/__init__.py index 1b7e107c..58e21183 100644 --- a/cmflib/cli/__init__.py +++ b/cmflib/cli/__init__.py @@ -14,8 +14,6 @@ # limitations under the License. ### -import sys - class CmfParserError(Exception): """Base class for CLI parser errors.""" diff --git a/cmflib/cli/parser.py b/cmflib/cli/parser.py index b71e5233..dc59407d 100644 --- a/cmflib/cli/parser.py +++ b/cmflib/cli/parser.py @@ -16,15 +16,12 @@ """Main parser for the cmf cli""" import argparse -import logging -import os -import sys -from cmflib.commands import artifact, metadata, init +from cmflib.commands import artifact, metadata, init, execution, pipeline from cmflib.cli import CmfParserError -COMMANDS = [artifact, metadata, init] +COMMANDS = [artifact, metadata, init, execution, pipeline] def _find_parser(parser, cmd_cls): diff --git a/cmflib/cli/utils.py b/cmflib/cli/utils.py index e8dc861b..0eafb18c 100644 --- a/cmflib/cli/utils.py +++ b/cmflib/cli/utils.py @@ -16,7 +16,6 @@ import subprocess import os -import sys def fix_subparsers(subparsers): subparsers.required = True diff --git a/cmflib/cmf.py b/cmflib/cmf.py index 58cc2ec3..cb653dd6 100644 --- a/cmflib/cmf.py +++ b/cmflib/cmf.py @@ -68,6 +68,9 @@ _init_amazonS3, _init_sshremote, _init_osdfremote, + _artifact_list, + _pipeline_list, + _execution_list, ) class Cmf: @@ -2300,3 +2303,63 @@ def non_related_args(type : str, args : dict): non_related_args=list(set(available_args)-set(dict_repository_args[repo])) return non_related_args + +def pipeline_list(filepath = "./mlmd"): + """ Display list of pipline for current mlmd. + + Example: + ```python + result = _pipeline_list("./mlmd_directory") + ``` + + Args: + filepath: File path to store the MLMD file. + Returns: + Output from the _pipeline_list function. + """ + + # Optional arguments: filepath( path to store the MLMD file) + output = _pipeline_list(filepath) + return output + + +def execution_list(pipeline_name: str, filepath = "./mlmd", execution_id: str = "", long = True): + """ Display list of execution for given pipeline. + Example: + ```python + result = _execution_list("example_pipeline", "./mlmd_directory", "example_execution_id", "long") + ``` + Args: + pipeline_name: Name of the pipeline. + filepath: Path to store the mlmd file. + execution_id: Executions for particular execution id. + long: Detailed summary regarding execution. + Returns: + Output from the _execution_list function. + """ + + # Required arguments: pipeline_name + # Optional arguments: filepath( path to store mlmd file), execution_id, long + output = _execution_list(pipeline_name, filepath, execution_id, long) + return output + + +def artifact_list(pipeline_name: str, filepath = "./mlmd", artifact_name: str = "", long = True): + """ Display list of artifact for given pipeline. + Example: + ```python + result = _artifact_list("example_pipeline", "./mlmd_directory", "example_artifact_name", "long") + ``` + Args: + pipeline_name: Name of the pipeline. + filepath: Path to store the mlmd file. + artifact_name: Artifacts for particular artifact name. + long: Detailed summary regarding artifact. + Returns: + Output from the _artifact_list function. + """ + + # Required arguments: pipeline_name + # Optional arguments: filepath( path to store mlmd file), artifact_name, long + output = _artifact_list(pipeline_name, filepath, artifact_name, long) + return output diff --git a/cmflib/cmf_commands_wrapper.py b/cmflib/cmf_commands_wrapper.py index 8a349ca8..a5d9a420 100644 --- a/cmflib/cmf_commands_wrapper.py +++ b/cmflib/cmf_commands_wrapper.py @@ -282,3 +282,57 @@ def _init_osdfremote(path, key_id, key_path, key_issuer, git_remote_url, cmf_ser print(msg) return msg +def _artifact_list(pipeline_name, file_name, artifact_name, long): + cli_args = cli.parse_args( + [ + "artifact", + "list", + "-p", + pipeline_name, + "-f", + file_name, + "-a", + artifact_name, + "-l", + long + ] + ) + cmd = cli_args.func(cli_args) + msg = cmd.do_run() + print(msg) + return msg + +def _pipeline_list(file_name): + cli_args = cli.parse_args( + [ + "pipeline", + "list", + "-f", + file_name + ] + ) + cmd = cli_args.func(cli_args) + msg = cmd.do_run() + print(msg) + return msg + +def _execution_list(pipeline_name, file_name, execution_id, long): + cli_args = cli.parse_args( + [ + "execution", + "list", + "-p", + pipeline_name, + "-f", + file_name, + "-e", + execution_id, + "-l", + long + ] + ) + cmd = cli_args.func(cli_args) + msg = cmd.do_run() + print(msg) + return msg + diff --git a/cmflib/commands/artifact/__init__.py b/cmflib/commands/artifact/__init__.py index c79d85d3..52a9d3dc 100644 --- a/cmflib/commands/artifact/__init__.py +++ b/cmflib/commands/artifact/__init__.py @@ -16,19 +16,19 @@ import argparse -from cmflib.commands.artifact import pull, push +from cmflib.commands.artifact import pull, push, list from cmflib.cli.utils import * -SUB_COMMANDS = [pull, push] +SUB_COMMANDS = [pull, push, list] # This parser adds positional arguments to the main parser def add_parser(subparsers, parent_parser): - ARTIFACT_HELP = "Command for artifact pull/push." + ARTIFACT_HELP = "Command for artifact pull/push/list." artifact_parser = subparsers.add_parser( "artifact", parents=[parent_parser], - description="Pull or Push artifacts as per current cmf configuration.", + description="Pull, Push or List artifacts as per current cmf configuration.", help=ARTIFACT_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) diff --git a/cmflib/commands/artifact/list.py b/cmflib/commands/artifact/list.py new file mode 100644 index 00000000..536e66f1 --- /dev/null +++ b/cmflib/commands/artifact/list.py @@ -0,0 +1,115 @@ +### +# Copyright (2024) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +import argparse +import os +import pandas as pd + +from cmflib.cli.command import CmdBase +from cmflib import cmfquery + +class CmdArtifactsList(CmdBase): + def update_dataframe(self, df): + for c in df.columns: + if c.startswith('custom_properties_'): + df.rename(columns = {c:c.replace('custom_properties_','')}, inplace = True) + else: + df = df.drop(c, axis = 1) + return df + + def search_artifact(self, df): + for index, row in df.iterrows(): + name = row['name'].split(":")[0] + file_name = name.split('/')[-1] + if file_name == self.args.artifact_name: + return row['id'] + return -1 + + def run(self): + current_directory = os.getcwd() + # default path for mlmd file name + mlmd_file_name = "./mlmd" + if self.args.file_name: + mlmd_file_name = self.args.file_name + if mlmd_file_name == "mlmd": + mlmd_file_name = "./mlmd" + current_directory = os.path.dirname(mlmd_file_name) + if not os.path.exists(mlmd_file_name): + return f"ERROR: {mlmd_file_name} doesn't exists in {current_directory} directory." + + # Creating cmfquery object + query = cmfquery.CmfQuery(mlmd_file_name) + + df = query.get_all_artifacts_by_context(self.args.pipeline_name) + + if not df.empty: + if self.args.artifact_name: + artifact_id = self.search_artifact(df) + if(artifact_id != -1): + df = df.query(f'id == {int(artifact_id)}') + else: + df = "Artifact name does not exist.." + else: + df = "Pipeline does not exist..." + + if not isinstance(df, str): + if self.args.long: + pd.set_option('display.max_rows', None) # Set to None to display all rows + pd.set_option('display.max_columns', None) # Set to None to display all columns + else: + df = self.update_dataframe(df) + return df + +def add_parser(subparsers, parent_parser): + ARTIFACT_LIST_HELP = "Display list of artifact as present in current mlmd" + + parser = subparsers.add_parser( + "list", + parents=[parent_parser], + description="Display artifact list", + help=ARTIFACT_LIST_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + required_argumets = parser.add_argument_group("required arguments") + + required_argumets.add_argument( + "-p", + "--pipeline_name", + required=True, + help="Specify pipeline name.", + metavar="", + ) + + parser.add_argument( + "-f", "--file_name", help="Specify mlmd file name.", metavar="", + ) + + parser.add_argument( + "-a", + "--artifact_name", + help="Specify artifact name.", + metavar="", + ) + + parser.add_argument( + "-l", + "--long", + action='store_true', + help="Display detailed summary of artifact", + ) + + parser.set_defaults(func=CmdArtifactsList) \ No newline at end of file diff --git a/cmflib/commands/artifact/pull.py b/cmflib/commands/artifact/pull.py index a58898a1..0cbdded0 100644 --- a/cmflib/commands/artifact/pull.py +++ b/cmflib/commands/artifact/pull.py @@ -159,7 +159,6 @@ def search_artifact(self, input_dict): def run(self): # check whether the mlmd file exist or not in current directory - pipeline_name = self.args.pipeline_name current_directory = os.getcwd() mlmd_file_name = "./mlmd" if self.args.file_name: diff --git a/cmflib/commands/artifact/push.py b/cmflib/commands/artifact/push.py index 6957c8c6..6efd5afb 100644 --- a/cmflib/commands/artifact/push.py +++ b/cmflib/commands/artifact/push.py @@ -17,8 +17,6 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -import time from cmflib import cmfquery from cmflib.cli.command import CmdBase diff --git a/cmflib/commands/execution/__init__.py b/cmflib/commands/execution/__init__.py new file mode 100644 index 00000000..0639b0cc --- /dev/null +++ b/cmflib/commands/execution/__init__.py @@ -0,0 +1,43 @@ +### +# Copyright (2024) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +import argparse + +from cmflib.commands.execution import list +from cmflib.cli.utils import * + +SUB_COMMANDS = [list] + +# This parser adds positional argumets to the main parser +def add_parser(subparsers, parent_parser): + LIST_HELP = "Command to list executions." + + list_parser = subparsers.add_parser( + "executions", + parents=[parent_parser], + description="Display list of executions as present in current mlmd", + help=LIST_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + list_subparsers = list_parser.add_subparsers( + dest="cmd", help="Use `cmf execution CMD --help` for " "command-specific help." + ) + + fix_subparsers(list_subparsers) + for cmd in SUB_COMMANDS: + cmd.add_parser(list_subparsers, parent_parser) + diff --git a/cmflib/commands/execution/list.py b/cmflib/commands/execution/list.py new file mode 100644 index 00000000..262d39a8 --- /dev/null +++ b/cmflib/commands/execution/list.py @@ -0,0 +1,116 @@ +### +# Copyright (2024) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +import argparse +import os +import pandas as pd + +from cmflib.cli.command import CmdBase +from cmflib import cmfquery + +class CmdExecutionList(CmdBase): + def update_dataframe(self, df): + # This function return dataframe with custom_properties_ only. + for c in df.columns: + if c.startswith('custom_properties_'): + df.rename(columns = {c:c.replace('custom_properties_','')}, inplace = True) + else: + df = df.drop(c, axis = 1) + return df + + def run(self): + current_directory = os.getcwd() + # default path for mlmd file name + mlmd_file_name = "./mlmd" + if self.args.file_name: + mlmd_file_name = self.args.file_name + if mlmd_file_name == "mlmd": + mlmd_file_name = "./mlmd" + current_directory = os.path.dirname(mlmd_file_name) + if not os.path.exists(mlmd_file_name): + return f"ERROR: {mlmd_file_name} doesn't exists in {current_directory} directory." + + # Creating cmfquery object + query = cmfquery.CmfQuery(mlmd_file_name) + + df = query.get_all_executions_in_pipeline(self.args.pipeline_name) + + # If dataframe is empty that means pipeline name is not exist + if df.empty: + df = "Pipeline does not exist.." + else: + # If the new mlmd came[not in the case of Test-env] which is not pushed inside server, + # it doesn't exist column named with "Python_Env" + if "Python_Env" in df.columns: + # Dropping Python_Env column + df = df.drop(['Python_Env'], axis=1) # Type of df is series of integers + if self.args.execution_id: + try: + if int(self.args.execution_id) in list(df['id']): # Converting series to list + df = df.query(f'id == {int(self.args.execution_id)}') + else: + df = "Execution id does not exist.." + except: + df = "Execution id does not exist.." + + if not isinstance(df, str): + if self.args.long: + pd.set_option('display.max_rows', None) # Set to None to display all rows + pd.set_option('display.max_columns', None) # Set to None to display all columns + else: + df = self.update_dataframe(df) + return df + +def add_parser(subparsers, parent_parser): + EXECUTION_LIST_HELP = "Display list of executions as present in current mlmd" + + parser = subparsers.add_parser( + "list", + parents=[parent_parser], + description="Display list of executions", + help=EXECUTION_LIST_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + required_argumets = parser.add_argument_group("required arguments") + + required_argumets.add_argument( + "-p", + "--pipeline_name", + required=True, + help="Specify pipeline name.", + metavar="", + ) + + parser.add_argument( + "-f", "--file_name", help="Specify mlmd file name.", metavar="", + ) + + parser.add_argument( + "-e", + "--execution_id", + help="Specify execution id.", + metavar="", + ) + + parser.add_argument( + "-l", + "--long", + action='store_true', + help="Display detailed summary of executions.", + ) + + parser.set_defaults(func=CmdExecutionList) \ No newline at end of file diff --git a/cmflib/commands/init/amazonS3.py b/cmflib/commands/init/amazonS3.py index d12e6487..cdfc3826 100644 --- a/cmflib/commands/init/amazonS3.py +++ b/cmflib/commands/init/amazonS3.py @@ -17,8 +17,6 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -import sys from cmflib.cli.command import CmdBase from cmflib.dvc_wrapper import ( @@ -26,7 +24,6 @@ git_checkout_new_branch, git_initial_commit, git_add_remote, - check_git_repo, dvc_quiet_init, dvc_add_remote_repo, dvc_add_attribute, diff --git a/cmflib/commands/init/local.py b/cmflib/commands/init/local.py index 6bed0f0f..741ccc19 100644 --- a/cmflib/commands/init/local.py +++ b/cmflib/commands/init/local.py @@ -17,8 +17,6 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -import sys from cmflib.cli.command import CmdBase from cmflib.dvc_wrapper import ( @@ -26,10 +24,8 @@ git_checkout_new_branch, git_initial_commit, git_add_remote, - check_git_repo, dvc_quiet_init, dvc_add_remote_repo, - dvc_add_attribute, ) from cmflib.utils.cmf_config import CmfConfig from cmflib.utils.helper_functions import is_git_repo diff --git a/cmflib/commands/init/minioS3.py b/cmflib/commands/init/minioS3.py index 371da9a8..345484a0 100644 --- a/cmflib/commands/init/minioS3.py +++ b/cmflib/commands/init/minioS3.py @@ -17,8 +17,6 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -import sys from cmflib.cli.command import CmdBase from cmflib.dvc_wrapper import ( @@ -26,7 +24,6 @@ git_checkout_new_branch, git_initial_commit, git_add_remote, - check_git_repo, dvc_quiet_init, dvc_add_remote_repo, dvc_add_attribute, diff --git a/cmflib/commands/init/osdfremote.py b/cmflib/commands/init/osdfremote.py index d63f4ff8..c187d894 100644 --- a/cmflib/commands/init/osdfremote.py +++ b/cmflib/commands/init/osdfremote.py @@ -18,8 +18,6 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -import sys from cmflib.cli.command import CmdBase from cmflib.dvc_wrapper import ( @@ -27,7 +25,6 @@ git_checkout_new_branch, git_initial_commit, git_add_remote, - check_git_repo, dvc_quiet_init, dvc_add_remote_repo, dvc_add_attribute, @@ -35,7 +32,6 @@ from cmflib.utils.cmf_config import CmfConfig from cmflib.utils.helper_functions import is_git_repo from cmflib.utils.helper_functions import generate_osdf_token -from cmflib.utils.helper_functions import is_url class CmdInitOSDFRemote(CmdBase): def run(self): diff --git a/cmflib/commands/init/show.py b/cmflib/commands/init/show.py index 65daa7d1..fa6e7d84 100644 --- a/cmflib/commands/init/show.py +++ b/cmflib/commands/init/show.py @@ -17,9 +17,7 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -from cmflib import cmfquery from cmflib.cli.command import CmdBase from cmflib.cli.utils import find_root from cmflib.dvc_wrapper import dvc_get_config diff --git a/cmflib/commands/init/sshremote.py b/cmflib/commands/init/sshremote.py index 51965c85..245fe2f2 100644 --- a/cmflib/commands/init/sshremote.py +++ b/cmflib/commands/init/sshremote.py @@ -18,8 +18,6 @@ #!/usr/bin/env python3 import argparse import os -import subprocess -import sys from cmflib.cli.command import CmdBase from cmflib.dvc_wrapper import ( @@ -27,7 +25,6 @@ git_checkout_new_branch, git_initial_commit, git_add_remote, - check_git_repo, dvc_quiet_init, dvc_add_remote_repo, dvc_add_attribute, diff --git a/cmflib/commands/metadata/export.py b/cmflib/commands/metadata/export.py index b53096fa..fb8317a5 100644 --- a/cmflib/commands/metadata/export.py +++ b/cmflib/commands/metadata/export.py @@ -28,8 +28,6 @@ def run(self): current_directory = os.getcwd() full_path_to_dump = "" - data = "" - mlmd_data = "" mlmd_file_name = "./mlmd" diff --git a/cmflib/commands/metadata/pull.py b/cmflib/commands/metadata/pull.py index e1a72d36..7c50cbf7 100644 --- a/cmflib/commands/metadata/pull.py +++ b/cmflib/commands/metadata/pull.py @@ -16,10 +16,8 @@ #!/usr/bin/env python3 import argparse -import json import os from cmflib import cmf_merger -from cmflib import cmfquery from cmflib.cli.command import CmdBase from cmflib.cli.utils import find_root from cmflib.server_interface import server_interface @@ -44,12 +42,9 @@ def run(self): current_directory = os.getcwd() full_path_to_dump = "" - data = "" cmd = "pull" - mlmd_data = "" status = 0 exec_id = None - execution_flag = 0 if self.args.file_name: # setting directory where mlmd file will be dumped if not os.path.isdir(self.args.file_name): temp = os.path.dirname(self.args.file_name) diff --git a/cmflib/commands/pipeline/__init__.py b/cmflib/commands/pipeline/__init__.py new file mode 100644 index 00000000..9b1a2066 --- /dev/null +++ b/cmflib/commands/pipeline/__init__.py @@ -0,0 +1,43 @@ +### +# Copyright (2024) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +import argparse + +from cmflib.commands.pipeline import list +from cmflib.cli.utils import * + +SUB_COMMANDS = [list] + +# This parser adds positional argumets to the main parser +def add_parser(subparsers, parent_parser): + LIST_HELP = "Command to list pipeline." + + list_parser = subparsers.add_parser( + "pipeline", + parents=[parent_parser], + description="Display list of pipelines as present in current mlmd", + help=LIST_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + list_subparsers = list_parser.add_subparsers( + dest="cmd", help="Use `cmf pipeline CMD --help` for " "command-specific help." + ) + + fix_subparsers(list_subparsers) + for cmd in SUB_COMMANDS: + cmd.add_parser(list_subparsers, parent_parser) + diff --git a/cmflib/commands/pipeline/list.py b/cmflib/commands/pipeline/list.py new file mode 100644 index 00000000..206aa4a9 --- /dev/null +++ b/cmflib/commands/pipeline/list.py @@ -0,0 +1,56 @@ +### +# Copyright (2024) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +import argparse +import os + +from cmflib.cli.command import CmdBase +from cmflib import cmfquery + +class CmdPipelineList(CmdBase): + def run(self): + current_directory = os.getcwd() + # default path for mlmd file name + mlmd_file_name = "./mlmd" + if self.args.file_name: + mlmd_file_name = self.args.file_name + if mlmd_file_name == "mlmd": + mlmd_file_name = "./mlmd" + current_directory = os.path.dirname(mlmd_file_name) + if not os.path.exists(mlmd_file_name): + return f"ERROR: {mlmd_file_name} doesn't exists in {current_directory} directory." + + # Creating cmfquery object + query = cmfquery.CmfQuery(mlmd_file_name) + + return [pipeline.name for pipeline in query._get_pipelines()] + +def add_parser(subparsers, parent_parser): + PIPELINE_LIST_HELP = "Display list of pipelines as present in current mlmd" + + parser = subparsers.add_parser( + "list", + parents=[parent_parser], + description="Display list of pipeline", + help=PIPELINE_LIST_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "-f", "--file_name", help="Specify mlmd file name.", metavar="", + ) + + parser.set_defaults(func=CmdPipelineList) diff --git a/docs/cmf_client/cmf_client.md b/docs/cmf_client/cmf_client.md index ecf1588d..bffe8161 100644 --- a/docs/cmf_client/cmf_client.md +++ b/docs/cmf_client/cmf_client.md @@ -1,4 +1,11 @@ # Getting started with cmf-client commands + +# cmf +``` +Usage: cmf [-h] {init, artifact, metadata, executions, pipeline} +``` +The `cmf` command is a comprehensive tool designed to initialize an artifact repository and perform various operations on artifacts, executions, pipeline and metadata. + ## cmf init ``` Usage: cmf init [-h] {minioS3,amazonS3,local,sshremote,osdfremote,show} @@ -9,7 +16,6 @@ Usage: cmf init [-h] {minioS3,amazonS3,local,sshremote,osdfremote,show} Usage: cmf init show ``` `cmf init show` displays current cmf configuration. - ### cmf init minioS3 ``` Usage: cmf init minioS3 [-h] --url [url] @@ -43,8 +49,6 @@ Optional Arguments --neo4j-user [neo4j_user] Specify neo4j user. (default: None) --neo4j-password [neo4j_password] Specify neo4j password. (default: None) --neo4j-uri [neo4j_uri] Specify neo4j uri. Eg bolt://localhost:7687 (default: None) - - ``` ### cmf init local ``` @@ -211,11 +215,12 @@ Optional Arguments --neo4j-uri [neo4j_uri] Specify neo4j uri. Eg bolt://localhost:7687 (default: None) ``` + ## cmf artifact ``` -Usage: cmf artifact [-h] {pull,push} +Usage: cmf artifact [-h] {pull,push,list} ``` -`cmf artifact` pull or push artifacts from or to the user configured artifact repository, respectively. +`cmf artifact` pull, push or list artifacts from or to the user configured artifact repository, respectively. ### cmf artifact pull ``` Usage: cmf artifact pull [-h] -p [pipeline_name] -f [file_name] -a [artifact_name] @@ -230,9 +235,9 @@ Required Arguments ``` Optional Arguments ``` - -h, --help show this help message and exit + -h, --help show this help message and exit. -a [artifact_name], --artifact_name [artifact_name] Specify artifact name only; don't use folder name or absolute path. - -f [file_name],--file-name [file_name] Specify mlmd file name. + -f [file_name], --file-name [file_name] Specify mlmd file name. ``` ### cmf artifact push ``` @@ -249,8 +254,28 @@ Required Arguments Optional Arguments ``` -h, --help show this help message and exit. - -f [file_name],--file-name [file_name] Specify mlmd file name. + -f [file_name], --file-name [file_name] Specify mlmd file name. +``` +### cmf artifact list +``` +Usage: cmf artifact list [-h] -p [pipeline_name] -f [file_name] -a [artifact_name] -l +``` +`cmf artifact list` command display list of artifacts. +``` +cmf artifact list -p 'pipeline_name' -f '/path/to/mlmd-file-name' -a 'artifact_name' -l ``` +Required Arguments +``` + -p [pipeline_name], --pipeline-name [pipeline_name] Specify Pipeline name. +``` +Optional Arguments +``` + -h, --help show this help message and exit. + -f [file_name], --file-name [file_name] Specify mlmd file name. + -a [artifact_name], --artifact_name [artifact_name] Specify artifact name. + -l, --long Specify in which format you want to saw artifacts[By default short]. +``` + ## cmf metadata ``` Usage: cmf metadata [-h] {pull,push,export} @@ -270,8 +295,8 @@ Required Arguments ``` Optional Arguments ``` --h, --help show this help message and exit --e [exec_id], --execution [exec_id] Specify execution id +-h, --help show this help message and exit. +-e [exec_id], --execution [exec_id] Specify execution id. -f [file_name], --file_name [file_name] Specify mlmd file name with full path(either relative or absolute). ``` ### cmf metadata push @@ -286,12 +311,11 @@ Required Arguments ``` -p [pipeline_name], --pipeline_name [pipeline_name] Specify Pipeline name. ``` - Optional Arguments ``` - -h, --help show this help message and exit - -f [file_name], --file_name [file_name] Specify mlmd file name. - -e [exec_id], --execution [exec_id] Specify execution id. + -h, --help show this help message and exit. + -f [file_name], --file_name [file_name] Specify mlmd file name. + -e [exec_id], --execution [exec_id] Specify execution id. -t [tensorboard], --tensorboard [tensorboard] Specify path to tensorboard logs for the pipeline. ``` ### cmf metadata export @@ -306,10 +330,53 @@ Required Arguments ``` -p [pipeline_name], --pipeline_name [pipeline_name] Specify Pipeline name. ``` - Optional Arguments ``` - -h, --help show this help message and exit - -f [file_name], --file_name [file_name] Specify mlmd file name. + -h, --help show this help message and exit. + -f [file_name], --file_name [file_name] Specify mlmd file name. -j [json_file_name], --json_file_name [json_file_name] Specify json file name with full path. ``` + +## cmf executions +``` +Usage: cmf executions [-h] {list} +``` +`cmf executions` list executions from or to the user configured repository. +### cmf executions list +``` +Usage: cmf executions list [-h] -p [pipeline_name] -f [file_name] -e [execution_id] -l +``` +`cmf executions list` command display list of executions in current cmf configuration. +``` +cmf executions list -p 'pipeline_name' -f '/path/to/mlmd-file-name' -e 'execution_id' -l +``` +Required Arguments +``` + -p [pipeline_name], --pipeline-name [pipeline_name] Specify Pipeline name. +``` +Optional Arguments +``` + -h, --help show this help message and exit. + -f [file_name], --file-name [file_name] Specify mlmd file name. + -e [exe_id], --execution_id [exe_id] Specify execution id. + -l, --long Specify in which format you want to saw execution[By default short]. +``` + +## cmf pipeline +``` +Usage: cmf pipeline [-h] {list} +``` +`cmf pipeline` command to display list of pipelines. +### cmf pipeline list +``` +Usage: cmf pipeline list [-h] -f [file_name] +``` +`cmf pipeline list` command display list of pipelines in current cmf configuration. +``` +cmf pipeline list -f '/path/to/mlmd-file-name' +``` +Optional Arguments +``` + -h, --help show this help message and exit. + -f [file_name], --file-name [file_name] Specify mlmd file name. +```