Skip to content

Commit

Permalink
cmf wrapper modification (#188)
Browse files Browse the repository at this point in the history 
* Added missing cmf wrappers

* Added wrapper for osdfremote configuration

* Added session token parameter

---------

Co-authored-by: Varkha Sharma <[email protected]>
  • Loading branch information
@AyeshaSanadi @varkha-d-sharma
AyeshaSanadi and varkha-d-sharma authored Aug 14, 2024
1 parent c006963 commit 5b9f027
Show file tree
Hide file tree
Showing 2 changed files with 145 additions and 69 deletions.
162 changes: 108 additions & 54 deletions cmflib/cmf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,16 @@
from cmflib.cmf_commands_wrapper import (
_metadata_push,
_metadata_pull,
_metadata_export,
_artifact_pull,
_artifact_push,
_artifact_pull_single,
_cmf_cmd_init,
_init_local,
_init_minioS3,
_init_amazonS3,
_init_sshremote

_init_sshremote,
_init_osdfremote,
)

class Cmf:
Expand Down Expand Up @@ -1953,7 +1954,7 @@ def commit_existing(self, uri: str, props: t.Optional[t.Dict] = None, custom_pro
# print(last)
# os.symlink(str(index), slicedir + "/ " + last)

def metadata_push(pipeline_name,filepath,execution_id: str = ""):
def metadata_push(pipeline_name: str, filepath = "./mlmd", tensorboard_path: str = "", execution_id: str = ""):
""" Pushes MLMD file to CMF-server.
Example:
```python
Expand All @@ -1963,16 +1964,17 @@ def metadata_push(pipeline_name,filepath,execution_id: str = ""):
pipeline_name: Name of the pipeline.
filepath: Path to the MLMD file.
execution_id: Optional execution ID.
tensorboard_path: Path to tensorboard logs.
Returns:
Response output from the _metadata_push function.
"""
# Required arguments: pipeline_name, filepath (mlmd file path)
#Optional arguments: Execution_ID
output = _metadata_push(pipeline_name,filepath, execution_id)
# Required arguments: pipeline_name
# Optional arguments: Execution_ID, filepath (mlmd file path, tensorboard_path
output = _metadata_push(pipeline_name, filepath, execution_id, tensorboard_path)
return output

def metadata_pull(pipeline_name,filepath ="./mlmd", execution_id: str = ""):
def metadata_pull(pipeline_name: str, filepath = "./mlmd", execution_id: str = ""):
""" Pulls MLMD file from CMF-server.
Example:
```python
Expand All @@ -1985,12 +1987,30 @@ def metadata_pull(pipeline_name,filepath ="./mlmd", execution_id: str = ""):
Returns:
Message from the _metadata_pull function.
"""
# Required arguments: pipeline_name, filepath(file path to store mlmd file)
#Optional arguments: Execution_ID
output = _metadata_pull(pipeline_name,filepath, execution_id)
# Required arguments: pipeline_name
#Optional arguments: Execution_ID, filepath(file path to store mlmd file)
output = _metadata_pull(pipeline_name, filepath, execution_id)
return output

def metadata_export(pipeline_name: str, jsonfilepath: str = "", filepath = "./mlmd"):
""" Export local mlmd's metadata in json format to a json file.
Example:
```python
result = metadata_pull("example_pipeline", "./jsonfile", "./mlmd_directory")
```
Args:
pipeline_name: Name of the pipeline.
jsonfilepath: File path of json file.
filepath: File path to store the MLMD file.
Returns:
Message from the _metadata_pull function.
"""
# Required arguments: pipeline_name
#Optional arguments: jsonfilepath, filepath(file path to store mlmd file)
output = _metadata_export(pipeline_name, jsonfilepath, filepath)
return output

def artifact_pull(pipeline_name,filepath="./mlmd"):
def artifact_pull(pipeline_name: str, filepath = "./mlmd"):
""" Pulls artifacts from the initialized repository.
Example:
Expand All @@ -2001,17 +2021,16 @@ def artifact_pull(pipeline_name,filepath="./mlmd"):
Args:
pipeline_name: Name of the pipeline.
filepath: Path to store artifacts.
Returns:
Output from the _artifact_pull function.
"""

# Required arguments: Pipeline_name
# Optional arguments: filepath( path to store artifacts)
output = _artifact_pull(pipeline_name,filepath)
output = _artifact_pull(pipeline_name, filepath)
return output

def artifact_pull_single(pipeline_name,filepath,artifact_name):
def artifact_pull_single(pipeline_name: str, filepath: str, artifact_name: str):
""" Pulls a single artifact from the initialized repository.
Example:
```python
Expand All @@ -2026,22 +2045,25 @@ def artifact_pull_single(pipeline_name,filepath,artifact_name):
"""

# Required arguments: Pipeline_name
# Optional arguments: filepath( path to store artifacts)
output = _artifact_pull_single(pipeline_name,filepath,artifact_name)
# Optional arguments: filepath( path to store artifacts), artifact_name
output = _artifact_pull_single(pipeline_name, filepath, artifact_name)
return output

def artifact_push():
def artifact_push(pipeline_name: str, filepath = "./mlmd"):
""" Pushes artifacts to the initialized repository.
Example:
```python
result = artifact_push()
result = artifact_push("example_pipeline", "./mlmd_directory")
```
Args:
pipeline_name: Name of the pipeline.
filepath: Path to store the artifact.
Returns:
Output from the _artifact_push function.
"""
output = _artifact_push()

output = _artifact_push(pipeline_name, filepath)
return output

def cmf_init_show():
Expand All @@ -2057,20 +2079,25 @@ def cmf_init_show():
output=_cmf_cmd_init()
return output

def cmf_init(type: str="",
path: str="",
git_remote_url: str="",
def cmf_init(type: str = "",
path: str = "",
git_remote_url: str = "",
cmf_server_url: str = "",
neo4j_user: str = "",
neo4j_password: str = "",
neo4j_uri: str = "",
url: str="",
endpoint_url: str="",
access_key_id: str="",
secret_key: str="",
user: str="",
password: str="",
port: int=0
url: str = "",
endpoint_url: str = "",
access_key_id: str = "",
secret_key: str = "",
session_token: str = "",
user: str = "",
password: str = "",
port: int = 0,
osdf_path: str = "",
key_id: str = "",
key_path: str = "",
key_issuer: str = "",
):

""" Initializes the CMF configuration based on the provided parameters.
Expand All @@ -2097,38 +2124,44 @@ def cmf_init(type: str="",
endpoint_url: Endpoint URL for MinioS3.
access_key_id: Access key ID for MinioS3 or AmazonS3.
secret_key: Secret key for MinioS3 or AmazonS3.
session_token: Session token for AmazonS3.
user: SSH remote username.
password: SSH remote password.
port: SSH remote port
Returns:
Output based on the initialized repository type.
"""

if type=="":
if type == "":
return print("Error: Type is not provided")
if type not in ["local","minioS3","amazonS3","sshremote"]:
return print("Error: Type value is undefined"+ " "+type+".Expected: "+",".join(["local","minioS3","amazonS3","sshremote"]))
if type not in ["local","minioS3","amazonS3","sshremote","osdfremote"]:
return print("Error: Type value is undefined"+ " "+type+".Expected: "+",".join(["local","minioS3","amazonS3","sshremote","osdfremote"]))

if neo4j_user!="" and neo4j_password != "" and neo4j_uri != "":
if neo4j_user != "" and neo4j_password != "" and neo4j_uri != "":
pass
elif neo4j_user == "" and neo4j_password == "" and neo4j_uri == "":
pass
else:
return print("Error: Enter all neo4j parameters.")

args={'path':path,
'git_remote_url':git_remote_url,
'url':url,
'endpoint_url':endpoint_url,
'access_key_id':access_key_id,
'secret_key':secret_key,
'user':user,
'password':password,
args={'path': path,
'git_remote_url': git_remote_url,
'url': url,
'endpoint_url': endpoint_url,
'access_key_id': access_key_id,
'secret_key': secret_key,
'session_token': session_token,
'user': user,
'password': password,
'osdf_path': osdf_path,
'key_id': key_id,
'key_path': key_path,
'key-issuer': key_issuer,
}

status_args=non_related_args(type,args)
status_args=non_related_args(type, args)

if type=="local" and path!= "" and git_remote_url!= "" :
if type == "local" and path != "" and git_remote_url != "" :
"""Initialize local repository"""
output = _init_local(
path, git_remote_url, cmf_server_url, neo4j_user, neo4j_password, neo4j_uri
Expand All @@ -2137,7 +2170,7 @@ def cmf_init(type: str="",
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")
return output

elif type=="minioS3" and url!= "" and endpoint_url!= "" and access_key_id!= "" and secret_key!= "" and git_remote_url!= "":
elif type == "minioS3" and url != "" and endpoint_url != "" and access_key_id != "" and secret_key != "" and git_remote_url != "":
"""Initialize minioS3 repository"""
output = _init_minioS3(
url,
Expand All @@ -2154,12 +2187,13 @@ def cmf_init(type: str="",
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")
return output

elif type=="amazonS3" and url!= "" and access_key_id!= "" and secret_key!= "" and git_remote_url!= "":
elif type == "amazonS3" and url != "" and access_key_id != "" and secret_key != "" and git_remote_url != "":
"""Initialize amazonS3 repository"""
output = _init_amazonS3(
url,
access_key_id,
secret_key,
session_token,
git_remote_url,
cmf_server_url,
neo4j_user,
Expand All @@ -2171,7 +2205,7 @@ def cmf_init(type: str="",

return output

elif type=="sshremote" and path !="" and user!="" and port!=0 and password!="" and git_remote_url!="":
elif type == "sshremote" and path != "" and user != "" and port != 0 and password != "" and git_remote_url != "":
"""Initialize sshremote repository"""
output = _init_sshremote(
path,
Expand All @@ -2189,18 +2223,38 @@ def cmf_init(type: str="",

return output

elif type == "osdfremote" and osdf_path != "" and key_id != "" and key_path != 0 and key_issuer != "" and git_remote_url != "":
"""Initialize osdfremote repository"""
output = _init_osdfremote(
osdf_path,
key_id,
key_path,
key_issuer,
git_remote_url,
cmf_server_url,
neo4j_user,
neo4j_password,
neo4j_uri,
)
if status_args != []:
print("There are non-related arguments: "+",".join(status_args)+".Please remove them.")

return output

else:
print("Error: Enter all arguments")


def non_related_args(type:str,args:dict):
available_args=[i for i,j in args.items() if j!=""]
local=["path","git_remote_url"]
minioS3=["url","endpoint_url","access_key_id","secret_key","git_remote_url"]
amazonS3=["url","access_key_id","secret_key","git_remote_url"]
sshremote=["path","user","port","password","git_remote_url"]
def non_related_args(type : str, args : dict):
available_args=[i for i, j in args.items() if j != ""]
local=["path", "git_remote_url"]
minioS3=["url", "endpoint_url", "access_key_id", "secret_key", "git_remote_url"]
amazonS3=["url", "access_key_id", "secret_key", "session_token", "git_remote_url"]
sshremote=["path", "user", "port", "password", "git_remote_url"]
osdfremote=["osdf_path", "key_id", "key_path", "key-issuer", "git_remote_url"]


dict_repository_args={"local":local,"minioS3":minioS3,"amazonS3":amazonS3,"sshremote":sshremote}
dict_repository_args={"local" : local, "minioS3" : minioS3, "amazonS3" : amazonS3, "sshremote" : sshremote}

for repo,arg in dict_repository_args.items():
if repo ==type:
Expand Down
Loading

0 comments on commit 5b9f027

Please sign in to comment.