CI Feature to detect stalled experiments #2049
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import sys | ||
| import os | ||
|
|
||
| from wxflow import Executable, which, Logger | ||
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter | ||
|
|
||
| logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False) | ||
|
|
||
|
|
||
| def input_args(): | ||
| """ | ||
| Parse command-line arguments. | ||
|
|
||
| Returns | ||
| ------- | ||
| args : Namespace | ||
| The parsed command-line arguments. | ||
| """ | ||
|
|
||
| description = """ | ||
| Using rocotostat to get the status of all jobs this scripts | ||
| determines rocoto_state: if all cycles are done, then rocoto_state is Done. | ||
| If all cycles are not done, then rocoto_state is Running. | ||
| If the check_stalled is used then rocotorun is issued and | ||
| rocotostat is run again and checks if all jobs have not advanced, then | ||
| rocoto_state is Stalled and the script exits with -1. | ||
| """ | ||
|
|
||
| parser = ArgumentParser(description=description, | ||
| formatter_class=ArgumentDefaultsHelpFormatter) | ||
|
|
||
| parser.add_argument('-w', help='workflow_document', type=str) | ||
| parser.add_argument('-d', help='database_file', type=str) | ||
| parser.add_argument('--check_stalled', help='check if any jobs do not advance (stalled)', action='store_true', required=False) | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| return args | ||
|
|
||
| def rocoto_statcount(): | ||
| """ | ||
| Run rocotostat and process its output. | ||
| """ | ||
|
|
||
| args = input_args() | ||
|
|
||
| rocotostat = which("rocotostat") | ||
| if not rocotostat: | ||
| logger.exception("rocotostat not found in PATH") | ||
| sys.exit(-1) | ||
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| xml_file_path = os.path.abspath(args.w) | ||
| db_file_path = os.path.abspath(args.d) | ||
|
|
||
| rocotostat_all = which("rocotostat") | ||
|
There was a problem hiding this comment. Why do this again? You already have @aerorahul I have not been able to find a way to use There was a problem hiding this comment. Could not find a way to "change" arguments once the are set on the Executable object. |
||
| rocotostat.add_default_arg(['-w',xml_file_path,'-d',db_file_path,'-s']) | ||
| rocotostat_all.add_default_arg(['-w',xml_file_path,'-d',db_file_path,'-a']) | ||
|
|
||
| rocotostat_output = rocotostat(output=str) | ||
| rocotostat_output = rocotostat_output.splitlines()[1:] | ||
| rocotostat_output = [line.split()[0:2] for line in rocotostat_output] | ||
|
|
||
| rocotostat_output_all = rocotostat_all(output=str) | ||
| rocotostat_output_all = rocotostat_output_all.splitlines()[1:] | ||
| rocotostat_output_all = [line.split()[0:4] for line in rocotostat_output_all] | ||
| rocotostat_output_all = [line for line in rocotostat_output_all if len(line) != 1] | ||
|
There was a problem hiding this comment. I can show you how to reduce this and simplify it considerably There was a problem hiding this comment. Sure that would be great, you can jot down pseudo code it you like |
||
|
|
||
| rocoto_status = { | ||
| 'Cycles' : len(rocotostat_output), | ||
| 'Done_Cycles' : sum([ sublist.count('Done') for sublist in rocotostat_output ]) | ||
| } | ||
|
|
||
| status_cases = [ 'SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'PENDING', 'QUEUED'] | ||
WalterKolczynski-NOAA marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| for case in status_cases: | ||
| rocoto_status[case] = sum([ sublist.count(case) for sublist in rocotostat_output_all ]) | ||
|
|
||
| return rocoto_status | ||
|
|
||
| if __name__ == '__main__': | ||
|
|
||
| args = input_args() | ||
|
|
||
| rocoto_status = rocoto_statcount() | ||
| for status in rocoto_status: | ||
| print(f'Number of {status} : {rocoto_status[status]}') | ||
| rocoto_state = 'Running' | ||
| if rocoto_status['Cycles'] == rocoto_status['Done_Cycles']: | ||
| rocoto_state = 'Done' | ||
|
|
||
| if args.check_stalled: | ||
| if rocoto_state != 'Done': | ||
| rocoto_run = which("rocotorun") | ||
|
There was a problem hiding this comment. Having seen this, I think we should have a |
||
| rocoto_run.add_default_arg(['-w',args.w,'-d',args.d]) | ||
| rocoto_run() | ||
| rocoto_status2 = rocoto_statcount() | ||
| if rocoto_status2 == rocoto_status: | ||
| rocoto_state = 'Stalled' | ||
| print(f'Rocoto State : {rocoto_state}') | ||
| sys.exit(-1) | ||
| else: | ||
| rocoto_state = 'Running' | ||
TerrenceMcGuinness-NOAA marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| print(f'Rocoto State : {rocoto_state}') | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Small typo