CI Feature to detect stalled experiments #2049
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import sys | ||
| import os | ||
|
|
||
| from wxflow import Executable, which, Logger, CommandNotFoundError | ||
| from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter | ||
|
|
||
| logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False) | ||
|
|
||
|
|
||
| def input_args(): | ||
| """ | ||
| Parse command-line arguments. | ||
|
|
||
| Returns | ||
| ------- | ||
| args : Namespace | ||
| The parsed command-line arguments. | ||
| """ | ||
|
|
||
| description = """ | ||
| Using rocotostat to get the status of all jobs this scripts | ||
| determines rocoto_state: if all cycles are done, then rocoto_state is Done. | ||
| Assuming rocotorun had just been run, and the rocoto_state is not Done, then | ||
| rocoto_state is Stalled if there are no jobs that are RUNNING, SUBMITTING, or QUEUED. | ||
| """ | ||
|
Comment on lines
+22
to
+27
There was a problem hiding this comment. This seems more like a description for the whole script or |
||
|
|
||
| parser = ArgumentParser(description=description, | ||
| formatter_class=ArgumentDefaultsHelpFormatter) | ||
|
|
||
| parser.add_argument('-w', help='workflow_document', type=str) | ||
| parser.add_argument('-d', help='database_file', type=str) | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
| return args | ||
|
|
||
|
|
||
| def rocoto_statcount(): | ||
| """ | ||
| Run rocotostat and process its output. | ||
| """ | ||
|
|
||
| args = input_args() | ||
|
|
||
| try: | ||
| rocotostat = which("rocotostat") | ||
| except CommandNotFoundError: | ||
| logger.exception("rocotostat not found in PATH") | ||
| raise CommandNotFoundError("rocotostat not found in PATH") | ||
|
|
||
| xml_file_path = os.path.abspath(args.w) | ||
| db_file_path = os.path.abspath(args.d) | ||
|
|
||
| rocotostat_all = which("rocotostat") | ||
|
There was a problem hiding this comment. Why do this again? You already have @aerorahul I have not been able to find a way to use There was a problem hiding this comment. Could not find a way to "change" arguments once the are set on the Executable object. |
||
| rocotostat.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-s']) | ||
| rocotostat_all.add_default_arg(['-w', xml_file_path, '-d', db_file_path, '-a']) | ||
|
|
||
| rocotostat_output = rocotostat(output=str) | ||
| rocotostat_output = rocotostat_output.splitlines()[1:] | ||
| rocotostat_output = [line.split()[0:2] for line in rocotostat_output] | ||
|
|
||
| rocotostat_output_all = rocotostat_all(output=str) | ||
| rocotostat_output_all = rocotostat_output_all.splitlines()[1:] | ||
| rocotostat_output_all = [line.split()[0:4] for line in rocotostat_output_all] | ||
| rocotostat_output_all = [line for line in rocotostat_output_all if len(line) != 1] | ||
|
|
||
| rocoto_status = { | ||
| 'Cycles': len(rocotostat_output), | ||
| 'Done_Cycles': sum([sublist.count('Done') for sublist in rocotostat_output]) | ||
| } | ||
|
|
||
| status_cases = ['SUCCEEDED', 'FAIL', 'DEAD', 'RUNNING', 'SUBMITTING', 'QUEUED'] | ||
| for case in status_cases: | ||
| rocoto_status[case] = sum([sublist.count(case) for sublist in rocotostat_output_all]) | ||
|
|
||
| return rocoto_status | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
|
|
||
| args = input_args() | ||
|
|
||
| rocoto_status = rocoto_statcount() | ||
| for status in rocoto_status: | ||
| print(f'Number of {status} : {rocoto_status[status]}') | ||
| if rocoto_status['Cycles'] == rocoto_status['Done_Cycles']: | ||
| rocoto_state = 'Done' | ||
| elif 'UNKNOWN' in rocoto_status: | ||
| rocoto_state = 'Unknown' | ||
| print(f'Rocoto State : {rocoto_state}') | ||
| elif rocoto_status['RUNNING'] + rocoto_status['SUBMITTING'] + rocoto_status['QUEUED'] == 0: | ||
| rocoto_state = 'Stalled' | ||
| print(f'Rocoto State : {rocoto_state}') | ||
| sys.exit(-1) | ||
| else: | ||
| rocoto_state = 'Running' | ||
| print(f'Rocoto State : {rocoto_state}') | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Check notice
Code scanning / shellcheck
This is actually an end quote, but due to next char it looks suspect.