diff --git a/ci/scripts/tests/test_rocotostat.py b/ci/scripts/tests/test_rocotostat.py index 52737c4e8f..caf9ae4e6b 100755 --- a/ci/scripts/tests/test_rocotostat.py +++ b/ci/scripts/tests/test_rocotostat.py @@ -7,15 +7,25 @@ from rocotostat import rocoto_statcount, rocotostat_summary, is_done, is_stalled, CommandNotFoundError from wxflow import which -workflow_file = os.path.join(script_dir, "testdata/rocotostat/workflow.xml") -database_file = os.path.join(script_dir, "testdata/rocotostat/database.db") +test_data_url ='https://noaa-nws-global-pds.s3.amazonaws.com/data/CI/' + +testdata_path = 'testdata/rocotostat' +testdata_full_path = os.path.join(script_dir, testdata_path) + +wget = which('wget') +if not os.listdir(os.path.join(testdata_full_path)): + wget.add_default_arg(['-P', testdata_full_path, + test_data_url + str(testdata_path) + '/workflow.xml', + test_data_url + str(testdata_path) + '/database.db', + ]) + wget() try: - rocotostat = which("rocotostat") + rocotostat = which('rocotostat') except CommandNotFoundError: raise CommandNotFoundError("rocotostat not found in PATH") -rocotostat.add_default_arg(['-w', workflow_file, '-d', database_file]) +rocotostat.add_default_arg(['-w', os.path.join(testdata_path,'workflow.xml'), '-d', os.path.join(testdata_path, 'database.db')]) def test_rocoto_statcount(): @@ -45,10 +55,21 @@ def test_rocoto_done(): def test_rocoto_stalled(): + testdata_path = 'testdata/rocotostat_stalled' + testdata_full_path = os.path.join(script_dir, testdata_path) + + wget = which('wget') + if not os.listdir(os.path.join(testdata_full_path)): + wget.add_default_arg(['-P', testdata_full_path, + test_data_url + str(testdata_path) + '/stalled.xml', + test_data_url + str(testdata_path) + '/stalled.db', + ]) + wget() - workflow_file = os.path.join(script_dir, "testdata/rocotostat_stalled/stalled.xml") - database_file = os.path.join(script_dir, "testdata/rocotostat_stalled/stalled.db") + rocotostat = which('rocotostat') + rocotostat.add_default_arg(['-w', os.path.join(testdata_path,'stalled.xml'), '-d', os.path.join(testdata_path, 'stalled.db')]) result = rocoto_statcount(rocotostat) + assert result['SUCCEEDED'] == 110 assert is_stalled(result) diff --git a/ci/scripts/tests/testdata/rocotostat/database.db b/ci/scripts/tests/testdata/rocotostat/database.db deleted file mode 100644 index baca6c7513..0000000000 Binary files a/ci/scripts/tests/testdata/rocotostat/database.db and /dev/null differ diff --git a/ci/scripts/tests/testdata/rocotostat/stub b/ci/scripts/tests/testdata/rocotostat/stub new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci/scripts/tests/testdata/rocotostat/workflow.xml b/ci/scripts/tests/testdata/rocotostat/workflow.xml deleted file mode 100644 index d5171f647a..0000000000 --- a/ci/scripts/tests/testdata/rocotostat/workflow.xml +++ /dev/null @@ -1,272 +0,0 @@ - - - - - - -]> - - - - STUB - - - 202103231200 202103231200 24:00:00 - - - - stage_ic.sh - - C48_ATM_0ad155f1_gfsstage_ic_@H - nems - batch - orion - 00:15:00 - 1:ppn=1:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsstage_ic.log - - RUN_ENVIRemc - HOMEgfsgfs - EXPDIRRUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - - - - /ICSDIR/prototype_ICs/workflow_C48_refactored/@Y@m@d@H/atmos/gfs_ctrl.nc - - - - - - - - /CI/ORION/2555/gfs/jobs/rocoto/fcst.sh - - C48_ATM_0ad155f1_gfsfcst_@H - nems - batch - orion - 03:00:00 - 1:ppn=40:tpp=1 - --export=NONE - - /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsfcst.log - - RUN_ENVIRemc - HOMEgfs/CI/ORION/2555/gfs - EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/CI/ORION/2555/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - - - - - - - - - - - - _f000-f006 _f009-f015 _f018-f024 _f027-f033 _f036-f042 _f045-f051 _f054-f060 _f063-f069 _f072-f078 _f081-f087 _f090-f096 _f099-f105 _f108-f114 _f117-f120 - f006 f015 f024 f033 f042 f051 f060 f069 f078 f087 f096 f105 f114 f120 - f000_f003_f006 f009_f012_f015 f018_f021_f024 f027_f030_f033 f036_f039_f042 f045_f048_f051 f054_f057_f060 f063_f066_f069 f072_f075_f078 f081_f084_f087 f090_f093_f096 f099_f102_f105 f108_f111_f114 f117_f120 - - - - /CI/ORION/2555/gfs/jobs/rocoto/atmos_products.sh - - C48_ATM_0ad155f1_gfsatmos_prod#grp#_@H - nems - batch - orion - 00:15:00 - 1:ppn=24:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsatmos_prod#grp#.log - - RUN_ENVIRemc - HOMEgfs/CI/ORION/2555/gfs - EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/CI/ORION/2555/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - FHRLST#lst# - COMPONENTatmos - - - &ROTDIR;/gfs.@Y@m@d/@H//model_data/atmos/master/gfs.t@Hz.master.grb2#dep# - - - - - - - - - /CI/ORION/2555/gfs/jobs/rocoto/tracker.sh - - C48_ATM_0ad155f1_gfstracker_@H - nems - batch - orion - 00:10:00 - 1:ppn=1:tpp=1 - 4G - --export=NONE - - /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfstracker.log - - RUN_ENVIRemc - HOMEgfs/CI/ORION/2555/gfs - EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/CI/ORION/2555/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - - - - - - - - - - /CI/ORION/2555/gfs/jobs/rocoto/genesis.sh - - C48_ATM_0ad155f1_gfsgenesis_@H - nems - batch - orion - 00:25:00 - 1:ppn=1:tpp=1 - 10G - --export=NONE - - /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsgenesis.log - - RUN_ENVIRemc - HOMEgfs/CI/ORION/2555/gfs - EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/CI/ORION/2555/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - - - - - - - - - - /CI/ORION/2555/gfs/jobs/rocoto/arch.sh - - C48_ATM_0ad155f1_gfsarch_@H - nems - batch - service - 06:00:00 - 1:ppn=1:tpp=1 - 4096M - --export=NONE - - /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsarch.log - - RUN_ENVIRemc - HOMEgfs/CI/ORION/2555/gfs - EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/CI/ORION/2555/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - - - - - - - - - - - - - - /CI/ORION/2555/gfs/jobs/rocoto/cleanup.sh - - C48_ATM_0ad155f1_gfscleanup_@H - nems - batch - orion - 00:15:00 - 1:ppn=1:tpp=1 - 4096M - --export=NONE - - /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfscleanup.log - - RUN_ENVIRemc - HOMEgfs/CI/ORION/2555/gfs - EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1 - NETgfs - CDUMPgfs - RUNgfs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/CI/ORION/2555/RUNTESTS/COMROOT - DATAROOT/RUNDIRS/C48_ATM_0ad155f1 - - - - - - - - - - diff --git a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.db b/ci/scripts/tests/testdata/rocotostat_stalled/stalled.db deleted file mode 100644 index 976ee0605c..0000000000 Binary files a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.db and /dev/null differ diff --git a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.xml b/ci/scripts/tests/testdata/rocotostat_stalled/stalled.xml deleted file mode 100644 index dd4e5a6a68..0000000000 --- a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.xml +++ /dev/null @@ -1,443 +0,0 @@ - - - - - - -]> - - - - /RUNTESTS/EXPDIR/C48_S2SWA_gefs/logs/@Y@m@d@H.log - - - 202103231200 202103231200 24:00:00 - - - - /gefs/jobs/rocoto/stage_ic.sh - - C48_S2SWA_gefs_stage_ic_@H - nems - batch - orion - 00:15:00 - 1:ppn=1:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/stage_ic.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - - - - /work/noaa/global/glopara/data/ICSDIR/prototype_ICs/gefs_test/@Y@m@d@H/mem000/atmos/gfs_ctrl.nc - /work/noaa/global/glopara/data/ICSDIR/prototype_ICs/gefs_test/@Y@m@d@H/mem000/wave/@Y@m@d.@H0000.restart.glo_500 - - - - - - - - /gefs/jobs/rocoto/waveinit.sh - - C48_S2SWA_gefs_wave_init_@H - nems - batch - orion - 00:10:00 - 1:ppn=40:tpp=1 - 2GB - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/wave_init.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - - - - - - /gefs/jobs/rocoto/fcst.sh - - C48_S2SWA_gefs_fcst_mem000_@H - nems - batch - orion - 03:00:00 - 1:ppn=40:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/fcst_mem000.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - - - - - - - - - - - - - 001 002 - - - - /gefs/jobs/rocoto/fcst.sh - - C48_S2SWA_gefs_fcst_mem#member#_@H - nems - batch - orion - 03:00:00 - 1:ppn=40:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/fcst_mem#member#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - ENSMEM#member# - MEMDIRmem#member# - - - - - - - - - - - - - - - 000 001 002 - - - - 000 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120 - - - - /gefs/jobs/rocoto/atmos_products.sh - - C48_S2SWA_gefs_atmos_prod_mem#member#_f#fhr#_@H - nems - batch - orion - 00:15:00 - 1:ppn=24:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/atmos_prod_mem#member#_f#fhr#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - ENSMEM#member# - MEMDIRmem#member# - FHRLST#fhr# - COMPONENTatmos - - - &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/atmos/master/gefs.t@Hz.master.grb2f#fhr# - - - - - - - - - - - 000 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120 - - - - /gefs/jobs/rocoto/atmos_ensstat.sh - - C48_S2SWA_gefs_atmos_ensstat_f#fhr#_@H - nems - batch - orion - 00:30:00 - 1:ppn=6:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/atmos_ensstat_f#fhr#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - FHRLST#fhr# - - - - - - - - - - - - - - - - 000 001 002 - - - - 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120 - - - - /gefs/jobs/rocoto/oceanice_products.sh - - C48_S2SWA_gefs_ocean_prod_mem#member#_f#fhr#_@H - nems - batch - orion - 00:15:00 - 1:ppn=1:tpp=1 - 96GB - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/ocean_prod_mem#member#_f#fhr#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - ENSMEM#member# - MEMDIRmem#member# - FHRLST#fhr# - COMPONENTocean - - - - &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/ocean/history/gefs.ocean.t@Hz.24hr_avg.f#fhr#.nc - /gefs/ush/check_netcdf.sh &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/ocean/history/gefs.ocean.t@Hz.24hr_avg.f#fhr#.nc - - - - - - - - - - - - 000 001 002 - - - - 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120 - - - - /gefs/jobs/rocoto/oceanice_products.sh - - C48_S2SWA_gefs_ice_prod_mem#member#_f#fhr#_@H - nems - batch - orion - 00:15:00 - 1:ppn=1:tpp=1 - 96GB - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/ice_prod_mem#member#_f#fhr#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - ENSMEM#member# - MEMDIRmem#member# - FHRLST#fhr# - COMPONENTice - - - &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/ice/history/gefs.ice.t@Hz.24hr_avg.f#fhr#.nc - - - - - - - - - - - 000 001 002 - - - - /gefs/jobs/rocoto/wavepostsbs.sh - - C48_S2SWA_gefs_wave_post_grid_mem#member#_@H - nems - batch - orion - 03:00:00 - 1:ppn=40:tpp=1 - 10GB - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/wave_post_grid_mem#member#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - ENSMEM#member# - MEMDIRmem#member# - - - - &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/wave/history/gefswave.out_grd.glo_500.@Y@m@d.@H0000 - - - - - - - - - - 000 001 002 - - - - /gefs/jobs/rocoto/wavepostpnt.sh - - C48_S2SWA_gefs_wave_post_pnt_mem#member#_@H - nems - batch - orion - 04:00:00 - 5:ppn=40:tpp=1 - --export=NONE - - /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/wave_post_pnt_mem#member#.log - - RUN_ENVIRemc - HOMEgfs/gefs - EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs - NETgefs - CDUMPgefs - RUNgefs - CDATE@Y@m@d@H - PDY@Y@m@d - cyc@H - COMROOT/RUNTESTS/COMROOT - DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs - ENSMEM#member# - MEMDIRmem#member# - - - - - - - - - - - - diff --git a/ci/scripts/tests/testdata/rocotostat_stalled/stub b/ci/scripts/tests/testdata/rocotostat_stalled/stub new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py index fe8c84b5e7..169a66754a 100755 --- a/ci/scripts/utils/rocotostat.py +++ b/ci/scripts/utils/rocotostat.py @@ -3,7 +3,7 @@ import sys import os -from wxflow import Executable, which, Logger, CommandNotFoundError +from wxflow import which, Logger, CommandNotFoundError from argparse import ArgumentParser, FileType from collections import Counter