diff --git a/ci/scripts/tests/test_rocotostat.py b/ci/scripts/tests/test_rocotostat.py
index 52737c4e8f..caf9ae4e6b 100755
--- a/ci/scripts/tests/test_rocotostat.py
+++ b/ci/scripts/tests/test_rocotostat.py
@@ -7,15 +7,25 @@
from rocotostat import rocoto_statcount, rocotostat_summary, is_done, is_stalled, CommandNotFoundError
from wxflow import which
-workflow_file = os.path.join(script_dir, "testdata/rocotostat/workflow.xml")
-database_file = os.path.join(script_dir, "testdata/rocotostat/database.db")
+test_data_url ='https://noaa-nws-global-pds.s3.amazonaws.com/data/CI/'
+
+testdata_path = 'testdata/rocotostat'
+testdata_full_path = os.path.join(script_dir, testdata_path)
+
+wget = which('wget')
+if not os.listdir(os.path.join(testdata_full_path)):
+ wget.add_default_arg(['-P', testdata_full_path,
+ test_data_url + str(testdata_path) + '/workflow.xml',
+ test_data_url + str(testdata_path) + '/database.db',
+ ])
+ wget()
try:
- rocotostat = which("rocotostat")
+ rocotostat = which('rocotostat')
except CommandNotFoundError:
raise CommandNotFoundError("rocotostat not found in PATH")
-rocotostat.add_default_arg(['-w', workflow_file, '-d', database_file])
+rocotostat.add_default_arg(['-w', os.path.join(testdata_path,'workflow.xml'), '-d', os.path.join(testdata_path, 'database.db')])
def test_rocoto_statcount():
@@ -45,10 +55,21 @@ def test_rocoto_done():
def test_rocoto_stalled():
+ testdata_path = 'testdata/rocotostat_stalled'
+ testdata_full_path = os.path.join(script_dir, testdata_path)
+
+ wget = which('wget')
+ if not os.listdir(os.path.join(testdata_full_path)):
+ wget.add_default_arg(['-P', testdata_full_path,
+ test_data_url + str(testdata_path) + '/stalled.xml',
+ test_data_url + str(testdata_path) + '/stalled.db',
+ ])
+ wget()
- workflow_file = os.path.join(script_dir, "testdata/rocotostat_stalled/stalled.xml")
- database_file = os.path.join(script_dir, "testdata/rocotostat_stalled/stalled.db")
+ rocotostat = which('rocotostat')
+ rocotostat.add_default_arg(['-w', os.path.join(testdata_path,'stalled.xml'), '-d', os.path.join(testdata_path, 'stalled.db')])
result = rocoto_statcount(rocotostat)
+ assert result['SUCCEEDED'] == 110
assert is_stalled(result)
diff --git a/ci/scripts/tests/testdata/rocotostat/database.db b/ci/scripts/tests/testdata/rocotostat/database.db
deleted file mode 100644
index baca6c7513..0000000000
Binary files a/ci/scripts/tests/testdata/rocotostat/database.db and /dev/null differ
diff --git a/ci/scripts/tests/testdata/rocotostat/stub b/ci/scripts/tests/testdata/rocotostat/stub
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ci/scripts/tests/testdata/rocotostat/workflow.xml b/ci/scripts/tests/testdata/rocotostat/workflow.xml
deleted file mode 100644
index d5171f647a..0000000000
--- a/ci/scripts/tests/testdata/rocotostat/workflow.xml
+++ /dev/null
@@ -1,272 +0,0 @@
-
-
-
-
-
-
-]>
-
-
-
- STUB
-
-
- 202103231200 202103231200 24:00:00
-
-
-
- stage_ic.sh
-
- C48_ATM_0ad155f1_gfsstage_ic_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=1:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsstage_ic.log
-
- RUN_ENVIRemc
- HOMEgfsgfs
- EXPDIRRUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
-
-
-
- /ICSDIR/prototype_ICs/workflow_C48_refactored/@Y@m@d@H/atmos/gfs_ctrl.nc
-
-
-
-
-
-
-
- /CI/ORION/2555/gfs/jobs/rocoto/fcst.sh
-
- C48_ATM_0ad155f1_gfsfcst_@H
- nems
- batch
- orion
- 03:00:00
- 1:ppn=40:tpp=1
- --export=NONE
-
- /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsfcst.log
-
- RUN_ENVIRemc
- HOMEgfs/CI/ORION/2555/gfs
- EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/CI/ORION/2555/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
-
-
-
-
-
-
-
-
-
-
-
- _f000-f006 _f009-f015 _f018-f024 _f027-f033 _f036-f042 _f045-f051 _f054-f060 _f063-f069 _f072-f078 _f081-f087 _f090-f096 _f099-f105 _f108-f114 _f117-f120
- f006 f015 f024 f033 f042 f051 f060 f069 f078 f087 f096 f105 f114 f120
- f000_f003_f006 f009_f012_f015 f018_f021_f024 f027_f030_f033 f036_f039_f042 f045_f048_f051 f054_f057_f060 f063_f066_f069 f072_f075_f078 f081_f084_f087 f090_f093_f096 f099_f102_f105 f108_f111_f114 f117_f120
-
-
-
- /CI/ORION/2555/gfs/jobs/rocoto/atmos_products.sh
-
- C48_ATM_0ad155f1_gfsatmos_prod#grp#_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=24:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsatmos_prod#grp#.log
-
- RUN_ENVIRemc
- HOMEgfs/CI/ORION/2555/gfs
- EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/CI/ORION/2555/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
- FHRLST#lst#
- COMPONENTatmos
-
-
- &ROTDIR;/gfs.@Y@m@d/@H//model_data/atmos/master/gfs.t@Hz.master.grb2#dep#
-
-
-
-
-
-
-
-
- /CI/ORION/2555/gfs/jobs/rocoto/tracker.sh
-
- C48_ATM_0ad155f1_gfstracker_@H
- nems
- batch
- orion
- 00:10:00
- 1:ppn=1:tpp=1
- 4G
- --export=NONE
-
- /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfstracker.log
-
- RUN_ENVIRemc
- HOMEgfs/CI/ORION/2555/gfs
- EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/CI/ORION/2555/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
-
-
-
-
-
-
-
-
-
- /CI/ORION/2555/gfs/jobs/rocoto/genesis.sh
-
- C48_ATM_0ad155f1_gfsgenesis_@H
- nems
- batch
- orion
- 00:25:00
- 1:ppn=1:tpp=1
- 10G
- --export=NONE
-
- /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsgenesis.log
-
- RUN_ENVIRemc
- HOMEgfs/CI/ORION/2555/gfs
- EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/CI/ORION/2555/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
-
-
-
-
-
-
-
-
-
- /CI/ORION/2555/gfs/jobs/rocoto/arch.sh
-
- C48_ATM_0ad155f1_gfsarch_@H
- nems
- batch
- service
- 06:00:00
- 1:ppn=1:tpp=1
- 4096M
- --export=NONE
-
- /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfsarch.log
-
- RUN_ENVIRemc
- HOMEgfs/CI/ORION/2555/gfs
- EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/CI/ORION/2555/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
-
-
-
-
-
-
-
-
-
-
-
-
-
- /CI/ORION/2555/gfs/jobs/rocoto/cleanup.sh
-
- C48_ATM_0ad155f1_gfscleanup_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=1:tpp=1
- 4096M
- --export=NONE
-
- /CI/ORION/2555/RUNTESTS/COMROOT/C48_ATM_0ad155f1/logs/@Y@m@d@H/gfscleanup.log
-
- RUN_ENVIRemc
- HOMEgfs/CI/ORION/2555/gfs
- EXPDIR/CI/ORION/2555/RUNTESTS/EXPDIR/C48_ATM_0ad155f1
- NETgfs
- CDUMPgfs
- RUNgfs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/CI/ORION/2555/RUNTESTS/COMROOT
- DATAROOT/RUNDIRS/C48_ATM_0ad155f1
-
-
-
-
-
-
-
-
-
-
diff --git a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.db b/ci/scripts/tests/testdata/rocotostat_stalled/stalled.db
deleted file mode 100644
index 976ee0605c..0000000000
Binary files a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.db and /dev/null differ
diff --git a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.xml b/ci/scripts/tests/testdata/rocotostat_stalled/stalled.xml
deleted file mode 100644
index dd4e5a6a68..0000000000
--- a/ci/scripts/tests/testdata/rocotostat_stalled/stalled.xml
+++ /dev/null
@@ -1,443 +0,0 @@
-
-
-
-
-
-
-]>
-
-
-
- /RUNTESTS/EXPDIR/C48_S2SWA_gefs/logs/@Y@m@d@H.log
-
-
- 202103231200 202103231200 24:00:00
-
-
-
- /gefs/jobs/rocoto/stage_ic.sh
-
- C48_S2SWA_gefs_stage_ic_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=1:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/stage_ic.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
-
-
-
- /work/noaa/global/glopara/data/ICSDIR/prototype_ICs/gefs_test/@Y@m@d@H/mem000/atmos/gfs_ctrl.nc
- /work/noaa/global/glopara/data/ICSDIR/prototype_ICs/gefs_test/@Y@m@d@H/mem000/wave/@Y@m@d.@H0000.restart.glo_500
-
-
-
-
-
-
-
- /gefs/jobs/rocoto/waveinit.sh
-
- C48_S2SWA_gefs_wave_init_@H
- nems
- batch
- orion
- 00:10:00
- 1:ppn=40:tpp=1
- 2GB
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/wave_init.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
-
-
-
-
-
- /gefs/jobs/rocoto/fcst.sh
-
- C48_S2SWA_gefs_fcst_mem000_@H
- nems
- batch
- orion
- 03:00:00
- 1:ppn=40:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/fcst_mem000.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
-
-
-
-
-
-
-
-
-
-
-
-
- 001 002
-
-
-
- /gefs/jobs/rocoto/fcst.sh
-
- C48_S2SWA_gefs_fcst_mem#member#_@H
- nems
- batch
- orion
- 03:00:00
- 1:ppn=40:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/fcst_mem#member#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- ENSMEM#member#
- MEMDIRmem#member#
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 000 001 002
-
-
-
- 000 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120
-
-
-
- /gefs/jobs/rocoto/atmos_products.sh
-
- C48_S2SWA_gefs_atmos_prod_mem#member#_f#fhr#_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=24:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/atmos_prod_mem#member#_f#fhr#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- ENSMEM#member#
- MEMDIRmem#member#
- FHRLST#fhr#
- COMPONENTatmos
-
-
- &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/atmos/master/gefs.t@Hz.master.grb2f#fhr#
-
-
-
-
-
-
-
-
-
-
- 000 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120
-
-
-
- /gefs/jobs/rocoto/atmos_ensstat.sh
-
- C48_S2SWA_gefs_atmos_ensstat_f#fhr#_@H
- nems
- batch
- orion
- 00:30:00
- 1:ppn=6:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/atmos_ensstat_f#fhr#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- FHRLST#fhr#
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 000 001 002
-
-
-
- 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120
-
-
-
- /gefs/jobs/rocoto/oceanice_products.sh
-
- C48_S2SWA_gefs_ocean_prod_mem#member#_f#fhr#_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=1:tpp=1
- 96GB
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/ocean_prod_mem#member#_f#fhr#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- ENSMEM#member#
- MEMDIRmem#member#
- FHRLST#fhr#
- COMPONENTocean
-
-
-
- &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/ocean/history/gefs.ocean.t@Hz.24hr_avg.f#fhr#.nc
- /gefs/ush/check_netcdf.sh &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/ocean/history/gefs.ocean.t@Hz.24hr_avg.f#fhr#.nc
-
-
-
-
-
-
-
-
-
-
-
- 000 001 002
-
-
-
- 006 012 018 024 030 036 042 048 054 060 066 072 078 084 090 096 102 108 114 120
-
-
-
- /gefs/jobs/rocoto/oceanice_products.sh
-
- C48_S2SWA_gefs_ice_prod_mem#member#_f#fhr#_@H
- nems
- batch
- orion
- 00:15:00
- 1:ppn=1:tpp=1
- 96GB
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/ice_prod_mem#member#_f#fhr#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- ENSMEM#member#
- MEMDIRmem#member#
- FHRLST#fhr#
- COMPONENTice
-
-
- &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/ice/history/gefs.ice.t@Hz.24hr_avg.f#fhr#.nc
-
-
-
-
-
-
-
-
-
-
- 000 001 002
-
-
-
- /gefs/jobs/rocoto/wavepostsbs.sh
-
- C48_S2SWA_gefs_wave_post_grid_mem#member#_@H
- nems
- batch
- orion
- 03:00:00
- 1:ppn=40:tpp=1
- 10GB
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/wave_post_grid_mem#member#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- ENSMEM#member#
- MEMDIRmem#member#
-
-
-
- &ROTDIR;/gefs.@Y@m@d/@H/mem#member#/model_data/wave/history/gefswave.out_grd.glo_500.@Y@m@d.@H0000
-
-
-
-
-
-
-
-
-
- 000 001 002
-
-
-
- /gefs/jobs/rocoto/wavepostpnt.sh
-
- C48_S2SWA_gefs_wave_post_pnt_mem#member#_@H
- nems
- batch
- orion
- 04:00:00
- 5:ppn=40:tpp=1
- --export=NONE
-
- /RUNTESTS/COMROOT/C48_S2SWA_gefs/logs/@Y@m@d@H/wave_post_pnt_mem#member#.log
-
- RUN_ENVIRemc
- HOMEgfs/gefs
- EXPDIR/RUNTESTS/EXPDIR/C48_S2SWA_gefs
- NETgefs
- CDUMPgefs
- RUNgefs
- CDATE@Y@m@d@H
- PDY@Y@m@d
- cyc@H
- COMROOT/RUNTESTS/COMROOT
- DATAROOT/work/noaa/stmp/mterry/RUNDIRS/C48_S2SWA_gefs
- ENSMEM#member#
- MEMDIRmem#member#
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/ci/scripts/tests/testdata/rocotostat_stalled/stub b/ci/scripts/tests/testdata/rocotostat_stalled/stub
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ci/scripts/utils/rocotostat.py b/ci/scripts/utils/rocotostat.py
index fe8c84b5e7..169a66754a 100755
--- a/ci/scripts/utils/rocotostat.py
+++ b/ci/scripts/utils/rocotostat.py
@@ -3,7 +3,7 @@
import sys
import os
-from wxflow import Executable, which, Logger, CommandNotFoundError
+from wxflow import which, Logger, CommandNotFoundError
from argparse import ArgumentParser, FileType
from collections import Counter