diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh index cdbf1db2a8..4b8b672905 100755 --- a/.cicd/scripts/srw_ftest.sh +++ b/.cicd/scripts/srw_ftest.sh @@ -102,7 +102,7 @@ cd ${workspace} cd ${EXPTDIR} pwd -cp ${workspace}/ush/wrappers/* . +cp ${workspace}/ush/wrappers/*.sh . # Set parameters that the task scripts require ... export JOBSdir=${workspace}/jobs diff --git a/ush/wrappers/job_cards/README b/ush/wrappers/job_cards/README new file mode 100644 index 0000000000..14b770a838 --- /dev/null +++ b/ush/wrappers/job_cards/README @@ -0,0 +1,13 @@ +Here are included examples of job cards for two different workload managers: +Slurm (sbatch) and PBS (qsub). +Each directory containds examples of job cards for seven different tasks: +get_ics, get_lbcs, make_grid, make_ics, make_lbcs, make_orog, make_sfc_climo, +run_fcst and run_post. + +Some machines have slightly different commands and examples of these differences are +included inside each job card sample. + +When using Rocoto workflow, you can find printout of your particular +job card in log files. For that print make sure to use -v option when +invoking rocoto commands (rocotorun, rocotorewind, rocotoboot, ...). + diff --git a/ush/wrappers/job_cards/qsub/get_ics.qsub b/ush/wrappers/job_cards/qsub/get_ics.qsub new file mode 100644 index 0000000000..da3343808a --- /dev/null +++ b/ush/wrappers/job_cards/qsub/get_ics.qsub @@ -0,0 +1,23 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=1:mpiprocs=1:ncpus=1:mem=2G +#PBS -q main +#PBS -l walltime=00:30:00 +#PBS -j oe -o get_extrn_ics.log +#PBS -N get_extrn_ics + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export nprocs='1' +export subcyc='00' +export ENSMEM_INDX='#mem#' +export ICS_OR_LBCS='ICS' + +$USHdir/load_modules_run_task.sh "get_extrn_ics" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES + + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/get_lbcs.qsub b/ush/wrappers/job_cards/qsub/get_lbcs.qsub new file mode 100644 index 0000000000..1095e1ed95 --- /dev/null +++ b/ush/wrappers/job_cards/qsub/get_lbcs.qsub @@ -0,0 +1,22 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=1:mpiprocs=1:ncpus=1:mem=2G +#PBS -q main +#PBS -l walltime=01:30:00 +#PBS -j oe -o get_extrn_lbcs.log +#PBS -N get_extrn_lbcs + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export nprocs='1' +export subcyc='00' +export ENSMEM_INDX='#mem#' +export ICS_OR_LBCS='LBCS' + +$USHdir/load_modules_run_task.sh "get_extrn_lbcs" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/make_grid.qsub b/ush/wrappers/job_cards/qsub/make_grid.qsub new file mode 100644 index 0000000000..a26aeb1878 --- /dev/null +++ b/ush/wrappers/job_cards/qsub/make_grid.qsub @@ -0,0 +1,16 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=1:mpiprocs=24:ncpus=24 +#PBS -q main +#PBS -l walltime=00:20:00 +#PBS -j oe -o make_grid.log +#PBS -N make_grid + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` + +$USHdir/load_modules_run_task.sh "make_grid" $JOBSdir/JREGIONAL_MAKE_GRID + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/make_ics.qsub b/ush/wrappers/job_cards/qsub/make_ics.qsub new file mode 100644 index 0000000000..193114f26f --- /dev/null +++ b/ush/wrappers/job_cards/qsub/make_ics.qsub @@ -0,0 +1,22 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=4:mpiprocs=12:ncpus=12 +#PBS -q main +#PBS -l walltime=00:30:00 +#PBS -j oe -o make_ics.log +#PBS -N make_ics + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export nprocs='48' +export subcyc='00' +export ENSMEM_INDX='000' +export SLASH_ENSMEM_SUBDIR='/' + +$USHdir/load_modules_run_task.sh "make_ics" $JOBSdir/JREGIONAL_MAKE_ICS + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/make_lbcs.qsub b/ush/wrappers/job_cards/qsub/make_lbcs.qsub new file mode 100644 index 0000000000..83ae53bd91 --- /dev/null +++ b/ush/wrappers/job_cards/qsub/make_lbcs.qsub @@ -0,0 +1,24 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=4:mpiprocs=12:ncpus=12 +#PBS -q main +#PBS -l walltime=00:30:00 +#PBS -j oe -o make_lbcs.log +#PBS -N make_lbcs + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export nprocs='48' +export subcyc='00' +export ENSMEM_INDX='000' +export SLASH_ENSMEM_SUBDIR='/' +export bcgrp='00' +export bcgrpnum='1' + +$USHdir/load_modules_run_task.sh "make_lbcs" $JOBSdir/JREGIONAL_MAKE_LBCS + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/make_orog.qsub b/ush/wrappers/job_cards/qsub/make_orog.qsub new file mode 100644 index 0000000000..8ab292f8b2 --- /dev/null +++ b/ush/wrappers/job_cards/qsub/make_orog.qsub @@ -0,0 +1,17 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=1:mpiprocs=24:ncpus=24 +#PBS -q main +#PBS -l walltime=00:20:00 +#PBS -j oe -o make_orog.log +#PBS -N make_orog + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export nprocs='24' + +$USHdir/load_modules_run_task.sh "make_orog" $JOBSdir/JREGIONAL_MAKE_OROG + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/make_sfc_climo.qsub b/ush/wrappers/job_cards/qsub/make_sfc_climo.qsub new file mode 100644 index 0000000000..8103ea6e6f --- /dev/null +++ b/ush/wrappers/job_cards/qsub/make_sfc_climo.qsub @@ -0,0 +1,17 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=2:mpiprocs=24:ncpus=24 +#PBS -q main +#PBS -l walltime=00:20:00 +#PBS -j oe -o make_sfc_climo.log +#PBS -N make_sfc_climo + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export nprocs='48' + +$USHdir/load_modules_run_task.sh "make_sfc_climo" $JOBSdir/JREGIONAL_MAKE_SFC_CLIMO + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/run_fcst.qsub b/ush/wrappers/job_cards/qsub/run_fcst.qsub new file mode 100644 index 0000000000..931a84ce4c --- /dev/null +++ b/ush/wrappers/job_cards/qsub/run_fcst.qsub @@ -0,0 +1,21 @@ +#! /bin/sh +#PBS -A epic +#PBS -l select=1:mpiprocs=36:ncpus=36 +#PBS -q main +#PBS -l walltime=00:30:00 +#PBS -j oe -o run_fcst.log +#PBS -N run_fcst + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export subcyc='00' +export ENSMEM_INDX='000' +export SLASH_ENSMEM_SUBDIR='/' + +$USHdir/load_modules_run_task.sh "run_fcst" $JOBSdir/JREGIONAL_RUN_FCST + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/qsub/run_post.qsub b/ush/wrappers/job_cards/qsub/run_post.qsub new file mode 100644 index 0000000000..296ce7b59e --- /dev/null +++ b/ush/wrappers/job_cards/qsub/run_post.qsub @@ -0,0 +1,25 @@ +#! /bin/sh +#PBS -A epic +#PBS -j oe -o run_post.log +#PBS -l select=2:mpiprocs=24:ncpus=24 +#PBS -q main +#PBS -l walltime=00:15:00 +#PBS -N run_post + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export SLASH_ENSMEM_SUBDIR='/' +export ENSMEM_INDX='000' +export nprocs='48' + +num_fcst_hrs=${FCST_LEN_HRS} +for (( i=0; i<=$((num_fcst_hrs)); i++ )); do + export fhr=`printf "%03i" ${i}` +$USHdir/load_modules_run_task.sh "run_post" $JOBSdir/JREGIONAL_RUN_POST +done + +# for Cheyenne replace main with regular diff --git a/ush/wrappers/job_cards/sbatch/get_ics.sbatch b/ush/wrappers/job_cards/sbatch/get_ics.sbatch new file mode 100644 index 0000000000..44e67f3c44 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/get_ics.sbatch @@ -0,0 +1,34 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=1-1 +#SBATCH --tasks-per-node=1 +#SBATCH --partition=service +#SBATCH --qos=batch +#SBATCH -t 00:05:00 +#SBATCH -o get_extrn_ics.log +#SBATCH --mem=2048 +#SBATCH --job-name=get_extrn_ics +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export nprocs='1' +export subcyc='00' +export ENSMEM_INDX='#mem#' +export ICS_OR_LBCS='ICS' + +$USHdir/load_modules_run_task.sh "get_extrn_ics" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=es --partition=eslogin_c5 --export=NONE + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=es --export=NONE diff --git a/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch b/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch new file mode 100644 index 0000000000..d5f037153b --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/get_lbcs.sbatch @@ -0,0 +1,34 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=1-1 +#SBATCH --tasks-per-node=1 +#SBATCH --partition=service +#SBATCH --qos=batch +#SBATCH -t 00:05:00 +#SBATCH -o get_extrn_lbcs.log +#SBATCH --mem=2048 +#SBATCH --job-name=get_extrn_lbcs +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export nprocs='1' +export subcyc='00' +export ENSMEM_INDX='#mem#' +export ICS_OR_LBCS='LBCS' + +$USHdir/load_modules_run_task.sh "get_extrn_lbcs" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=es --partition=eslogin_c5 --export=NONE + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=es --export=NONE diff --git a/ush/wrappers/job_cards/sbatch/make_grid.sbatch b/ush/wrappers/job_cards/sbatch/make_grid.sbatch new file mode 100644 index 0000000000..d452d64426 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/make_grid.sbatch @@ -0,0 +1,35 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=1-1 +#SBATCH --tasks-per-node=24 +#SBATCH --qos=batch +#SBATCH -t 00:05:00 +#SBATCH -o make_grid.log +#SBATCH --job-name=make_grid +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` + +$USHdir/load_modules_run_task.sh "make_grid" $JOBSdir/JREGIONAL_MAKE_GRID + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE +#export nprocs='24' + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE +#export nprocs='24' + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet +#export nprocs='24' + +# Orion differences: +##SBATCH --partition=orion diff --git a/ush/wrappers/job_cards/sbatch/make_ics.sbatch b/ush/wrappers/job_cards/sbatch/make_ics.sbatch new file mode 100644 index 0000000000..20b18605a0 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/make_ics.sbatch @@ -0,0 +1,41 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=4-4 +#SBATCH --tasks-per-node=12 +#SBATCH --qos=batch +#SBATCH -t 00:30:00 +#SBATCH -o make_ics.log +#SBATCH --job-name=make_ics +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export subcyc='00' +export ENSMEM_INDX='000' +export SLASH_ENSMEM_SUBDIR='/' +export NWGES_DIR=$PWD'/../../../nco_dirs/nwges/20190615' + +$USHdir/load_modules_run_task.sh "make_ics" $JOBSdir/JREGIONAL_MAKE_ICS + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE +#export nprocs='48' + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE +#export nprocs='48' + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet +#export nprocs='48' + +# Orion differences: +##SBATCH --partition=orion diff --git a/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch b/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch new file mode 100644 index 0000000000..a0ab0071a4 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/make_lbcs.sbatch @@ -0,0 +1,43 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=4-4 +#SBATCH --tasks-per-node=12 +#SBATCH --qos=batch +#SBATCH -t 00:30:00 +#SBATCH -o make_lbcs.log +#SBATCH --job-name=make_lbcs +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export subcyc='00' +export ENSMEM_INDX='000' +export SLASH_ENSMEM_SUBDIR='/' +export NWGES_DIR=$PWD'/../../../nco_dirs/nwges/2019061518' +export bcgrp='00' +export bcgrpnum='1' + +$USHdir/load_modules_run_task.sh "make_lbcs" $JOBSdir/JREGIONAL_MAKE_LBCS + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE +#export nprocs='48' + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE +#export nprocs='48' + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet +#export nprocs='48' + +# Orion differences: +##SBATCH --partition=orion diff --git a/ush/wrappers/job_cards/sbatch/make_orog.sbatch b/ush/wrappers/job_cards/sbatch/make_orog.sbatch new file mode 100644 index 0000000000..eebed3b5ce --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/make_orog.sbatch @@ -0,0 +1,35 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=1-1 +#SBATCH --tasks-per-node=24 +#SBATCH --qos=batch +#SBATCH -t 00:20:00 +#SBATCH -o make_orog.log +#SBATCH --job-name=make_orog +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` + +$USHdir/load_modules_run_task.sh "make_orog" $JOBSdir/JREGIONAL_MAKE_OROG + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE +#export nprocs='24' + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE +#export nprocs='24' + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet +#export nprocs='24' + +# Orion differences: +##SBATCH --partition=orion diff --git a/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch b/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch new file mode 100644 index 0000000000..47c7aed043 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/make_sfc_climo.sbatch @@ -0,0 +1,35 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --nodes=2-2 +#SBATCH --tasks-per-node=24 +#SBATCH --qos=batch +#SBATCH -t 00:20:00 +#SBATCH -o make_sfc_climo.log +#SBATCH --job-name=make_sfc_climo +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` + +$USHdir/load_modules_run_task.sh "make_sfc_climo" $JOBSdir/JREGIONAL_MAKE_SFC_CLIMO + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE +#export nprocs='48' + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE +#export nprocs='48' + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet +#export nprocs='48' + +# Orion differences: +##SBATCH --partition=orion diff --git a/ush/wrappers/job_cards/sbatch/run_fcst.sbatch b/ush/wrappers/job_cards/sbatch/run_fcst.sbatch new file mode 100644 index 0000000000..6a146a5627 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/run_fcst.sbatch @@ -0,0 +1,41 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH --qos=batch +#SBATCH -t 02:00:00 +#SBATCH -o run_fcst.log +#SBATCH --job-name=run_fcst +#SBATCH --ntasks=12 +#SBATCH --cpus-per-task 1 --exclusive --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export subcyc='00' +export ENSMEM_INDX='000' +export SLASH_ENSMEM_SUBDIR='/' + +$USHdir/load_modules_run_task.sh "run_fcst" $JOBSdir/JREGIONAL_RUN_FCST + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE +#export nprocs='48' + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE +#export nprocs='48' + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet + +# Orion differences: +##SBATCH --nodes=1-1 +##SBATCH --tasks-per-node=40 +##SBATCH --partition=orion +##SBATCH --export=NONE diff --git a/ush/wrappers/job_cards/sbatch/run_post.sbatch b/ush/wrappers/job_cards/sbatch/run_post.sbatch new file mode 100644 index 0000000000..5b180b82e1 --- /dev/null +++ b/ush/wrappers/job_cards/sbatch/run_post.sbatch @@ -0,0 +1,41 @@ +#! /bin/sh +#SBATCH --account=epic +#SBATCH -o run_post.log +#SBATCH --nodes=2-2 +#SBATCH --tasks-per-node=24 +#SBATCH --qos=batch +#SBATCH -t 00:30:00 +#SBATCH --job-name=run_post +#SBATCH --export=NONE + +# Prepared for Hera mchine + +#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---# +export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh' +export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2` +export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8` +export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10` +export SLASH_ENSMEM_SUBDIR='/' +export ENSMEM_INDX='000' +export nprocs='48' + +num_fcst_hrs=${FCST_LEN_HRS} +for (( i=0; i<=$((num_fcst_hrs)); i++ )); do + export fhr=`printf "%03i" ${i}` +$USHdir/load_modules_run_task.sh "run_post" $JOBSdir/JREGIONAL_RUN_POST +done + +# Gaea-c5 differences: +##SBATCH --qos=normal +##SBATCH --clusters=c5 --partition=batch --export=NONE + +# Gaea differences: +##SBATCH --qos=windfall +##SBATCH --clusters=c4 --export=NONE + +# Jet differences: +##SBATCH --partition=sjet,vjet,kjet,xjet + +# Orion differences: +##SBATCH --partition=orion diff --git a/ush/wrappers/qsub_job.sh b/ush/wrappers/qsub_job.sh deleted file mode 100755 index 8cd49aa57c..0000000000 --- a/ush/wrappers/qsub_job.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -#PBS -A XXXXXXXXX -#PBS -q regular -#PBS -l select=1:mpiprocs=24:ncpus=24 -##PBS -l select=1:mpiprocs=1:ncpus=1 # USE FOR MET VERIFICATION -#PBS -l walltime=02:30:00 -#PBS -N run_make_grid -#PBS -j oe -o /path/to/exptdir/log/run_make_grid.log # NEED TO SET -cd /path/to/exptdir # NEED TO SET -set -x -# -source /etc/profile.d/modules.sh -module load ncarenv/1.3 -module load intel/19.0.2 -module load mpt/2.19 -module load ncarcompilers/0.5.0 -module load netcdf/4.6.3 - -module use -a /glade/p/ral/jntp/GMTB/tools/modulefiles/intel-19.0.2/mpt-2.19 -module load esmf/8.0.0 -# -# Different modules are needed for the UFS_UTILS/mpi jobs... why are they using impi anyway??? -## make_sfc_climo make_ics make_lbcs -# -##module load ncarenv/1.3 -##module load intel/19.0.2 -##module load ncarcompilers/0.5.0 -##module load impi/2019.2.187 -##module load netcdf/4.6.3 -# -##module use -a /glade/p/ral/jntp/GMTB/tools/modulefiles/intel-19.0.2/impi-2019.2.187 -##module load esmf/8.0.0_bs50 -# -./run_make_grid.sh -# -# -# Additional modules are needed for MET verification jobs -# -#module use /glade/p/ral/jntp/MET/MET_releases/modulefiles -#module load met/10.0.0 -#module load conda/latest -#conda activate /glade/p/ral/jntp/UFS_SRW_app/conda/python_graphics - -#./run_pointvx.sh # Run grod-to-point deterministic vx -#./run_gridvx.sh # Run grid-stat deterministic vx -#./run_pointensvx.sh # Run grid-to-point ensemble/probabilsitic vx -#./rungridensvx.sh # Run grid-to-grid ensemble/probabilsitic vx diff --git a/ush/wrappers/sq_job.sh b/ush/wrappers/sq_job.sh deleted file mode 100755 index a9fa3c1805..0000000000 --- a/ush/wrappers/sq_job.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -#SBATCH -e /path/to/exptdir/log/run_make_grid.log # NEED TO SET -#SBATCH --account=XXXXXXXXX -#SBATCH --qos=batch -#SBATCH --ntasks=48 -##SBATCH --ntasks=1 # USE FOR MET VERIFICATION -#SBATCH --time=20 -#SBATCH --job-name="run_make_grid" -cd /path/to/exptdir # NEED TO SET -set -x -. /apps/lmod/lmod/init/sh - -module purge -module load hpss - -module load intel/18.0.5.274 -module load impi/2018.0.4 -module load wgrib2 -############ -# use this netcdf for most of the tasks -module load netcdf/4.7.0 -############ - -############ -# use this version for make_sfc_climo, make_ics and make_lbcs -#module load netcdf/4.6.1 -############ - -module load hdf5/1.10.5 - -############ -# use this for the forecast model -#module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles -#module load hdf5_parallel/1.10.6 -#module load netcdf_parallel/4.7.4 -#module load esmf/8.0.0_ParallelNetCDF -############ - - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -conda activate regional_workflow - -./run_make_grid.sh -# -# -# Additional modules are needed for MET verification jobs -# -#module use -a /contrib/anaconda/modulefiles -#module load intel/18.0.5.274 -#module load anaconda/latest -#module use -a /contrib/met/modulefiles/ -#module load met/10.0.0 - -#./run_pointvx.sh # Run grod-to-point deterministic vx -#./run_gridvx.sh # Run grid-stat deterministic vx -#./run_pointensvx.sh # Run grid-to-point ensemble/probabilsitic vx -#./run_gridensvx.sh # Run grid-to-grid ensemble/probabilsitic vx