[develop] Add job cards for wrappers for individual machines and tasks (

#924)

Added job cards for individual tasks, similar to wrapper scripts, but tailored for each job scheduler used on NOAA HPCs (PBS Pro and Slurm).

---------

Co-authored-by: Ratko Vasic <[email protected]>
Co-authored-by: Michael Kavulich <[email protected]>

.cicd/scripts/srw_ftest.sh

@@ -102,7 +102,7 @@ cd ${workspace}
 
 cd ${EXPTDIR}
 pwd
-cp ${workspace}/ush/wrappers/* .
+cp ${workspace}/ush/wrappers/*.sh .
 
 # Set parameters that the task scripts require ...
 export JOBSdir=${workspace}/jobs

ush/wrappers/job_cards/README

@@ -0,0 +1,13 @@
+Here are included examples of job cards for two different workload managers:
+Slurm (sbatch) and PBS (qsub).
+Each directory containds examples of job cards for seven different tasks:
+get_ics, get_lbcs, make_grid, make_ics, make_lbcs, make_orog, make_sfc_climo,
+run_fcst and run_post.
+
+Some machines have slightly different commands and examples of these differences are 
+included inside each job card sample.
+
+When using Rocoto workflow, you can find printout of your particular
+job card in log files. For that print make sure to use -v option when
+invoking rocoto commands (rocotorun, rocotorewind, rocotoboot, ...).
+

ush/wrappers/job_cards/qsub/get_ics.qsub

@@ -0,0 +1,23 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=1:mpiprocs=1:ncpus=1:mem=2G
+#PBS -q main
+#PBS -l walltime=00:30:00
+#PBS -j oe -o get_extrn_ics.log
+#PBS -N get_extrn_ics
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export nprocs='1'
+export subcyc='00'
+export ENSMEM_INDX='#mem#'
+export ICS_OR_LBCS='ICS'
+
+$USHdir/load_modules_run_task.sh "get_extrn_ics" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES
+
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/get_lbcs.qsub

@@ -0,0 +1,22 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=1:mpiprocs=1:ncpus=1:mem=2G
+#PBS -q main
+#PBS -l walltime=01:30:00
+#PBS -j oe -o get_extrn_lbcs.log
+#PBS -N get_extrn_lbcs
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export nprocs='1'
+export subcyc='00'
+export ENSMEM_INDX='#mem#'
+export ICS_OR_LBCS='LBCS'
+
+$USHdir/load_modules_run_task.sh "get_extrn_lbcs" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/make_grid.qsub

@@ -0,0 +1,16 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=1:mpiprocs=24:ncpus=24
+#PBS -q main
+#PBS -l walltime=00:20:00
+#PBS -j oe -o make_grid.log
+#PBS -N make_grid
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+
+$USHdir/load_modules_run_task.sh "make_grid" $JOBSdir/JREGIONAL_MAKE_GRID
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/make_ics.qsub

@@ -0,0 +1,22 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=4:mpiprocs=12:ncpus=12
+#PBS -q main
+#PBS -l walltime=00:30:00
+#PBS -j oe -o make_ics.log
+#PBS -N make_ics
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export nprocs='48'
+export subcyc='00'
+export ENSMEM_INDX='000'
+export SLASH_ENSMEM_SUBDIR='/'
+
+$USHdir/load_modules_run_task.sh "make_ics" $JOBSdir/JREGIONAL_MAKE_ICS
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/make_lbcs.qsub

@@ -0,0 +1,24 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=4:mpiprocs=12:ncpus=12
+#PBS -q main
+#PBS -l walltime=00:30:00
+#PBS -j oe -o make_lbcs.log
+#PBS -N make_lbcs
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export nprocs='48'
+export subcyc='00'
+export ENSMEM_INDX='000'
+export SLASH_ENSMEM_SUBDIR='/'
+export bcgrp='00'
+export bcgrpnum='1'
+
+$USHdir/load_modules_run_task.sh "make_lbcs" $JOBSdir/JREGIONAL_MAKE_LBCS
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/make_orog.qsub

@@ -0,0 +1,17 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=1:mpiprocs=24:ncpus=24
+#PBS -q main
+#PBS -l walltime=00:20:00
+#PBS -j oe -o make_orog.log
+#PBS -N make_orog
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export nprocs='24'
+
+$USHdir/load_modules_run_task.sh "make_orog" $JOBSdir/JREGIONAL_MAKE_OROG
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/make_sfc_climo.qsub

@@ -0,0 +1,17 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=2:mpiprocs=24:ncpus=24
+#PBS -q main
+#PBS -l walltime=00:20:00
+#PBS -j oe -o make_sfc_climo.log
+#PBS -N make_sfc_climo
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export nprocs='48'
+
+$USHdir/load_modules_run_task.sh "make_sfc_climo" $JOBSdir/JREGIONAL_MAKE_SFC_CLIMO
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/run_fcst.qsub

@@ -0,0 +1,21 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -l select=1:mpiprocs=36:ncpus=36
+#PBS -q main
+#PBS -l walltime=00:30:00
+#PBS -j oe -o run_fcst.log
+#PBS -N run_fcst
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export subcyc='00'
+export ENSMEM_INDX='000'
+export SLASH_ENSMEM_SUBDIR='/'
+
+$USHdir/load_modules_run_task.sh "run_fcst" $JOBSdir/JREGIONAL_RUN_FCST
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/qsub/run_post.qsub

@@ -0,0 +1,25 @@
+#! /bin/sh
+#PBS -A epic
+#PBS -j oe -o run_post.log
+#PBS -l select=2:mpiprocs=24:ncpus=24
+#PBS -q main
+#PBS -l walltime=00:15:00
+#PBS -N run_post
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export SLASH_ENSMEM_SUBDIR='/'
+export ENSMEM_INDX='000'
+export nprocs='48'
+
+num_fcst_hrs=${FCST_LEN_HRS}
+for (( i=0; i<=$((num_fcst_hrs)); i++ )); do
+  export fhr=`printf "%03i" ${i}`
+$USHdir/load_modules_run_task.sh "run_post" $JOBSdir/JREGIONAL_RUN_POST
+done
+
+# for Cheyenne replace main with regular

ush/wrappers/job_cards/sbatch/get_ics.sbatch

@@ -0,0 +1,34 @@
+#! /bin/sh
+#SBATCH --account=epic
+#SBATCH --nodes=1-1
+#SBATCH --tasks-per-node=1
+#SBATCH --partition=service
+#SBATCH --qos=batch
+#SBATCH -t 00:05:00
+#SBATCH -o get_extrn_ics.log
+#SBATCH --mem=2048
+#SBATCH --job-name=get_extrn_ics
+#SBATCH --export=NONE
+
+# Prepared for Hera mchine
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export nprocs='1'
+export subcyc='00'
+export ENSMEM_INDX='#mem#'
+export ICS_OR_LBCS='ICS'
+
+$USHdir/load_modules_run_task.sh "get_extrn_ics" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES
+
+# Gaea-c5 differences:
+##SBATCH --qos=normal
+##SBATCH --clusters=es --partition=eslogin_c5 --export=NONE
+
+# Gaea    differences:
+##SBATCH --qos=windfall
+##SBATCH --clusters=es --export=NONE

ush/wrappers/job_cards/sbatch/get_lbcs.sbatch

@@ -0,0 +1,34 @@
+#! /bin/sh
+#SBATCH --account=epic
+#SBATCH --nodes=1-1
+#SBATCH --tasks-per-node=1
+#SBATCH --partition=service
+#SBATCH --qos=batch
+#SBATCH -t 00:05:00
+#SBATCH -o get_extrn_lbcs.log
+#SBATCH --mem=2048
+#SBATCH --job-name=get_extrn_lbcs
+#SBATCH --export=NONE
+
+# Prepared for Hera mchine
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export nprocs='1'
+export subcyc='00'
+export ENSMEM_INDX='#mem#'
+export ICS_OR_LBCS='LBCS'
+
+$USHdir/load_modules_run_task.sh "get_extrn_lbcs" $JOBSdir/JREGIONAL_GET_EXTRN_MDL_FILES
+
+# Gaea-c5 differences:
+##SBATCH --qos=normal
+##SBATCH --clusters=es --partition=eslogin_c5 --export=NONE
+
+# Gaea    differences:
+##SBATCH --qos=windfall
+##SBATCH --clusters=es --export=NONE

ush/wrappers/job_cards/sbatch/make_grid.sbatch

@@ -0,0 +1,35 @@
+#! /bin/sh
+#SBATCH --account=epic
+#SBATCH --nodes=1-1
+#SBATCH --tasks-per-node=24
+#SBATCH --qos=batch
+#SBATCH -t 00:05:00
+#SBATCH -o make_grid.log
+#SBATCH --job-name=make_grid
+#SBATCH --export=NONE
+
+# Prepared for Hera mchine
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+
+$USHdir/load_modules_run_task.sh "make_grid" $JOBSdir/JREGIONAL_MAKE_GRID
+
+# Gaea-c5 differences:
+##SBATCH --qos=normal
+##SBATCH --clusters=c5 --partition=batch --export=NONE
+#export nprocs='24'
+
+# Gaea    differences:
+##SBATCH --qos=windfall
+##SBATCH --clusters=c4 --export=NONE
+#export nprocs='24'
+
+# Jet     differences:
+##SBATCH --partition=sjet,vjet,kjet,xjet
+#export nprocs='24'
+
+# Orion   differences:
+##SBATCH --partition=orion

ush/wrappers/job_cards/sbatch/make_ics.sbatch

@@ -0,0 +1,41 @@
+#! /bin/sh
+#SBATCH --account=epic
+#SBATCH --nodes=4-4
+#SBATCH --tasks-per-node=12
+#SBATCH --qos=batch
+#SBATCH -t 00:30:00
+#SBATCH -o make_ics.log
+#SBATCH --job-name=make_ics
+#SBATCH --export=NONE
+
+# Prepared for Hera mchine
+
+#--- For this to work properly, make sure the location of your working directory is correct relative to the var_defns.sh file ---#
+export GLOBAL_VAR_DEFNS_FP=$PWD'/../var_defns.sh'
+export USHdir=`grep USHdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export JOBSdir=`grep JOBSdir $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2`
+export PDY=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c1-8`
+export cyc=`grep DATE_FIRST_CYCL $GLOBAL_VAR_DEFNS_FP | cut -d\' -f2 | cut -c9-10`
+export subcyc='00'
+export ENSMEM_INDX='000'
+export SLASH_ENSMEM_SUBDIR='/'
+export NWGES_DIR=$PWD'/../../../nco_dirs/nwges/20190615'
+
+$USHdir/load_modules_run_task.sh "make_ics" $JOBSdir/JREGIONAL_MAKE_ICS
+
+# Gaea-c5 differences:
+##SBATCH --qos=normal
+##SBATCH --clusters=c5 --partition=batch --export=NONE
+#export nprocs='48'
+
+# Gaea    differences:
+##SBATCH --qos=windfall
+##SBATCH --clusters=c4 --export=NONE
+#export nprocs='48'
+
+# Jet     differences:
+##SBATCH --partition=sjet,vjet,kjet,xjet
+#export nprocs='48'
+
+# Orion   differences:
+##SBATCH --partition=orion