Merge branch 'NOAA-EMC:develop' into upstream/develop

Externals.cfg

@@ -57,7 +57,7 @@ protocol = git
 required = False
 
 [GDASApp]
-hash = ac8fdb1
+hash = d347d22
 local_path = sorc/gdas.cd
 repo_url = https://github.com/NOAA-EMC/GDASApp.git
 protocol = git

ci/cases/weekly/C384C192_hybatmda.yaml

@@ -9,7 +9,7 @@ arguments:
   resens: 192
   comrot: ${RUNTESTS}/COMROT
   expdir: ${RUNTESTS}/EXPDIR
-  icsdir: /scratch1/NCEPDEV/stmp2/Rahul.Mahajan/ICSDIR/IC_C384
+  icsdir: ${ICSDIR_ROOT}/C384C192
   idate: 2023040118
   edate: 2023040200
   nens: 2

ci/cases/weekly/C384_atm3DVar.yaml

@@ -5,10 +5,11 @@ experiment:
 arguments:
   pslot: ${pslot}
   app: ATM
-  resdet: C384
+  resdet: 384
+  resens: 192
   comrot: ${RUNTESTS}/COMROT
   expdir: ${RUNTESTS}/EXPDIR
-  icsdir: /scratch1/NCEPDEV/stmp2/Rahul.Mahajan/ICSDIR/IC_C384
+  icsdir: ${ICSDIR_ROOT}/C384C192
   idate: 2023040118
   edate: 2023040200
   nens: 0

ci/platforms/hera.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/bash
+
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
+export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+export STMP="/scratch1/NCEPDEV/stmp2/${USER}"
 export SLURM_ACCOUNT=nems
-export ICSDIR_ROOT="/scratch1/NCEPDEV/global/glopara/data/ICSDIR"
 export max_concurrent_cases=2
 export max_concurrent_pr=2

ci/platforms/orion.sh

@@ -2,6 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+export STMP="/work/noaa/stmp/${USER}"
 export SLURM_ACCOUNT=nems
 export max_concurrent_cases=2
 export max_concurrent_pr=2

ci/scripts/driver.sh

@@ -153,6 +153,7 @@ for pr in ${pr_list}; do
     for yaml_config in "${HOMEgfs_PR}/ci/cases/pr/"*.yaml; do
       case=$(basename "${yaml_config}" .yaml) || true
       pslot="${case}_${pr_sha}"
+      rm -Rf "${STMP}/RUNDIRS/${pslot}"
       export pslot
       set +e
       "${HOMEgfs_PR}/ci/scripts/create_experiment.py" --yaml "${HOMEgfs_PR}/ci/cases/pr/${case}.yaml" --dir "${HOMEgfs_PR}"

ci/scripts/driver_weekly.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+set -eux
+
+##############################################################################################
+#
+# Script description: Top level driver script for running
+#                     weekly CI regression tests
+#
+# Abstract:
+#
+# This script runs the high resolution cases found in $HOMEgfs/ci/cases/weekly
+# from the develop branch for the global-workflow repo that are intended to run on a weekly basis
+# from a cron job. When run it will clone and build a new branch from the EMC's global-workflow and
+# and create a pr using GitHub CLI by moving and replacing the yaml case files in
+# ${HOMEgfs}/ci/cases/weekly to {HOMEgfs}/ci/cases/pr.  Then the requisite labels are added
+# so that the current BASH CI framework can then run these cases.  Since this script
+# creates a PR with the CI-Ready labels, the BASH CI framework will automatically run these cases 
+# from that point so it is only required to run this script once from a single machine.
+##############################################################################################
+
+#################################################################
+# TODO using static build for GitHub CLI until fixed in HPC-Stack
+#################################################################
+export GH=${HOME}/bin/gh
+export REPO_URL=${REPO_URL:-"https://github.com/NOAA-EMC/global-workflow.git"}
+
+################################################################
+# Setup the relative paths to scripts and PS4 for better logging
+################################################################
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../.." >/dev/null 2>&1 && pwd )"
+scriptname=$(basename "${BASH_SOURCE[0]}")
+echo "Begin ${scriptname} at $(date -u)" || true
+export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
+
+#########################################################################
+# Set up runtime environment variables for accounts on supported machines
+#########################################################################
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+case ${MACHINE_ID} in
+  hera | orion)
+    echo "Running Automated Testing on ${MACHINE_ID}"
+    source "${HOMEgfs}/ci/platforms/${MACHINE_ID}.sh"
+    ;;
+  *)
+    echo "Unsupported platform. Exiting with error."
+    exit 1
+    ;;
+esac
+
+######################################################
+# setup runtime env for correct python install and git
+######################################################
+set +x
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
+module load "module_gwsetup.${MACHINE_ID}"
+set -x
+
+#########################################################
+# Create a new branch from develop and move yaml files
+#########################################################
+branch="weekly_ci_$(date +%Y%m%d)"
+develop_dir="${GFS_CI_ROOT}/develop_temp"
+echo "Creating new branch ${branch} from develop on ${MACHINE_ID} in ${develop_dir}"
+rm -Rf "${develop_dir}"
+mkdir -p "${develop_dir}"
+cd "${develop_dir}" || exit 1
+git clone "${REPO_URL}"
+cd global-workflow || exit 1
+git checkout -b "${branch}"
+
+######################################################
+# move yaml files from ci/cases/weekly to ci/cases/pr 
+# and push new branch for PR weekly CI tests to GitHub
+
+rm -Rf ci/cases/pr
+mv ci/cases/weekly ci/cases/pr
+git add ci/cases
+git commit -m "Moved weekly cases files into pr for high resolution testing"
+git push --set-upstream origin "${branch}"
+
+####################################################################
+# Create Pull Request using GitHub CLI and add labels for CI testing
+####################################################################
+
+REPO_OWNER="NOAA-EMC"
+REPO_NAME="global-workflow"
+BASE_BRANCH="develop"
+HEAD_BRANCH="${branch}"
+PULL_REQUEST_TITLE="[DO NOT MERGE] Weekly High Resolution CI Tests $(date +'%A %b %d, %Y')"
+PULL_REQUEST_BODY="${PULL_REQUEST_TITLE}"
+PULL_REQUEST_LABELS=("CI/CD" "CI-Orion-Ready" "CI-Hera-Ready")
+
+"${GH}" repo set-default "${REPO_OWNER}/${REPO_NAME}"
+"${GH}" pr create --title "${PULL_REQUEST_TITLE}" --body "${PULL_REQUEST_BODY}" --base "${BASE_BRANCH}" --head "${HEAD_BRANCH}"
+"${GH}" pr ready --undo
+
+# Add labels to the pull request
+for label in "${PULL_REQUEST_LABELS[@]}"
+do
+  "${GH}" pr edit --add-label "${label}"
+done
+cd "${GFS_CI_ROOT}"
+rm -Rf "${develop_dir}"

docs/source/index.rst

@@ -10,7 +10,7 @@ Status
 ======
 
 * State of develop (HEAD) branch: GFSv17+ development
-* State of operations (dev/gfs.v16 branch): GFS v16.3.7 `tag: [gfs.v16.3.7] <https://github.com/NOAA-EMC/global-workflow/releases/tag/gfs.v16.3.7>`_
+* State of operations (dev/gfs.v16 branch): GFS v16.3.9 `tag: [gfs.v16.3.9] <https://github.com/NOAA-EMC/global-workflow/releases/tag/gfs.v16.3.9>`_
 
 =============
 Code managers

env/HERA.env

@@ -16,7 +16,7 @@ step=$1
 
 export npe_node_max=40
 export launcher="srun -l --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 #export I_MPI_ADJUST_ALLREDUCE=5
@@ -215,6 +215,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"
@@ -302,3 +303,4 @@ elif [[ "${step}" = "fit2obs" ]]; then
     export MPIRUN="${launcher} -n ${npe_fit2obs}"
 
 fi
+

env/JET.env

@@ -22,7 +22,7 @@ elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then
   export npe_node_max=40
 fi
 export launcher="srun -l --epilog=/apps/local/bin/report-mem --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -201,6 +201,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Jet
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"

env/ORION.env

@@ -16,7 +16,7 @@ step=$1
 
 export npe_node_max=40
 export launcher="srun -l --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export MPI_BUFS_PER_PROC=2048
@@ -214,6 +214,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Orion
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"

env/S4.env

@@ -21,7 +21,7 @@ elif [[ ${PARTITION_BATCH} = "ivy" ]]; then
    export npe_node_max=20
 fi
 export launcher="srun -l --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -187,6 +187,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on S4
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"

env/WCOSS2.env

@@ -198,6 +198,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_np:-${npe_post}} -ppn ${npe_node_post} --cpu-bind depth --depth ${NTHREADS_NP}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on WCOSS2
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -np ${npe_dwn} ${mpmd_opt}"

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_CHKPT

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_POST

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanalprep"

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_RUN

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}"

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalprep" -c "base ocnanal ocnanalprep"
 

jobs/JGLOBAL_ARCHIVE

@@ -12,7 +12,7 @@ export CDUMP=${RUN/enkf}
 YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMOS_GEMPAK \
   COM_ATMOS_GENESIS COM_ATMOS_HISTORY COM_ATMOS_INPUT COM_ATMOS_MASTER COM_ATMOS_RESTART \
   COM_ATMOS_TRACK COM_ATMOS_WMO \
-  COM_CHEM_HISTORY \
+  COM_CHEM_HISTORY COM_CHEM_ANALYSIS\
   COM_MED_RESTART \
   COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART \
   COM_OBS COM_TOP \

jobs/JGLOBAL_ATMOS_POST

@@ -51,7 +51,7 @@ fi
 for grid in '0p25' '0p50' '1p00'; do
   prod_dir="COM_ATMOS_GRIB_${grid}"
   GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
-  if [[ ! -d "${prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
 done
 
 if [ "${RUN}" = gfs ];then

jobs/JGLOBAL_PREP_OCEAN_OBS

@@ -1,5 +1,4 @@
 #!/bin/bash
-export STRICT="NO"
 source "${HOMEgfs}/ush/preamble.sh"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "prepoceanobs" -c "base prepoceanobs"
 

parm/config/gfs/config.landanl

@@ -8,7 +8,7 @@ echo "BEGIN: config.landanl"
 # Get task specific resources
 . "${EXPDIR}/config.resources" landanl
 
-obs_list_name=gdas_land_adpsfc_only.yaml
+obs_list_name=gdas_land_gts_only.yaml
 if [[ "${cyc}" = "18" ]]; then
     obs_list_name=gdas_land_prototype.yaml
 fi

parm/config/gfs/config.preplandobs

@@ -10,7 +10,6 @@ echo "BEGIN: config.preplandobs"
 
 export GTS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_gts.yaml"
 export BUFR2IODAX="${HOMEgfs}/exec/bufr2ioda.x"
-export BUFR2IODAYAML="${HOMEgfs}/sorc/gdas.cd/test/testinput/bufr_adpsfc_snow.yaml"
 export FIMS_NML_TMPL="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/fims.nml.j2"
 export IMS_OBS_LIST="${HOMEgfs}/sorc/gdas.cd/parm/land/prep/prep_ims.yaml"
 export CALCFIMSEXE="${HOMEgfs}/exec/calcfIMS.exe"