debugging for ocean ensemble run

ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml

@@ -0,0 +1,32 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 48
+  resdetocean: 5.0
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  #icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500
+  idate: 2021032412
+  edate: 2021032418
+  #nens: 0
+  #icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500
+  #idate: 2021032312
+  #edate: 2021032318
+  #icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48O500
+  #idate: 2021032412
+  #edate: 2021032418
+  nens: 3
+  icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500ens
+  gfs_cyc: 0
+  start: warm
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - wcoss2
+  - orion
+  - hercules

jobs/JGLOBAL_MARINE_BMAT

@@ -2,17 +2,21 @@
 
 source "${HOMEgfs}/ush/preamble.sh"
 
-if (( 10#${ENSMEM:-0} > 0 )); then
+# source config.base, config.ocnanal and config.marinebmat
+# and pass marinebmat to ${machine}.env
+source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat"
+
+
+#if (( 10#${ENSMEM:-0} > 0 )); then
+if (( 10#${NMEM_ENS:-0} > 0 )); then
   export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}"
-  export DATA="${DATAjob}/${jobid}"
+  #export DATA="${DATAjob}/${jobid}"
+  export DATA=${DATA:-${DATAROOT}/${jobid:?}}
   # Create the directory to hold ensemble perturbations
   export DATAenspert="${DATAjob}/enspert"
   if [[ ! -d "${DATAenspert}" ]]; then mkdir -p "${DATAenspert}"; fi
 fi
 
-# source config.base, config.ocnanal and config.marinebmat
-# and pass marinebmat to ${machine}.env
-source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat"
 
 ##############################################
 # Set variables used in the script

ush/forecast_postdet.sh

@@ -428,11 +428,12 @@ MOM6_postdet() {
 
     # GEFS perturbations
     # TODO if [[ $RUN} == "gefs" ]] block maybe be needed
+    # yup
     #     to ensure it does not interfere with the GFS when ensemble is updated in the GFS
-    if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then
-      ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \
-      || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 )
-    fi
+    #if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then
+    #  ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \
+    #  || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 )
+    #fi
   fi  # if [[ "${RERUN}" == "NO" ]]; then
 
   # Link output files

ush/python/pygfs/task/marine_bmat.py

@@ -81,6 +81,7 @@ def initialize(self: Task) -> None:
         # stage backgrounds
         # TODO(G): Check ocean backgrounds dates for consistency
         bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config)
+        print("bkg_list: ",bkg_list)
         FileHandler(bkg_list).sync()
         for cice_fname in ['./INPUT/cice.res.nc', './bkg/ice.bkg.f006.nc', './bkg/ice.bkg.f009.nc']:
             mdau.cice_hist2fms(cice_fname, cice_fname)
@@ -214,6 +215,7 @@ def vertical_diffusion(self: Task) -> None:
         # compute the coefficients of the diffusion operator
         mdau.run(exec_cmd)
 
+
     @logit(logger)
     def ensemble_perturbations(self: Task) -> None:
         """Generate the 3D ensemble of perturbation for the 3DEnVAR
@@ -323,7 +325,7 @@ def finalize(self: Task) -> None:
         FileHandler({'copy': diagb_list}).sync()
 
         # Copy the ensemble perturbation diagnostics to the ROTDIR
-        if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3:
+        if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2:
             window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ')
             weight_list = []
             src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc")

ush/python/pygfs/utils/marine_da_utils.py

@@ -1,6 +1,7 @@
 import f90nml
-import os
 from logging import getLogger
+import os
+import re
 import xarray as xr
 
 from wxflow import (FileHandler,
@@ -97,3 +98,8 @@ def stage_ens_mem(task_config: AttrDict) -> None:
     letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf)
     logger.info(f"{letkf_stage_list}")
     FileHandler(letkf_stage_list).sync()
+
+    for _, ens_mem_file in letkf_stage_list['copy']:
+        print("ens_mem_file: ", ens_mem_file)
+        if re.fullmatch(r'ice\.\d+\.nc', os.path.basename(ens_mem_file)):
+            cice_hist2fms(ens_mem_file, ens_mem_file)

workflow/setup_expt.py

@@ -168,6 +168,12 @@ def link_files_from_src_to_dst(src_dir, dst_dir):
                 makedirs_if_missing(dst_dir)
                 link_files_from_src_to_dst(src_dir, dst_dir)
 
+                dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_ice_anl_dir)
+                src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ice_anl_dir)
+                makedirs_if_missing(dst_dir)
+                link_files_from_src_to_dst(src_dir, dst_dir)
+
+
             # Link mediator files
             if do_med:
                 dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir)