From 17494dece06ebff058f52f5004eb75ceba89ad35 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 1 Jul 2024 17:21:37 -0400 Subject: [PATCH 01/26] 3dfgat gdas.cd ctests work but needs tidy --- env/HERA.env | 8 +- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP | 4 + jobs/JGLOBAL_MARINE_BMAT | 56 ++++ jobs/rocoto/marinebmat.sh | 26 ++ parm/config/gfs/config.com | 2 + parm/config/gfs/config.marinebmat | 11 + parm/config/gfs/config.ocnanal | 1 + parm/config/gfs/config.resources | 2 +- scripts/exglobal_marine_bmat_run.py | 24 ++ sorc/link_workflow.sh | 3 +- ush/python/pygfs/task/marine_bmat.py | 412 ++++++++++++++++++++++++++ 11 files changed, 546 insertions(+), 3 deletions(-) create mode 100755 jobs/JGLOBAL_MARINE_BMAT create mode 100755 jobs/rocoto/marinebmat.sh create mode 100644 parm/config/gfs/config.marinebmat create mode 100755 scripts/exglobal_marine_bmat_run.py create mode 100644 ush/python/pygfs/task/marine_bmat.py diff --git a/env/HERA.env b/env/HERA.env index db63f0bfa5..78e8f15843 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -77,7 +77,7 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} - export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" elif [[ "${step}" = "aeroanlrun" ]]; then @@ -114,6 +114,12 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "marinebmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_MARINEBMAT="${launcher} -n ${npe_ocnanalbmat}" + elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP index 73c5c4db75..bf714939f5 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_PREP @@ -28,6 +28,10 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COM_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL \ COM_ICE_RESTART_PREV:COM_ICE_RESTART_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL + ############################################## # Begin JOB SPECIFIC work ############################################## diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT new file mode 100755 index 0000000000..1710eb987e --- /dev/null +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -0,0 +1,56 @@ +#!/bin/bash +source "${HOMEgfs}/ush/preamble.sh" +export DATA="${DATAROOT}/${RUN}marinebmat_${cyc}" +export DATA=${DATA:-${DATAROOT}/${RUN}marinebmat_${cyc}} +# source config.base, config.ocnanal and config.ocnanalbmat +# and pass marinebmat to ${machine}.env +source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" + +############################################## +# Set variables used in the script +############################################## +# shellcheck disable=SC2153 +GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} +GDUMP="gdas" + + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +# TODO: This is temporary, the plan is to prepare the bmatrix at the end of the +# cycle, so the backgrounds should be from the current cycle, not the previous +RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL + +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMOUT_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL + +mkdir -p "${COMOUT_OCEAN_BMATRIX}" +mkdir -p "${COMOUT_ICE_BMATRIX}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASMARINEBMATRUNPY:-${SCRgfs}/exglobal_marine_bmat_run.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/marinebmat.sh b/jobs/rocoto/marinebmat.sh new file mode 100755 index 0000000000..2ab0c9976c --- /dev/null +++ b/jobs/rocoto/marinebmat.sh @@ -0,0 +1,26 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ "${status}" -ne 0 ]] && exit "${status}" + +export job="marinebmat" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +socaToolsPATH="${HOMEgfs}/sorc/gdas.cd/sorc/soca/tools" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}"/jobs/JGLOBAL_MARINE_BMAT +echo "BMAT gets run here" +status=$? +exit "${status}" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index ec867e64ba..222ffdae95 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -82,11 +82,13 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model_data/ocean/history' declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model_data/ocean/restart' declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model_data/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' +declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model_data/ice/input' declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model_data/ice/history' declare -rx COM_ICE_RESTART_TMPL=${COM_BASE}'/model_data/ice/restart' diff --git a/parm/config/gfs/config.marinebmat b/parm/config/gfs/config.marinebmat new file mode 100644 index 0000000000..d88739dced --- /dev/null +++ b/parm/config/gfs/config.marinebmat @@ -0,0 +1,11 @@ +#!/bin/bash + +########## config.marinebmat ########## +# configuration for the marine B-matrix + +echo "BEGIN: config.marinebmat" + +# Get task specific resources +. "${EXPDIR}/config.resources" marinebmat + +echo "END: config.marinebmat" diff --git a/parm/config/gfs/config.ocnanal b/parm/config/gfs/config.ocnanal index 38a6cbd52a..8ef8cb5767 100644 --- a/parm/config/gfs/config.ocnanal +++ b/parm/config/gfs/config.ocnanal @@ -5,6 +5,7 @@ echo "BEGIN: config.ocnanal" +export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/soca_fix_stage.yaml.j2" export OBS_YAML_DIR="${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config" export OBS_LIST=@SOCA_OBS_LIST@ export OBS_YAML="${OBS_LIST}" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 2e910d4eb4..758e134a88 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -483,7 +483,7 @@ case ${step} in export memory_prepoceanobs="48GB" ;; - "ocnanalbmat") + "ocnanalbmat" | "marinebmat") npes=16 case ${OCNRES} in "025") npes=480;; diff --git a/scripts/exglobal_marine_bmat_run.py b/scripts/exglobal_marine_bmat_run.py new file mode 100755 index 0000000000..8a255cfbd9 --- /dev/null +++ b/scripts/exglobal_marine_bmat_run.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# exglobal_marine_bmat_run.py +# This script creates an marineBmat object +# and runs the execute method +# which executes all the steps necessary to create the global marine B-matrix +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_bmat import MarineBMat + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Create an instance of the MarineBMat task + marineBMat = MarineBMat(config) + marineBMat.execute() + + diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 4973ab8d7d..8f70a9915b 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -78,7 +78,7 @@ case "${machine}" in "gaea") FIX_DIR="/gpfs/f5/epic/proj-shared/global/glopara/data/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" - exit 1 + #exit 0 ;; esac @@ -368,6 +368,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ + "gdas_soca_diagb.x" \ "fv3jedi_plot_field.x" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py new file mode 100644 index 0000000000..4433c2dfde --- /dev/null +++ b/ush/python/pygfs/task/marine_bmat.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python3 + +import f90nml +import os +import glob +import gzip +import tarfile +from logging import getLogger +from typing import Dict, List, Any +import xarray as xr +import calc_scales +import subprocess + +from wxflow import (AttrDict, + FileHandler, + add_to_datetime, to_timedelta, to_YMDH, + chdir, + parse_j2yaml, save_as_yaml, + logit, + Executable, + WorkflowException, + Task) + +logger = getLogger(__name__.split('.')[-1]) + + +class MarineBMat(Task): + """ + Class for global marine B-matrix tasks + """ + @logit(logger, name="MarineBMat") + def __init__(self, config): + super().__init__(config) + + _res_anl = self.task_config.OCNRES + _gcyc_str = str(self.task_config.gcyc).zfill(2) + _cyc_str = str(self.task_config.cyc).zfill(2) + _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'gcyc_str': _gcyc_str, + 'cyc_str': _cyc_str, + 'HOMEgdas': _home_gdas, + 'MARINE_WINDOW_BEGIN': _window_begin, + 'MARINE_WINDOW_END': _window_end, + 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, + 'BERROR_YAML_DIR': os.path.join(_home_gdas, 'parm', 'soca', 'berror'), + 'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml') + } + ) + + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) + + + @logit(logger) + def _run(self, exec_cmd): + """Run the executable command + TODO: Move this method somewhere else + """ + logger.info(f"Executing {exec_cmd}") + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + + @logit(logger) + def _link_executable(self, exe_name: str) -> None: + """Link the executable to the DATA directory + TODO: Move this method somewhere else + """ + logger.info(f"Link executable {exe_name}") + logger.warn("Linking is not permitted per EE2.") + exe_src = os.path.join(self.task_config.EXECgfs, exe_name) + exe_dest = os.path.join(self.task_config.DATA, exe_name) + if os.path.exists(exe_dest): + os.remove(exe_dest) + os.symlink(exe_src, exe_dest) + + + @logit(logger) + def _prep_input_nml(self) -> None: + """Prepare the input.nml file + TODO: Move this method somewhere else + """ + # stage input.nml + mom_input_nml_tmpl_src = os.path.join(self.task_config.HOMEgdas, 'parm', 'soca', 'fms', 'input.nml') + mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') + FileHandler({'copy': [[mom_input_nml_tmpl_src, mom_input_nml_tmpl]]}).sync() + + # swap date and stacksize + domain_stack_size = self.task_config.DOMAIN_STACK_SIZE + ymdhms = [int(s) for s in self.task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] + with open(mom_input_nml_tmpl, 'r') as nml_file: + nml = f90nml.read(nml_file) + nml['ocean_solo_nml']['date_init'] = ymdhms + nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) + nml.write('mom_input.nml') + + + @logit(logger) + def _cice_hist2fms(self, input_filename, output_filename) -> None: + """ Reformat the CICE history file to be read by SOCA/FMS + Simple reformatting utility to allow soca/fms to read the CICE history files + """ + + # open the CICE history file + ds = xr.open_dataset(input_filename) + + if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables: + logger.info(f"*** Already reformatted, skipping.") + return + + # rename the dimensions to xaxis_1 and yaxis_1 + ds = ds.rename({'ni': 'xaxis_1', 'nj': 'yaxis_1'}) + + # rename the variables + ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'}) + + # Save the new netCDF file + ds.to_netcdf(output_filename, mode='w') + + + @logit(logger) + def initialize(self: Task) -> None: + """Initialize a global B-matrix + + This method will initialize a global B-Matrix. + This includes: + - staging the deterministic backgrounds (middle of window) + - staging SOCA fix files + - staging static ensemble members (optional) + - staging ensemble members (optional) + - generating the YAML files for the JEDI and GDASApp executables + - creating output directories + """ + super().initialize() + + # stage fix files + logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") + newdirs = [ os.path.join(self.task_config.DATA, 'INPUT') ] + FileHandler({'mkdir': newdirs}).sync() + soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) + FileHandler(soca_fix_list).sync() + + # prepare the MOM6 input.nml + self._prep_input_nml() + + # stage a single background + # TODO: check if we only need 1 deterministic background + # TODO: when we decide to move this task to the end of the previous cycle, + # the bkg will have to come from HISTORY not HISTORY_PREV + ocn_bkg_dir = self.task_config.COMIN_OCEAN_HISTORY_PREV + ice_bkg_dir = self.task_config.COMIN_ICE_HISTORY_PREV + logger.info("Staging background files from {ocn_bkg_dir} and {ice_bkg_dir}") + bkg_list = [] + bkg_list.append([os.path.join(ocn_bkg_dir, f"gdas.ocean.t{self.task_config.gcyc_str}z.inst.f009.nc"), + os.path.join(self.task_config.DATA, "ocn.bkg.nc")]) + bkg_list.append([os.path.join(ice_bkg_dir, f"gdas.ice.t{self.task_config.gcyc_str}z.inst.f009.nc"), + os.path.join(self.task_config.DATA, "ice.bkg.nc")]) + FileHandler({'copy': bkg_list}).sync() + self._cice_hist2fms("ice.bkg.nc", "ice.bkg.nc") + + # Copy MOM6 restart + # TODO: check if we can combine this with the step above + logger.info(f"Linking MOM6 restart to ocn.bkg.nc") + rst_list = [] + rst_list.append([os.path.join(self.task_config.DATA, "ocn.bkg.nc"), + os.path.join(self.task_config.DATA, "INPUT", "MOM.res.nc")]) + rst_list.append([os.path.join(self.task_config.DATA, "ice.bkg.nc"), + os.path.join(self.task_config.DATA, "INPUT", "cice.res.nc")]) + FileHandler({'copy': rst_list}).sync() + + # stage the grid generation yaml + FileHandler({'copy': [[self.task_config.GRID_GEN_YAML, + os.path.join(self.task_config.DATA, 'gridgen.yaml')]]}).sync() + + # generate the variance partitioning YAML file + logger.debug("Generate variance partitioning YAML file") + diagb_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_diagb.yaml.j2'), + data=self.task_config) + diagb_config.save(os.path.join(self.task_config.DATA, 'soca_diagb.yaml')) + + # generate the vertical decorrelation scale YAML file + logger.debug("Generate the vertical correlation scale YAML file") + vtscales_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_vtscales.yaml.j2'), + data=self.task_config) + vtscales_config.save(os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) + + # generate vertical diffusion scale YAML file + logger.debug("Generate vertical diffusion YAML file") + diffvz_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_parameters_diffusion_vt.yaml.j2'), + data=self.task_config) + diffvz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_vt.yaml')) + + + # generate the horizontal diffusion YAML files + if True: #task_config.COMPUTE_HORIZ_DIFF: + # stage the correlation scale configuration + logger.debug("Generate correlation scale YAML file") + FileHandler({'copy': [[os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_setcorscales.yaml'), + os.path.join(self.task_config.DATA, 'soca_setcorscales.yaml')]]}).sync() + + # generate horizontal diffusion scale YAML file + logger.debug("Generate horizontal diffusion scale YAML file") + diffhz_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_parameters_diffusion_hz.yaml.j2'), + data=self.task_config) + diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) + + # hybrid EnVAR case + if True: #self.task_config.DOHYBVAR: + # stage ensemble membersfiles for use in hybrid background error + logger.debug("Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") + + # generate ensemble recentering YAML file + logger.debug("Generate ensemble recentering YAML file: {self.task_config.abcd_yaml}") + + # generate hybrid weights YAML file + logger.debug("Generate ensemble recentering YAML file: {self.task_config.abcd_yaml}") + + # need output dir for ensemble perturbations and static B-matrix + logger.debug("Create empty output [ensb, diagb] directories to receive output from executables") + newdirs = [ + os.path.join(self.task_config.DATA, 'ensb'), + os.path.join(self.task_config.DATA, 'diagb'), + ] + FileHandler({'mkdir': newdirs}).sync() + + + @logit(logger) + def gridgen(self: Task) -> None: + + # link gdas_soca_gridgen.x + self._link_executable('gdas_soca_gridgen.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_gridgen.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('gridgen.yaml') + + self._run(exec_cmd) + + @logit(logger) + def variance_partitioning(self: Task) -> None: + + # link the variance partitioning executable, gdas_soca_diagb.x + self._link_executable('gdas_soca_diagb.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_diagb.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_diagb.yaml') + + self._run(exec_cmd) + + @logit(logger) + def horizontal_diffusion(self: Task) -> None: + """Generate the horizontal diffusion coefficients + """ + # link the executable that computes the correlation scales, gdas_soca_setcorscales.x, + # and prepare the command to run it + self._link_executable('gdas_soca_setcorscales.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_setcorscales.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_setcorscales.yaml') + + # create a files containing the correlation scales + self._run(exec_cmd) + + # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, + # and prepare the command to run it + self._link_executable('gdas_soca_error_covariance_toolbox.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_parameters_diffusion_hz.yaml') + + # compute the coefficients of the diffusion operator + self._run(exec_cmd) + + @logit(logger) + def vertical_diffusion(self: Task) -> None: + """Generate the vertical diffusion coefficients + """ + # compute the vertical correlation scales based on the MLD + calc_scales.run('soca_vtscales.yaml') + + # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, + # and prepare the command to run it + self._link_executable('gdas_soca_error_covariance_toolbox.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_parameters_diffusion_vt.yaml') + + # compute the coefficients of the diffusion operator + self._run(exec_cmd) + + + @logit(logger) + def ensemble_perturbations(self: Task) -> None: + """Generate the 3D ensemble of perturbation for the 3DEnVAR + + This method will generate ensemble perturbations re-balanced w.r.t the + deterministic background. + This includes: + - computing a storing the unbalanced ensemble perturbations' statistics + - recentering the ensemble members around the deterministic background and + accounting for the nonlinear steric recentering + - saving the recentered ensemble statistics + """ + # gdas_ens_handler.x + + @logit(logger) + def hybrid_weight(self: Task) -> None: + """Generate the hybrid weights for the 3DEnVAR + + This method will generate the 3D fields hybrid weights for the 3DEnVAR for each + variables. + TODO(G): Currently implemented for the specific case of the static ensemble members only + """ + # gdas_socahybridweights.x + + @logit(logger) + def execute(self: Task) -> None: + """Generate the full B-matrix + + This method will generate the full B-matrix according to the configuration. + """ + chdir(self.task_config.DATA) + self.initialize() + self.gridgen() # TODO: This should be optional in case the geometry file was staged + self.variance_partitioning() + self.horizontal_diffusion() # TODO: Make this optional once we've converged on an acceptable set of scales + self.vertical_diffusion() + self.ensemble_perturbations() # TODO: refactor this from the old scripts + self.hybrid_weight() # TODO: refactor this from the old scripts + self.finalize() + + @logit(logger) + def finalize(self: Task) -> None: + """Finalize the global B-matrix job + + This method will finalize the global B-matrix job. + This includes: + - copy the generated static, but cycle dependent background error files to the ROTDIR + - copy the generated YAML file from initialize to the ROTDIR + - keep the re-balanced ensemble perturbation files in the DATA/??? directory + - ... + + """ + # Copy the soca grid if it was created + grid_file = os.path.join(self.task_config.DATA, 'soca_gridspec.nc') + if os.path.exists(grid_file): + logger.info(f"Copying the soca grid file to the ROTDIR") + FileHandler({'copy': [[grid_file, + os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, 'soca_gridspec.nc')]]}).sync() + + # Copy the diffusion coefficient files to the ROTDIR + logger.info(f"Copying the diffusion coefficient files to the ROTDIR") + diffusion_coeff_list = [] + for diff_type in ['hz', 'vt']: + src = os.path.join(self.task_config.DATA, f"{diff_type}_ocean.nc") + dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{diff_type}_ocean.nc") + diffusion_coeff_list.append([src, dest]) + + src = os.path.join(self.task_config.DATA, f"hz_ice.nc") + dest = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.hz_ice.nc") + diffusion_coeff_list.append([src, dest]) + + FileHandler({'copy': diffusion_coeff_list}).sync() + + # Copy diag B files to ROTDIR + logger.info(f"Copying diag B files to the ROTDIR") + diagb_list = [] + window_end_iso = self.task_config.MARINE_WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ') + + # ocean diag B + src = os.path.join(self.task_config.DATA, 'diagb', f"ocn.bkgerr_stddev.incr.{window_end_iso}.nc") + dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ocean.bkgerr_stddev.nc") + diagb_list.append([src, dst]) + + # ice diag B + src = os.path.join(self.task_config.DATA, 'diagb', f"ice.bkgerr_stddev.incr.{window_end_iso}.nc") + dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ice.bkgerr_stddev.nc") + diagb_list.append([src, dst]) + + FileHandler({'copy': diagb_list}).sync() + + # Copy the YAML files to the OCEAN ROTDIR + yamls = glob.glob(os.path.join(self.task_config.DATA, '*.yaml')) + yaml_list = [] + for yaml_file in yamls: + dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}.{os.path.basename(yaml_file)}") + yaml_list.append([yaml_file, dest]) + FileHandler({'copy': yaml_list}).sync() From 5438a47e28a4f6eac455ee223308239b8cabce6e Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 1 Jul 2024 19:58:43 -0400 Subject: [PATCH 02/26] pynorms --- ush/python/pygfs/task/marine_bmat.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 4433c2dfde..72c8d693c0 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -57,14 +57,12 @@ def __init__(self, config): # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - @logit(logger) def _run(self, exec_cmd): """Run the executable command TODO: Move this method somewhere else """ logger.info(f"Executing {exec_cmd}") - try: logger.debug(f"Executing {exec_cmd}") exec_cmd() @@ -75,7 +73,6 @@ def _run(self, exec_cmd): pass - @logit(logger) def _link_executable(self, exe_name: str) -> None: """Link the executable to the DATA directory @@ -89,7 +86,6 @@ def _link_executable(self, exe_name: str) -> None: os.remove(exe_dest) os.symlink(exe_src, exe_dest) - @logit(logger) def _prep_input_nml(self) -> None: """Prepare the input.nml file @@ -109,7 +105,6 @@ def _prep_input_nml(self) -> None: nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) nml.write('mom_input.nml') - @logit(logger) def _cice_hist2fms(self, input_filename, output_filename) -> None: """ Reformat the CICE history file to be read by SOCA/FMS @@ -132,7 +127,6 @@ def _cice_hist2fms(self, input_filename, output_filename) -> None: # Save the new netCDF file ds.to_netcdf(output_filename, mode='w') - @logit(logger) def initialize(self: Task) -> None: """Initialize a global B-matrix @@ -150,7 +144,7 @@ def initialize(self: Task) -> None: # stage fix files logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") - newdirs = [ os.path.join(self.task_config.DATA, 'INPUT') ] + newdirs = [os.path.join(self.task_config.DATA, 'INPUT')] FileHandler({'mkdir': newdirs}).sync() soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_list).sync() @@ -205,9 +199,8 @@ def initialize(self: Task) -> None: data=self.task_config) diffvz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_vt.yaml')) - # generate the horizontal diffusion YAML files - if True: #task_config.COMPUTE_HORIZ_DIFF: + if True: # TODO(G): Missing logic to skip this section # stage the correlation scale configuration logger.debug("Generate correlation scale YAML file") FileHandler({'copy': [[os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_setcorscales.yaml'), @@ -220,7 +213,9 @@ def initialize(self: Task) -> None: diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) # hybrid EnVAR case - if True: #self.task_config.DOHYBVAR: + if True: + # TODO(G): copy logic/steps from old script in the block below + # ensemble DA is temporarily off, check self.task_config.DOHYBVAR. # stage ensemble membersfiles for use in hybrid background error logger.debug("Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") @@ -238,10 +233,8 @@ def initialize(self: Task) -> None: ] FileHandler({'mkdir': newdirs}).sync() - @logit(logger) def gridgen(self: Task) -> None: - # link gdas_soca_gridgen.x self._link_executable('gdas_soca_gridgen.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) @@ -253,7 +246,6 @@ def gridgen(self: Task) -> None: @logit(logger) def variance_partitioning(self: Task) -> None: - # link the variance partitioning executable, gdas_soca_diagb.x self._link_executable('gdas_soca_diagb.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) @@ -307,7 +299,6 @@ def vertical_diffusion(self: Task) -> None: # compute the coefficients of the diffusion operator self._run(exec_cmd) - @logit(logger) def ensemble_perturbations(self: Task) -> None: """Generate the 3D ensemble of perturbation for the 3DEnVAR From ee36b5e5da0a1ca14d580ed12eb221a8cf0ca9b9 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 2 Jul 2024 08:31:26 -0400 Subject: [PATCH 03/26] point to different soca tools --- jobs/rocoto/marinebmat.sh | 4 ++-- parm/config/gfs/config.ocnanal | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/jobs/rocoto/marinebmat.sh b/jobs/rocoto/marinebmat.sh index 2ab0c9976c..dee5f8059a 100755 --- a/jobs/rocoto/marinebmat.sh +++ b/jobs/rocoto/marinebmat.sh @@ -14,8 +14,8 @@ export jobid="${job}.$$" ############################################################### # setup python path for workflow utilities and tasks wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -socaToolsPATH="${HOMEgfs}/sorc/gdas.cd/sorc/soca/tools" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +socaToolsPATH="${HOMEgfs}/sorc/gdas.cd/ush/soca/tools" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${socaToolsPATH}" export PYTHONPATH ############################################################### diff --git a/parm/config/gfs/config.ocnanal b/parm/config/gfs/config.ocnanal index 8ef8cb5767..9db9375f63 100644 --- a/parm/config/gfs/config.ocnanal +++ b/parm/config/gfs/config.ocnanal @@ -5,23 +5,23 @@ echo "BEGIN: config.ocnanal" -export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/soca_fix_stage.yaml.j2" +export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage.yaml.j2" export OBS_YAML_DIR="${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config" export OBS_LIST=@SOCA_OBS_LIST@ export OBS_YAML="${OBS_LIST}" export FV3JEDI_STAGE_YAML="${HOMEgfs}/sorc/gdas.cd/test/soca/testinput/dumy.yaml" export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ -export SOCA_VARS=tocn,socn,ssh -export SABER_BLOCKS_YAML=@SABER_BLOCKS_YAML@ +#export SOCA_VARS=tocn,socn,ssh +#export SABER_BLOCKS_YAML=@SABER_BLOCKS_YAML@ export SOCA_NINNER=@SOCA_NINNER@ -export CASE_ANL=@CASE_ANL@ +#export CASE_ANL=@CASE_ANL@ export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin export COMIN_OBS=@COMIN_OBS@ # NICAS -export NICAS_RESOL=@NICAS_RESOL@ -export NICAS_GRID_SIZE=@NICAS_GRID_SIZE@ +#export NICAS_RESOL=@NICAS_RESOL@ +#export NICAS_GRID_SIZE=@NICAS_GRID_SIZE@ echo "END: config.ocnanal" From 8ae7f79165482ee33aa9fb4e1213b09de4624a54 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 3 Jul 2024 16:09:54 -0400 Subject: [PATCH 04/26] added perturbations for envar --- jobs/JGLOBAL_MARINE_BMAT | 15 ++- scripts/exglobal_marine_bmat_run.py | 4 +- ush/python/pygfs/__init__.py | 2 + ush/python/pygfs/task/marine_bmat.py | 154 +++++++--------------- ush/python/pygfs/utils/__init__.py | 0 ush/python/pygfs/utils/marine_da_utils.py | 100 ++++++++++++++ 6 files changed, 159 insertions(+), 116 deletions(-) create mode 100644 ush/python/pygfs/utils/__init__.py create mode 100644 ush/python/pygfs/utils/marine_da_utils.py diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 1710eb987e..dbad2dc8d6 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -10,10 +10,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebm # Set variables used in the script ############################################## # shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" +export GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +export gPDY=${GDATE:0:8} +export gcyc=${GDATE:8:2} +export GDUMP="gdas" +export GDUMP_ENS="enkf${GDUMP}" ############################################## @@ -21,12 +22,14 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -# TODO: This is temporary, the plan is to prepare the bmatrix at the end of the -# cycle, so the backgrounds should be from the current cycle, not the previous RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL +RUN="enkfgdas" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COMIN_OCEAN_HISTORY_ENS_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_ENS_PREV:COM_ICE_HISTORY_TMPL + YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ COMOUT_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL diff --git a/scripts/exglobal_marine_bmat_run.py b/scripts/exglobal_marine_bmat_run.py index 8a255cfbd9..e285e646ac 100755 --- a/scripts/exglobal_marine_bmat_run.py +++ b/scripts/exglobal_marine_bmat_run.py @@ -19,6 +19,6 @@ # Create an instance of the MarineBMat task marineBMat = MarineBMat(config) + marineBMat.initialize() marineBMat.execute() - - + marineBMat.finalize() diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index fa6b0b373e..b1208dfb3d 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -6,10 +6,12 @@ from .task.aero_analysis import AerosolAnalysis from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis +from .task.marine_bmat import MarineBMat from .task.snow_analysis import SnowAnalysis from .task.upp import UPP from .task.oceanice_products import OceanIceProducts from .task.gfs_forecast import GFSForecast +from .utils import marine_da_utils __docformat__ = "restructuredtext" __version__ = "0.1.0" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 72c8d693c0..ad4cbcec4b 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -1,24 +1,19 @@ #!/usr/bin/env python3 -import f90nml import os import glob -import gzip -import tarfile from logging import getLogger -from typing import Dict, List, Any -import xarray as xr import calc_scales -import subprocess +import pygfs.utils.marine_da_utils as mdau from wxflow import (AttrDict, FileHandler, - add_to_datetime, to_timedelta, to_YMDH, + add_to_datetime, to_timedelta, chdir, - parse_j2yaml, save_as_yaml, + parse_j2yaml, logit, Executable, - WorkflowException, + jinja, Task) logger = getLogger(__name__.split('.')[-1]) @@ -31,8 +26,7 @@ class MarineBMat(Task): @logit(logger, name="MarineBMat") def __init__(self, config): super().__init__(config) - - _res_anl = self.task_config.OCNRES + _gPDY = self.task_config.gPDY _gcyc_str = str(self.task_config.gcyc).zfill(2) _cyc_str = str(self.task_config.cyc).zfill(2) _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') @@ -50,83 +44,15 @@ def __init__(self, config): 'MARINE_WINDOW_END': _window_end, 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, 'BERROR_YAML_DIR': os.path.join(_home_gdas, 'parm', 'soca', 'berror'), - 'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml') + 'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml'), + 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca','ensda', 'stage_ens_mem.yaml.j2'), + 'ens_dir': os.path.join(self.task_config.DATA, 'ens') } ) # Extend task_config with local_dict self.task_config = AttrDict(**self.task_config, **local_dict) - @logit(logger) - def _run(self, exec_cmd): - """Run the executable command - TODO: Move this method somewhere else - """ - logger.info(f"Executing {exec_cmd}") - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - - @logit(logger) - def _link_executable(self, exe_name: str) -> None: - """Link the executable to the DATA directory - TODO: Move this method somewhere else - """ - logger.info(f"Link executable {exe_name}") - logger.warn("Linking is not permitted per EE2.") - exe_src = os.path.join(self.task_config.EXECgfs, exe_name) - exe_dest = os.path.join(self.task_config.DATA, exe_name) - if os.path.exists(exe_dest): - os.remove(exe_dest) - os.symlink(exe_src, exe_dest) - - @logit(logger) - def _prep_input_nml(self) -> None: - """Prepare the input.nml file - TODO: Move this method somewhere else - """ - # stage input.nml - mom_input_nml_tmpl_src = os.path.join(self.task_config.HOMEgdas, 'parm', 'soca', 'fms', 'input.nml') - mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') - FileHandler({'copy': [[mom_input_nml_tmpl_src, mom_input_nml_tmpl]]}).sync() - - # swap date and stacksize - domain_stack_size = self.task_config.DOMAIN_STACK_SIZE - ymdhms = [int(s) for s in self.task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] - with open(mom_input_nml_tmpl, 'r') as nml_file: - nml = f90nml.read(nml_file) - nml['ocean_solo_nml']['date_init'] = ymdhms - nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) - nml.write('mom_input.nml') - - @logit(logger) - def _cice_hist2fms(self, input_filename, output_filename) -> None: - """ Reformat the CICE history file to be read by SOCA/FMS - Simple reformatting utility to allow soca/fms to read the CICE history files - """ - - # open the CICE history file - ds = xr.open_dataset(input_filename) - - if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables: - logger.info(f"*** Already reformatted, skipping.") - return - - # rename the dimensions to xaxis_1 and yaxis_1 - ds = ds.rename({'ni': 'xaxis_1', 'nj': 'yaxis_1'}) - - # rename the variables - ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'}) - - # Save the new netCDF file - ds.to_netcdf(output_filename, mode='w') - @logit(logger) def initialize(self: Task) -> None: """Initialize a global B-matrix @@ -150,7 +76,7 @@ def initialize(self: Task) -> None: FileHandler(soca_fix_list).sync() # prepare the MOM6 input.nml - self._prep_input_nml() + mdau.prep_input_nml(self.task_config) # stage a single background # TODO: check if we only need 1 deterministic background @@ -165,7 +91,7 @@ def initialize(self: Task) -> None: bkg_list.append([os.path.join(ice_bkg_dir, f"gdas.ice.t{self.task_config.gcyc_str}z.inst.f009.nc"), os.path.join(self.task_config.DATA, "ice.bkg.nc")]) FileHandler({'copy': bkg_list}).sync() - self._cice_hist2fms("ice.bkg.nc", "ice.bkg.nc") + mdau.cice_hist2fms("ice.bkg.nc", "ice.bkg.nc") # Copy MOM6 restart # TODO: check if we can combine this with the step above @@ -213,47 +139,49 @@ def initialize(self: Task) -> None: diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) # hybrid EnVAR case - if True: - # TODO(G): copy logic/steps from old script in the block below - # ensemble DA is temporarily off, check self.task_config.DOHYBVAR. + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: # stage ensemble membersfiles for use in hybrid background error - logger.debug("Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") + logger.debug(f"Stage ensemble members for the hybrid background error") + mdau.stage_ens_mem(self.task_config) # generate ensemble recentering YAML file - logger.debug("Generate ensemble recentering YAML file: {self.task_config.abcd_yaml}") + logger.debug("Generate ensemble recentering YAML file") + ensrecenter_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_ensrecenter.yaml.j2'), + data=self.task_config) + ensrecenter_config.save(os.path.join(self.task_config.DATA, 'soca_ensrecenter.yaml')) - # generate hybrid weights YAML file + # generate ensemble weights YAML file logger.debug("Generate ensemble recentering YAML file: {self.task_config.abcd_yaml}") + hybridweights_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_ensweights.yaml.j2'), + data=self.task_config) + hybridweights_config.save(os.path.join(self.task_config.DATA, 'soca_ensweights.yaml')) # need output dir for ensemble perturbations and static B-matrix logger.debug("Create empty output [ensb, diagb] directories to receive output from executables") - newdirs = [ - os.path.join(self.task_config.DATA, 'ensb'), - os.path.join(self.task_config.DATA, 'diagb'), - ] + newdirs = [os.path.join(self.task_config.DATA, 'diagb')] FileHandler({'mkdir': newdirs}).sync() @logit(logger) def gridgen(self: Task) -> None: # link gdas_soca_gridgen.x - self._link_executable('gdas_soca_gridgen.x') + mdau.link_executable(self.task_config, 'gdas_soca_gridgen.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_gridgen.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('gridgen.yaml') - self._run(exec_cmd) + mdau.run(exec_cmd) @logit(logger) def variance_partitioning(self: Task) -> None: # link the variance partitioning executable, gdas_soca_diagb.x - self._link_executable('gdas_soca_diagb.x') + mdau.link_executable(self.task_config, 'gdas_soca_diagb.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_diagb.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_diagb.yaml') - self._run(exec_cmd) + mdau.run(exec_cmd) @logit(logger) def horizontal_diffusion(self: Task) -> None: @@ -261,25 +189,25 @@ def horizontal_diffusion(self: Task) -> None: """ # link the executable that computes the correlation scales, gdas_soca_setcorscales.x, # and prepare the command to run it - self._link_executable('gdas_soca_setcorscales.x') + mdau.link_executable(self.task_config, 'gdas_soca_setcorscales.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_setcorscales.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_setcorscales.yaml') # create a files containing the correlation scales - self._run(exec_cmd) + mdau.run(exec_cmd) # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, # and prepare the command to run it - self._link_executable('gdas_soca_error_covariance_toolbox.x') + mdau.link_executable(self.task_config, 'gdas_soca_error_covariance_toolbox.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_parameters_diffusion_hz.yaml') # compute the coefficients of the diffusion operator - self._run(exec_cmd) + mdau.run(exec_cmd) @logit(logger) def vertical_diffusion(self: Task) -> None: @@ -290,14 +218,14 @@ def vertical_diffusion(self: Task) -> None: # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, # and prepare the command to run it - self._link_executable('gdas_soca_error_covariance_toolbox.x') + mdau.link_executable(self.task_config, 'gdas_soca_error_covariance_toolbox.x') exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_soca_error_covariance_toolbox.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_parameters_diffusion_vt.yaml') # compute the coefficients of the diffusion operator - self._run(exec_cmd) + mdau.run(exec_cmd) @logit(logger) def ensemble_perturbations(self: Task) -> None: @@ -311,7 +239,14 @@ def ensemble_perturbations(self: Task) -> None: accounting for the nonlinear steric recentering - saving the recentered ensemble statistics """ - # gdas_ens_handler.x + mdau.link_executable(self.task_config, 'gdas_ens_handler.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_ens_handler.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_ensrecenter.yaml') + + # compute the coefficients of the diffusion operator + mdau.run(exec_cmd) @logit(logger) def hybrid_weight(self: Task) -> None: @@ -321,7 +256,12 @@ def hybrid_weight(self: Task) -> None: variables. TODO(G): Currently implemented for the specific case of the static ensemble members only """ - # gdas_socahybridweights.x + mdau.link_executable(self.task_config, 'gdas_socahybridweights.x') + exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) + exec_name = os.path.join(self.task_config.DATA, 'gdas_socahybridweights.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_hybridweights.yaml') + @logit(logger) def execute(self: Task) -> None: @@ -330,14 +270,12 @@ def execute(self: Task) -> None: This method will generate the full B-matrix according to the configuration. """ chdir(self.task_config.DATA) - self.initialize() self.gridgen() # TODO: This should be optional in case the geometry file was staged self.variance_partitioning() self.horizontal_diffusion() # TODO: Make this optional once we've converged on an acceptable set of scales self.vertical_diffusion() self.ensemble_perturbations() # TODO: refactor this from the old scripts self.hybrid_weight() # TODO: refactor this from the old scripts - self.finalize() @logit(logger) def finalize(self: Task) -> None: diff --git a/ush/python/pygfs/utils/__init__.py b/ush/python/pygfs/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py new file mode 100644 index 0000000000..c6d4cf5be1 --- /dev/null +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -0,0 +1,100 @@ +import f90nml +import os +from logging import getLogger +import xarray as xr + +from wxflow import FileHandler, logit, WorkflowException, AttrDict, parse_j2yaml, jinja + +logger = getLogger(__name__.split('.')[-1]) + +@logit(logger) +def run(exec_cmd): + """Run the executable command + """ + logger.info(f"Executing {exec_cmd}") + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + +@logit(logger) +def link_executable(task_config, exe_name: str) -> None: + """Link the executable to the DATA directory + """ + logger.info(f"Link executable {exe_name}") + logger.warn("Linking is not permitted per EE2.") + exe_src = os.path.join(task_config.EXECgfs, exe_name) + exe_dest = os.path.join(task_config.DATA, exe_name) + if os.path.exists(exe_dest): + os.remove(exe_dest) + os.symlink(exe_src, exe_dest) + +@logit(logger) +def prep_input_nml(task_config) -> None: + """Prepare the input.nml file + TODO: Use jinja2 to template the input.nml file + """ + # stage input.nml + mom_input_nml_tmpl_src = os.path.join(task_config.HOMEgdas, 'parm', 'soca', 'fms', 'input.nml') + mom_input_nml_tmpl = os.path.join(task_config.DATA, 'mom_input.nml.tmpl') + FileHandler({'copy': [[mom_input_nml_tmpl_src, mom_input_nml_tmpl]]}).sync() + + # swap date and stacksize + domain_stack_size = task_config.DOMAIN_STACK_SIZE + ymdhms = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] + with open(mom_input_nml_tmpl, 'r') as nml_file: + nml = f90nml.read(nml_file) + nml['ocean_solo_nml']['date_init'] = ymdhms + nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) + nml.write('mom_input.nml') + +@logit(logger) +def cice_hist2fms(input_filename, output_filename) -> None: + """ Reformat the CICE history file so it can be read by SOCA/FMS + Simple reformatting utility to allow soca/fms to read the CICE history files + """ + + # open the CICE history file + ds = xr.open_dataset(input_filename) + + if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables: + logger.info(f"*** Already reformatted, skipping.") + return + + # rename the dimensions to xaxis_1 and yaxis_1 + ds = ds.rename({'ni': 'xaxis_1', 'nj': 'yaxis_1'}) + + # rename the variables + ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'}) + + # Save the new netCDF file + ds.to_netcdf(output_filename, mode='w') + +@logit(logger) +def stage_ens_mem(task_config) -> None: + """ Copy the ensemble members to the DATA directory + Copy the ensemble members to the DATA directory and reformat the CICE history files + """ + # Copy the ensemble members to the DATA directory +# ensbkgconf = AttrDict() +# keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS', +# 'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL'] +# for key in keys: +# ensbkgconf[key] = task_config[key] +# ensbkgconf.RUN = 'enkfgdas' +# letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) +# FileHandler(letkf_stage_list).sync() + + logger.info("---------------- Stage ensemble members") + # make directories and stage ensemble background files + ensbkgconf = AttrDict(task_config) + ensbkgconf.RUN = task_config.GDUMP_ENS + logger.debug(f"{jinja.Jinja(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf).render}") + letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) + logger.info(f"{letkf_stage_list}") + FileHandler(letkf_stage_list).sync() From ba665f4339392de19814aff0f065ef2dbe849c7e Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 5 Jul 2024 13:39:42 -0400 Subject: [PATCH 05/26] working hybvar --- jobs/JGLOBAL_MARINE_BMAT | 11 +++++- ush/python/pygfs/task/marine_bmat.py | 43 +++++++++++++++++------ ush/python/pygfs/utils/marine_da_utils.py | 2 +- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index dbad2dc8d6..781380dbb1 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -1,11 +1,21 @@ #!/bin/bash + source "${HOMEgfs}/ush/preamble.sh" + export DATA="${DATAROOT}/${RUN}marinebmat_${cyc}" export DATA=${DATA:-${DATAROOT}/${RUN}marinebmat_${cyc}} # source config.base, config.ocnanal and config.ocnanalbmat # and pass marinebmat to ${machine}.env source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" +# If ensemble perturbations are used, save them in a DATAenspert directory that will +# exist throughout the DA cycle +if (( 10#${NMEM_ENS:-0} > 0 )); then + export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}" + export DATAenspert="${DATAjob}/enspert" + if [[ ! -d "${DATAenspert}" ]]; then mkdir -p "${DATAenspert}"; fi +fi + ############################################## # Set variables used in the script ############################################## @@ -16,7 +26,6 @@ export gcyc=${GDATE:8:2} export GDUMP="gdas" export GDUMP_ENS="enkf${GDUMP}" - ############################################## # Begin JOB SPECIFIC work ############################################## diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index ad4cbcec4b..08309e016b 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -33,6 +33,8 @@ def __init__(self, config): _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA) # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -46,7 +48,7 @@ def __init__(self, config): 'BERROR_YAML_DIR': os.path.join(_home_gdas, 'parm', 'soca', 'berror'), 'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml'), 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca','ensda', 'stage_ens_mem.yaml.j2'), - 'ens_dir': os.path.join(self.task_config.DATA, 'ens') + 'ENSPERT_RELPATH': _enspert_relpath } ) @@ -144,11 +146,11 @@ def initialize(self: Task) -> None: logger.debug(f"Stage ensemble members for the hybrid background error") mdau.stage_ens_mem(self.task_config) - # generate ensemble recentering YAML file + # generate ensemble recentering/rebalancing YAML file logger.debug("Generate ensemble recentering YAML file") - ensrecenter_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_ensrecenter.yaml.j2'), + ensrecenter_config = parse_j2yaml(path=os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_ensb.yaml.j2'), data=self.task_config) - ensrecenter_config.save(os.path.join(self.task_config.DATA, 'soca_ensrecenter.yaml')) + ensrecenter_config.save(os.path.join(self.task_config.DATA, 'soca_ensb.yaml')) # generate ensemble weights YAML file logger.debug("Generate ensemble recentering YAML file: {self.task_config.abcd_yaml}") @@ -243,9 +245,9 @@ def ensemble_perturbations(self: Task) -> None: exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_ens_handler.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_ensrecenter.yaml') + exec_cmd.add_default_arg('soca_ensb.yaml') - # compute the coefficients of the diffusion operator + # generate the ensemble perturbations mdau.run(exec_cmd) @logit(logger) @@ -260,7 +262,10 @@ def hybrid_weight(self: Task) -> None: exec_cmd = Executable(self.task_config.APRUN_MARINEBMAT) exec_name = os.path.join(self.task_config.DATA, 'gdas_socahybridweights.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('soca_hybridweights.yaml') + exec_cmd.add_default_arg('soca_ensweights.yaml') + + # compute the ensemble weights + mdau.run(exec_cmd) @logit(logger) @@ -274,8 +279,10 @@ def execute(self: Task) -> None: self.variance_partitioning() self.horizontal_diffusion() # TODO: Make this optional once we've converged on an acceptable set of scales self.vertical_diffusion() - self.ensemble_perturbations() # TODO: refactor this from the old scripts - self.hybrid_weight() # TODO: refactor this from the old scripts + # hybrid EnVAR case + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + self.ensemble_perturbations() # TODO: refactor this from the old scripts + self.hybrid_weight() # TODO: refactor this from the old scripts @logit(logger) def finalize(self: Task) -> None: @@ -285,7 +292,7 @@ def finalize(self: Task) -> None: This includes: - copy the generated static, but cycle dependent background error files to the ROTDIR - copy the generated YAML file from initialize to the ROTDIR - - keep the re-balanced ensemble perturbation files in the DATA/??? directory + - keep the re-balanced ensemble perturbation files in DATAenspert - ... """ @@ -331,6 +338,22 @@ def finalize(self: Task) -> None: FileHandler({'copy': diagb_list}).sync() + # Copy the ensemble perturbation diagnostics to the ROTDIR + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') + weight_list = [] + src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") + dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ocean.ens_weights.nc") + weight_list.append([src, dst]) + + src = os.path.join(self.task_config.DATA, f"ice.ens_weights.incr.{window_middle_iso}.nc") + dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, + f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ice.ens_weights.nc") + weight_list.append([src, dst]) + + FileHandler({'copy': weight_list}).sync() + # Copy the YAML files to the OCEAN ROTDIR yamls = glob.glob(os.path.join(self.task_config.DATA, '*.yaml')) yaml_list = [] diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index c6d4cf5be1..6107e0423c 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -37,7 +37,7 @@ def link_executable(task_config, exe_name: str) -> None: @logit(logger) def prep_input_nml(task_config) -> None: """Prepare the input.nml file - TODO: Use jinja2 to template the input.nml file + TODO: Use jinja2 instead of f90nml """ # stage input.nml mom_input_nml_tmpl_src = os.path.join(task_config.HOMEgdas, 'parm', 'soca', 'fms', 'input.nml') From 6900fc49fef26786caa955b1a3a458ca8f0a7916 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 9 Jul 2024 14:47:26 -0400 Subject: [PATCH 06/26] addressed some of the review comments --- env/HERA.env | 6 +-- env/HERCULES.env | 8 ++-- env/JET.env | 8 ++-- env/ORION.env | 8 ++-- env/S4.env | 6 +-- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT | 48 ---------------------- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY | 47 --------------------- jobs/JGLOBAL_MARINE_BMAT | 3 +- parm/config/gfs/config.ocnanalbmat | 11 ----- parm/config/gfs/config.resources | 12 +++--- ush/python/pygfs/__init__.py | 2 +- ush/python/pygfs/task/marine_bmat.py | 45 ++++++-------------- ush/python/pygfs/utils/marine_da_utils.py | 5 +++ workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 10 ++--- workflow/rocoto/tasks.py | 2 +- 16 files changed, 51 insertions(+), 174 deletions(-) delete mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT delete mode 100755 jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY delete mode 100644 parm/config/gfs/config.ocnanalbmat diff --git a/env/HERA.env b/env/HERA.env index 78e8f15843..aa3b7500a2 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -118,13 +118,13 @@ elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_MARINEBMAT="${launcher} -n ${npe_ocnanalbmat}" + export APRUN_MARINEBMAT="${launcher} -n ${npe_marinebmat}" -elif [[ "${step}" = "ocnanalbmat" ]]; then +elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd --multi-prog" - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}" + export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat}" elif [[ "${step}" = "ocnanalrun" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 79424f8639..3ce739cce9 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -112,15 +112,15 @@ case ${step} in export APRUN_APPLY_INCR="${launcher} -n 6" ;; - "ocnanalbmat") + "marinebmat") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_ocnanalbmat)) + nth_max=$((npe_node_max / npe_node_marinebmat)) - export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}} + export NTHREADS_OCNANAL=${nth_marinebmat:-${nth_max}} [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat} --cpus-per-task=${NTHREADS_OCNANAL}" ;; "ocnanalrun") diff --git a/env/JET.env b/env/JET.env index bb9826f331..956762a921 100755 --- a/env/JET.env +++ b/env/JET.env @@ -102,15 +102,15 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}" -elif [[ "${step}" = "ocnanalbmat" ]]; then +elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_ocnanalbmat)) + nth_max=$((npe_node_max / npe_node_marinebmat)) - export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}} + export NTHREADS_OCNANAL=${nth_marinebmat:-${nth_max}} [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}" + export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat}" elif [[ "${step}" = "ocnanalrun" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 502e99e192..2ec5a3b658 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -110,15 +110,15 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" -elif [[ "${step}" = "ocnanalbmat" ]]; then +elif [[ "${step}" = "marinebmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_ocnanalbmat)) + nth_max=$((npe_node_max / npe_node_marinebmat)) - export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}} + export NTHREADS_OCNANAL=${nth_marinebmat:-${nth_max}} [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat} --cpus-per-task=${NTHREADS_OCNANAL}" elif [[ "${step}" = "ocnanalrun" ]]; then diff --git a/env/S4.env b/env/S4.env index 190c7295f4..5e768d889d 100755 --- a/env/S4.env +++ b/env/S4.env @@ -65,7 +65,7 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${nth_atmensanlfv3inc:-${nth_max}} [[ ${NTHREADS_ATMENSANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMENSANLFV3INC=${nth_max} - export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc}" + export APRUN_ATMENSANLFV3INC="${launcher} -n ${npe_atmensanlfv3inc}" elif [[ "${step}" = "aeroanlrun" ]]; then @@ -102,10 +102,10 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}" -elif [[ "${step}" = "ocnanalbmat" ]]; then +elif [[ "${step}" = "marinebmat" ]]; then echo "WARNING: ${step} is not enabled on S4!" -elif [[ "${step}" = "ocnanalrun" ]]; then +elif [[ "${step}" = "marinerun" ]]; then echo "WARNING: ${step} is not enabled on S4!" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT deleted file mode 100755 index 90902ba3c3..0000000000 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -source "${HOMEgfs}/ush/preamble.sh" -export WIPE_DATA="NO" - -export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalbmat" -c "base ocnanal ocnanalbmat" - - -############################################## -# Set variables used in the script -############################################## - - -############################################## -# Begin JOB SPECIFIC work -############################################## - -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OCEAN_ANALYSIS - -mkdir -p "${COM_OCEAN_ANALYSIS}" - -export COMOUT=${COM_OCEAN_ANALYSIS} - -############################################################### -# Run relevant script - -EXSCRIPT=${GDASOCNBMATSH:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_bmat.sh} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" -fi - -########################################## -# Do not remove the Temporary working directory (do this in POST) -########################################## -cd "${DATAROOT}" || exit 1 - -exit 0 diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY deleted file mode 100755 index 3727ba9853..0000000000 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT_VRFY +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -source "${HOMEgfs}/ush/preamble.sh" -export WIPE_DATA="NO" -export DATA="${DATAROOT}/${RUN}ocnanal_${cyc}" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalrun" -c "base ocnanal ocnanalrun" - - -############################################## -# Set variables used in the script -############################################## - - -############################################## -# Begin JOB SPECIFIC work -############################################## - -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OCEAN_ANALYSIS - -mkdir -p "${COM_OCEAN_ANALYSIS}" - -export COMOUT=${COM_OCEAN_ANALYSIS} - -############################################################### -# Run relevant script - -EXSCRIPT=${GDASOCNMBATVRFYSH:-${HOMEgfs}/sorc/gdas.cd/scripts/exgdas_global_marine_analysis_bmat_vrfy.sh} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" -fi - -########################################## -# Do not remove the Temporary working directory (do this in POST) -########################################## -cd "${DATAROOT}" || exit 1 - -exit 0 diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 781380dbb1..4f13bb4c36 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -2,7 +2,6 @@ source "${HOMEgfs}/ush/preamble.sh" -export DATA="${DATAROOT}/${RUN}marinebmat_${cyc}" export DATA=${DATA:-${DATAROOT}/${RUN}marinebmat_${cyc}} # source config.base, config.ocnanal and config.ocnanalbmat # and pass marinebmat to ${machine}.env @@ -35,7 +34,7 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -RUN="enkfgdas" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_ENS_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_ENS_PREV:COM_ICE_HISTORY_TMPL diff --git a/parm/config/gfs/config.ocnanalbmat b/parm/config/gfs/config.ocnanalbmat deleted file mode 100644 index 024da5f51b..0000000000 --- a/parm/config/gfs/config.ocnanalbmat +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -########## config.ocnanalbmat ########## -# Ocn Analysis specific - -echo "BEGIN: config.ocnanalbmat" - -# Get task specific resources -. "${EXPDIR}/config.resources" ocnanalbmat - -echo "END: config.ocnanalbmat" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 758e134a88..8860d35081 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -25,7 +25,7 @@ if (( $# != 1 )); then echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" echo "wavegempak waveawipsbulls waveawipsgridded" echo "postsnd awips gempak npoess" - echo "ocnanalprep prepoceanobs ocnanalbmat ocnanalrun ocnanalecen ocnanalletkf ocnanalchkpt ocnanalpost ocnanalvrfy" + echo "ocnanalprep prepoceanobs marinebmat ocnanalrun ocnanalecen ocnanalletkf ocnanalchkpt ocnanalpost ocnanalvrfy" exit 1 fi @@ -483,7 +483,7 @@ case ${step} in export memory_prepoceanobs="48GB" ;; - "ocnanalbmat" | "marinebmat") + "marinebmat") npes=16 case ${OCNRES} in "025") npes=480;; @@ -494,11 +494,11 @@ case ${step} in exit 4 esac - export wtime_ocnanalbmat="00:30:00" - export npe_ocnanalbmat=${npes} - export nth_ocnanalbmat=1 + export wtime_marinebmat="00:30:00" + export npe_marinebmat=${npes} + export nth_marinebmat=1 export is_exclusive=True - export npe_node_ocnanalbmat=$(( npe_node_max / nth_ocnanalbmat )) + export npe_node_marinebmat=$(( npe_node_max / nth_marinebmat )) ;; "ocnanalrun") diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index b1208dfb3d..c0b72bbc35 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -11,7 +11,7 @@ from .task.upp import UPP from .task.oceanice_products import OceanIceProducts from .task.gfs_forecast import GFSForecast -from .utils import marine_da_utils +from .utils import marine_da_utils __docformat__ = "restructuredtext" __version__ = "0.1.0" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 08309e016b..f2a7e7f945 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -47,7 +47,8 @@ def __init__(self, config): 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, 'BERROR_YAML_DIR': os.path.join(_home_gdas, 'parm', 'soca', 'berror'), 'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml'), - 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca','ensda', 'stage_ens_mem.yaml.j2'), + 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'ensda', 'stage_ens_mem.yaml.j2'), + 'MARINE_DET_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_det_bkg_stage.yaml.j2'), 'ENSPERT_RELPATH': _enspert_relpath } ) @@ -72,38 +73,18 @@ def initialize(self: Task) -> None: # stage fix files logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") - newdirs = [os.path.join(self.task_config.DATA, 'INPUT')] - FileHandler({'mkdir': newdirs}).sync() soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_list).sync() # prepare the MOM6 input.nml mdau.prep_input_nml(self.task_config) - # stage a single background - # TODO: check if we only need 1 deterministic background - # TODO: when we decide to move this task to the end of the previous cycle, - # the bkg will have to come from HISTORY not HISTORY_PREV - ocn_bkg_dir = self.task_config.COMIN_OCEAN_HISTORY_PREV - ice_bkg_dir = self.task_config.COMIN_ICE_HISTORY_PREV - logger.info("Staging background files from {ocn_bkg_dir} and {ice_bkg_dir}") - bkg_list = [] - bkg_list.append([os.path.join(ocn_bkg_dir, f"gdas.ocean.t{self.task_config.gcyc_str}z.inst.f009.nc"), - os.path.join(self.task_config.DATA, "ocn.bkg.nc")]) - bkg_list.append([os.path.join(ice_bkg_dir, f"gdas.ice.t{self.task_config.gcyc_str}z.inst.f009.nc"), - os.path.join(self.task_config.DATA, "ice.bkg.nc")]) - FileHandler({'copy': bkg_list}).sync() - mdau.cice_hist2fms("ice.bkg.nc", "ice.bkg.nc") - - # Copy MOM6 restart - # TODO: check if we can combine this with the step above - logger.info(f"Linking MOM6 restart to ocn.bkg.nc") - rst_list = [] - rst_list.append([os.path.join(self.task_config.DATA, "ocn.bkg.nc"), - os.path.join(self.task_config.DATA, "INPUT", "MOM.res.nc")]) - rst_list.append([os.path.join(self.task_config.DATA, "ice.bkg.nc"), - os.path.join(self.task_config.DATA, "INPUT", "cice.res.nc")]) - FileHandler({'copy': rst_list}).sync() + # stage backgrounds + # TODO(G): Check ocean backgrounds dates for consistency + bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) + FileHandler(bkg_list).sync() + for cice_fname in ['./INPUT/cice.res.nc', './bkg/ice.bkg.f006.nc', './bkg/ice.bkg.f009.nc']: + mdau.cice_hist2fms(cice_fname, cice_fname) # stage the grid generation yaml FileHandler({'copy': [[self.task_config.GRID_GEN_YAML, @@ -141,7 +122,7 @@ def initialize(self: Task) -> None: diffhz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_hz.yaml')) # hybrid EnVAR case - if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: # stage ensemble membersfiles for use in hybrid background error logger.debug(f"Stage ensemble members for the hybrid background error") mdau.stage_ens_mem(self.task_config) @@ -159,9 +140,8 @@ def initialize(self: Task) -> None: hybridweights_config.save(os.path.join(self.task_config.DATA, 'soca_ensweights.yaml')) # need output dir for ensemble perturbations and static B-matrix - logger.debug("Create empty output [ensb, diagb] directories to receive output from executables") - newdirs = [os.path.join(self.task_config.DATA, 'diagb')] - FileHandler({'mkdir': newdirs}).sync() + logger.debug("Create empty diagb directories to receive output from executables") + FileHandler({'mkdir': [os.path.join(self.task_config.DATA, 'diagb')]}).sync() @logit(logger) def gridgen(self: Task) -> None: @@ -267,7 +247,6 @@ def hybrid_weight(self: Task) -> None: # compute the ensemble weights mdau.run(exec_cmd) - @logit(logger) def execute(self: Task) -> None: """Generate the full B-matrix @@ -280,7 +259,7 @@ def execute(self: Task) -> None: self.horizontal_diffusion() # TODO: Make this optional once we've converged on an acceptable set of scales self.vertical_diffusion() # hybrid EnVAR case - if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: self.ensemble_perturbations() # TODO: refactor this from the old scripts self.hybrid_weight() # TODO: refactor this from the old scripts diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 6107e0423c..f86f65e005 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -7,6 +7,7 @@ logger = getLogger(__name__.split('.')[-1]) + @logit(logger) def run(exec_cmd): """Run the executable command @@ -22,6 +23,7 @@ def run(exec_cmd): pass + @logit(logger) def link_executable(task_config, exe_name: str) -> None: """Link the executable to the DATA directory @@ -34,6 +36,7 @@ def link_executable(task_config, exe_name: str) -> None: os.remove(exe_dest) os.symlink(exe_src, exe_dest) + @logit(logger) def prep_input_nml(task_config) -> None: """Prepare the input.nml file @@ -53,6 +56,7 @@ def prep_input_nml(task_config) -> None: nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) nml.write('mom_input.nml') + @logit(logger) def cice_hist2fms(input_filename, output_filename) -> None: """ Reformat the CICE history file so it can be read by SOCA/FMS @@ -75,6 +79,7 @@ def cice_hist2fms(input_filename, output_filename) -> None: # Save the new netCDF file ds.to_netcdf(output_filename, mode='w') + @logit(logger) def stage_ens_mem(task_config) -> None: """ Copy the ensemble members to the DATA directory diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 175ddb07bf..8771729e3a 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -43,7 +43,7 @@ def _get_app_configs(self): configs += ['anal', 'analdiag'] if self.do_jediocnvar: - configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun'] + configs += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] if self.do_hybvar: configs += ['ocnanalecen'] configs += ['ocnanalchkpt', 'ocnanalpost'] @@ -143,7 +143,7 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['anal'] if self.do_jediocnvar: - gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun'] + gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] if self.do_hybvar: gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 60a08549b6..f1e8a7172b 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -678,21 +678,21 @@ def ocnanalprep(self): return task - def ocnanalbmat(self): + def marinebmat(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalprep'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('ocnanalbmat') - task_name = f'{self.cdump}ocnanalbmat' + resources = self.get_resource('marinebmat') + task_name = f'{self.cdump}marinebmat' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalbmat.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marinebmat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -705,7 +705,7 @@ def ocnanalbmat(self): def ocnanalrun(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalbmat'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index a126992cee..83e998ac0e 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -15,7 +15,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', + 'ocnanalprep', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From 8e137d0f347114f967bdb3a651968ec0625e53a5 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 9 Jul 2024 16:20:29 -0400 Subject: [PATCH 07/26] minor tidy --- jobs/JGLOBAL_MARINE_BMAT | 2 +- jobs/rocoto/ocnanalbmat.sh | 19 ------------------- sorc/link_workflow.sh | 2 +- 3 files changed, 2 insertions(+), 21 deletions(-) delete mode 100755 jobs/rocoto/ocnanalbmat.sh diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 4f13bb4c36..1d6409c8fe 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export DATA=${DATA:-${DATAROOT}/${RUN}marinebmat_${cyc}} -# source config.base, config.ocnanal and config.ocnanalbmat +# source config.base, config.ocnanal and config.marinebmat # and pass marinebmat to ${machine}.env source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" diff --git a/jobs/rocoto/ocnanalbmat.sh b/jobs/rocoto/ocnanalbmat.sh deleted file mode 100755 index e62db9115a..0000000000 --- a/jobs/rocoto/ocnanalbmat.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -# Source UFSDA workflow modules -. "${HOMEgfs}/ush/load_ufsda_modules.sh" -status=$? -[[ "${status}" -ne 0 ]] && exit "${status}" - -export job="ocnanalbmat" -export jobid="${job}.$$" - -############################################################### -# Execute the JJOB -"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_BMAT -echo "BMAT gets run here" -status=$? -exit "${status}" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 38ada3716b..81af6c21ac 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -78,7 +78,7 @@ case "${machine}" in "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; *) echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR" - #exit 0 + exit 1 ;; esac From 216df7f232fd93a7247d9865194bf149c6d92ae3 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 9 Jul 2024 16:26:29 -0400 Subject: [PATCH 08/26] fixed shell norms? --- jobs/rocoto/marinebmat.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/rocoto/marinebmat.sh b/jobs/rocoto/marinebmat.sh index dee5f8059a..ee14697edd 100755 --- a/jobs/rocoto/marinebmat.sh +++ b/jobs/rocoto/marinebmat.sh @@ -21,6 +21,5 @@ export PYTHONPATH ############################################################### # Execute the JJOB "${HOMEgfs}"/jobs/JGLOBAL_MARINE_BMAT -echo "BMAT gets run here" status=$? exit "${status}" From 612b456a15f54dbc19e57a7023f821cc6e3ccd77 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 9 Jul 2024 16:29:50 -0400 Subject: [PATCH 09/26] more shell norms --- jobs/JGLOBAL_MARINE_BMAT | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 1d6409c8fe..bcf0e85b10 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -19,7 +19,8 @@ fi # Set variables used in the script ############################################## # shellcheck disable=SC2153 -export GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +export GDATE export gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} export GDUMP="gdas" From c03465325a5ecc20dbdf39a08fdbb56e6f7d5a9f Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Wed, 10 Jul 2024 19:25:31 +0000 Subject: [PATCH 10/26] job deps and arch --- jobs/JGLOBAL_ARCHIVE | 2 ++ jobs/rocoto/marinebmat.sh | 7 ++--- parm/archive/gdasocean_analysis.yaml.j2 | 31 +++++++++++++--------- ush/python/pygfs/task/marine_bmat.py | 7 ++++- workflow/rocoto/gfs_tasks.py | 35 ++++++++++++++----------- 5 files changed, 51 insertions(+), 31 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 28bd820de1..6e9d671285 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -35,6 +35,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ diff --git a/jobs/rocoto/marinebmat.sh b/jobs/rocoto/marinebmat.sh index ee14697edd..47785a9e6c 100755 --- a/jobs/rocoto/marinebmat.sh +++ b/jobs/rocoto/marinebmat.sh @@ -13,9 +13,10 @@ export jobid="${job}.$$" ############################################################### # setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -socaToolsPATH="${HOMEgfs}/sorc/gdas.cd/ush/soca/tools" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${socaToolsPATH}" +# TODO(G): ad to the linking script instead? +# Temporary, should be using JCSDA module when ready +socaToolsPATH="${HOMEgfs}/sorc/gdas.cd/ush/soca" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${socaToolsPATH}" export PYTHONPATH ############################################################### diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index d127ee0b75..ba5a582a1c 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -3,23 +3,30 @@ gdasocean_analysis: name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" required: + # analysis and analysis increments - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' - {% set ocngrid_cycle = '%02d' % (((cycle_HH | int) - 3) % 24) %} - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/gdas.t{{ ocngrid_cycle }}z.ocngrid.nc' {% for domain in ["ocn", "ice"] %} - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.bkgerr_stddev.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' - {% if NMEM_ENS > 2 %} - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.recentering_error.nc' - {% endif %} + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' {% endfor %} + + # static background error + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.bkgerr_stddev.nc' + - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ice.bkgerr_stddev.nc' + + # ensemble background error {% if NMEM_ENS > 2 %} - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_steric_stddev.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_unbal_stddev.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.ssh_total_stddev.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.steric_explained_variance.nc' + - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ice.ens_weights.nc' + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.ens_weights.nc' + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.recentering_error.nc' + {% for diag_type in ["ssh_steric_stddev", "ssh_unbal_stddev", "ssh_total_stddev", "steric_explained_variance"] %} + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.{{ diag_type }}.nc' + {% endfor %} {% endif %} + + # obs space diags - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocn.*.stats.csv' - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc4' + + # runtime configs - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/yaml/*.yaml' diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index f2a7e7f945..6f99fbe779 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -34,7 +34,10 @@ def __init__(self, config): _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert - _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA) + if self.task_config.NMEM_ENS > 0: + _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA) + else: + _enspert_relpath = None # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -331,6 +334,8 @@ def finalize(self: Task) -> None: f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ice.ens_weights.nc") weight_list.append([src, dst]) + # TODO(G): missing ssh_steric_stddev, ssh_unbal_stddev, ssh_total_stddev and steric_explained_variance + FileHandler({'copy': weight_list}).sync() # Copy the YAML files to the OCEAN ROTDIR diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 55d0a119f6..0da73a2ad9 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -654,23 +654,24 @@ def prepoceanobs(self): return task - def ocnanalprep(self): + def marinebmat(self): + + ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}prepoceanobs'} - deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': 'gdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('ocnanalprep') - task_name = f'{self.cdump}ocnanalprep' + resources = self.get_resource('marinebmat') + task_name = f'{self.cdump}marinebmat' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalprep.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marinebmat.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -680,21 +681,25 @@ def ocnanalprep(self): return task - def marinebmat(self): + def ocnanalprep(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalprep'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}prepoceanobs'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'task', 'name': f'{self.cdump}marinebmat'} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': 'gdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('marinebmat') - task_name = f'{self.cdump}marinebmat' + resources = self.get_resource('ocnanalprep') + task_name = f'{self.cdump}ocnanalprep' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marinebmat.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/ocnanalprep.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -707,7 +712,7 @@ def marinebmat(self): def ocnanalrun(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}marinebmat'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}ocnanalprep'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) From 2311b4f6f7a4fb28442dae89ad250fce91abe98c Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 11 Jul 2024 14:36:46 +0000 Subject: [PATCH 11/26] removed dpdcy to soca calc_scales module --- ush/python/pygfs/task/marine_bmat.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 6f99fbe779..fe28f7a3f3 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -3,7 +3,6 @@ import os import glob from logging import getLogger -import calc_scales import pygfs.utils.marine_da_utils as mdau from wxflow import (AttrDict, @@ -30,6 +29,7 @@ def __init__(self, config): _gcyc_str = str(self.task_config.gcyc).zfill(2) _cyc_str = str(self.task_config.cyc).zfill(2) _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') + _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") @@ -52,7 +52,8 @@ def __init__(self, config): 'GRID_GEN_YAML': os.path.join(_home_gdas, 'parm', 'soca', 'gridgen', 'gridgen.yaml'), 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'ensda', 'stage_ens_mem.yaml.j2'), 'MARINE_DET_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_det_bkg_stage.yaml.j2'), - 'ENSPERT_RELPATH': _enspert_relpath + 'ENSPERT_RELPATH': _enspert_relpath, + 'CALC_SCALE_EXEC':_calc_scale_exec } ) @@ -199,7 +200,13 @@ def vertical_diffusion(self: Task) -> None: """Generate the vertical diffusion coefficients """ # compute the vertical correlation scales based on the MLD - calc_scales.run('soca_vtscales.yaml') + FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), + os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() + exec_cmd = Executable("python") + exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('soca_vtscales.yaml') + mdau.run(exec_cmd) # link the executable that computes the correlation scales, gdas_soca_error_covariance_toolbox.x, # and prepare the command to run it From faa84bf68ba49a9f4560273eff9f9a7dfe57d065 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 11 Jul 2024 14:44:43 +0000 Subject: [PATCH 12/26] removed dpdcy to soca calc_scales module 2 --- jobs/rocoto/marinebmat.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/jobs/rocoto/marinebmat.sh b/jobs/rocoto/marinebmat.sh index 47785a9e6c..9b72e5e12c 100755 --- a/jobs/rocoto/marinebmat.sh +++ b/jobs/rocoto/marinebmat.sh @@ -11,14 +11,6 @@ status=$? export job="marinebmat" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -# TODO(G): ad to the linking script instead? -# Temporary, should be using JCSDA module when ready -socaToolsPATH="${HOMEgfs}/sorc/gdas.cd/ush/soca" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${socaToolsPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}"/jobs/JGLOBAL_MARINE_BMAT From c93a43b7c63b394ee9e3b10aa4af98d891461fcf Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 12 Jul 2024 15:34:17 +0000 Subject: [PATCH 13/26] multiple res --- ci/cases/gfsv17/ocnanal.yaml | 5 +++-- ci/cases/yamls/soca_gfs_defaults_ci.yaml | 2 +- jobs/JGLOBAL_MARINE_BMAT | 5 ++--- parm/config/gfs/config.ocnanal | 17 +++++----------- parm/config/gfs/yaml/defaults.yaml | 10 +++------ ush/python/pygfs/task/marine_bmat.py | 26 ++++++++++-------------- ush/python/pygfs/task/marine_letkf.py | 2 +- 7 files changed, 26 insertions(+), 41 deletions(-) diff --git a/ci/cases/gfsv17/ocnanal.yaml b/ci/cases/gfsv17/ocnanal.yaml index 9024afcb31..5f320d23f6 100644 --- a/ci/cases/gfsv17/ocnanal.yaml +++ b/ci/cases/gfsv17/ocnanal.yaml @@ -14,9 +14,10 @@ base: DO_VERFRAD: "YES" DO_VRFY_OCEANDA: "NO" FHMAX_GFS: 240 + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} ocnanal: - SOCA_INPUT_FIX_DIR: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca + SOCA_INPUT_FIX_DIR: /scratch1/NCEPDEV/global/glopara/fix/gdas/soca/20240624/1440x1080x75/soca CASE_ANL: 'C24' SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 100 @@ -25,6 +26,6 @@ ocnanal: NICAS_GRID_SIZE: 15000 prepoceanobs: - SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml + SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml OBSPREP_YAML: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml DMPDIR: /scratch1/NCEPDEV/da/common/ diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml index 3d75cc911a..d0afe85e85 100644 --- a/ci/cases/yamls/soca_gfs_defaults_ci.yaml +++ b/ci/cases/yamls/soca_gfs_defaults_ci.yaml @@ -1,5 +1,5 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_JEDIOCNVAR: "YES" diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index bcf0e85b10..74c3e7b2a4 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -20,9 +20,8 @@ fi ############################################## # shellcheck disable=SC2153 GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -export GDATE -export gPDY=${GDATE:0:8} -export gcyc=${GDATE:8:2} +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} export GDUMP="gdas" export GDUMP_ENS="enkf${GDUMP}" diff --git a/parm/config/gfs/config.ocnanal b/parm/config/gfs/config.ocnanal index fd4230fea2..4d58f2dedf 100644 --- a/parm/config/gfs/config.ocnanal +++ b/parm/config/gfs/config.ocnanal @@ -5,23 +5,16 @@ echo "BEGIN: config.ocnanal" -export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage.yaml.j2" export OBS_YAML_DIR="${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config" -export OBS_LIST=@SOCA_OBS_LIST@ -export OBS_YAML="${OBS_LIST}" -export FV3JEDI_STAGE_YAML="${HOMEgfs}/sorc/gdas.cd/test/soca/testinput/dumy.yaml" +export OBS_LIST=@SOCA_OBS_LIST@ # TODO(GA): doesn't look necessary as is to have +export OBS_YAML="${OBS_LIST}" # OBS_LIST and OBS_YAML pick one or add logic export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ -#export SOCA_VARS=tocn,socn,ssh -#export SABER_BLOCKS_YAML=@SABER_BLOCKS_YAML@ export SOCA_NINNER=@SOCA_NINNER@ -#export CASE_ANL=@CASE_ANL@ export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent -export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin -export SOCA_FIX_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage.yaml.j2" export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2" +export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2" -# NICAS -#export NICAS_RESOL=@NICAS_RESOL@ -#export NICAS_GRID_SIZE=@NICAS_GRID_SIZE@ +export JEDI_BIN=${HOMEgfs}/sorc/gdas.cd/build/bin # TODO(GA): remove once analysis "run" + # and "checkpoint" are refactored echo "END: config.ocnanal" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 2d662a9bcb..ebf6aee7ad 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -22,8 +22,8 @@ base: FHMAX_ENKF_GFS: 12 atmanl: - JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" - STATICB_TYPE: "gsibec" + JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" + STATICB_TYPE: "gsibec" LAYOUT_X_ATMANL: 8 LAYOUT_Y_ATMANL: 8 IO_LAYOUT_X: 1 @@ -45,13 +45,9 @@ snowanl: IO_LAYOUT_Y: 1 ocnanal: - SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. - CASE_ANL: "C48" # TODO: Check in gdasapp if used anywhere for SOCA + SOCA_INPUT_FIX_DIR: "/scratch1/NCEPDEV/global/glopara/fix/gdas/soca/20240624/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 - SABER_BLOCKS_YAML: "" - NICAS_RESOL: 1 - NICAS_GRID_SIZE: 15000 prepoceanobs: SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index fe28f7a3f3..7132fa3c11 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -25,14 +25,11 @@ class MarineBMat(Task): @logit(logger, name="MarineBMat") def __init__(self, config): super().__init__(config) - _gPDY = self.task_config.gPDY - _gcyc_str = str(self.task_config.gcyc).zfill(2) - _cyc_str = str(self.task_config.cyc).zfill(2) _home_gdas = os.path.join(self.task_config.HOMEgfs, 'sorc', 'gdas.cd') _calc_scale_exec = os.path.join(self.task_config.HOMEgfs, 'ush', 'soca', 'calc_scales.py') _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert if self.task_config.NMEM_ENS > 0: _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA) @@ -42,8 +39,6 @@ def __init__(self, config): # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { - 'gcyc_str': _gcyc_str, - 'cyc_str': _cyc_str, 'HOMEgdas': _home_gdas, 'MARINE_WINDOW_BEGIN': _window_begin, 'MARINE_WINDOW_END': _window_end, @@ -53,7 +48,8 @@ def __init__(self, config): 'MARINE_ENSDA_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'ensda', 'stage_ens_mem.yaml.j2'), 'MARINE_DET_STAGE_BKG_YAML_TMPL': os.path.join(_home_gdas, 'parm', 'soca', 'soca_det_bkg_stage.yaml.j2'), 'ENSPERT_RELPATH': _enspert_relpath, - 'CALC_SCALE_EXEC':_calc_scale_exec + 'CALC_SCALE_EXEC': _calc_scale_exec, + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", } ) @@ -113,7 +109,7 @@ def initialize(self: Task) -> None: diffvz_config.save(os.path.join(self.task_config.DATA, 'soca_parameters_diffusion_vt.yaml')) # generate the horizontal diffusion YAML files - if True: # TODO(G): Missing logic to skip this section + if True: # TODO(G): skip this section once we have optimized the scales # stage the correlation scale configuration logger.debug("Generate correlation scale YAML file") FileHandler({'copy': [[os.path.join(self.task_config.BERROR_YAML_DIR, 'soca_setcorscales.yaml'), @@ -298,12 +294,12 @@ def finalize(self: Task) -> None: for diff_type in ['hz', 'vt']: src = os.path.join(self.task_config.DATA, f"{diff_type}_ocean.nc") dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{diff_type}_ocean.nc") + f"{self.task_config.APREFIX}{diff_type}_ocean.nc") diffusion_coeff_list.append([src, dest]) src = os.path.join(self.task_config.DATA, f"hz_ice.nc") dest = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.hz_ice.nc") + f"{self.task_config.APREFIX}hz_ice.nc") diffusion_coeff_list.append([src, dest]) FileHandler({'copy': diffusion_coeff_list}).sync() @@ -316,13 +312,13 @@ def finalize(self: Task) -> None: # ocean diag B src = os.path.join(self.task_config.DATA, 'diagb', f"ocn.bkgerr_stddev.incr.{window_end_iso}.nc") dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ocean.bkgerr_stddev.nc") + f"{self.task_config.APREFIX}ocean.bkgerr_stddev.nc") diagb_list.append([src, dst]) # ice diag B src = os.path.join(self.task_config.DATA, 'diagb', f"ice.bkgerr_stddev.incr.{window_end_iso}.nc") dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ice.bkgerr_stddev.nc") + f"{self.task_config.APREFIX}ice.bkgerr_stddev.nc") diagb_list.append([src, dst]) FileHandler({'copy': diagb_list}).sync() @@ -333,12 +329,12 @@ def finalize(self: Task) -> None: weight_list = [] src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ocean.ens_weights.nc") + f"{self.task_config.APREFIX}ocean.ens_weights.nc") weight_list.append([src, dst]) src = os.path.join(self.task_config.DATA, f"ice.ens_weights.incr.{window_middle_iso}.nc") dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.ice.ens_weights.nc") + f"{self.task_config.APREFIX}ice.ens_weights.nc") weight_list.append([src, dst]) # TODO(G): missing ssh_steric_stddev, ssh_unbal_stddev, ssh_total_stddev and steric_explained_variance @@ -350,6 +346,6 @@ def finalize(self: Task) -> None: yaml_list = [] for yaml_file in yamls: dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{self.task_config.RUN}.t{self.task_config.cyc:02d}.{os.path.basename(yaml_file)}") + f"{self.task_config.APREFIX}{os.path.basename(yaml_file)}") yaml_list.append([yaml_file, dest]) FileHandler({'copy': yaml_list}).sync() diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 0fdd3d9aba..36c26d594b 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -72,7 +72,7 @@ def initialize(self): ensbkgconf.RUN = 'enkfgdas' soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, ensbkgconf) FileHandler(soca_ens_bkg_stage_list).sync() - soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_STAGE_YAML_TMPL, self.task_config) + soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) FileHandler(letkf_stage_list).sync() From 6ab9a5de07e0b66ca56438d1e737457da514a4d4 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 12 Jul 2024 13:31:23 -0400 Subject: [PATCH 14/26] revert account --- ci/cases/yamls/soca_gfs_defaults_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml index d0afe85e85..3d75cc911a 100644 --- a/ci/cases/yamls/soca_gfs_defaults_ci.yaml +++ b/ci/cases/yamls/soca_gfs_defaults_ci.yaml @@ -1,5 +1,5 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_JEDIOCNVAR: "YES" From f6aaf60d52cbee9d12ed8b955eaa4538bb087f55 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 12 Jul 2024 20:05:46 +0000 Subject: [PATCH 15/26] host dpdt fix dir --- ci/cases/gfsv17/ocnanal.yaml | 2 +- ci/cases/yamls/soca_gfs_defaults_ci.yaml | 2 +- parm/config/gfs/yaml/defaults.yaml | 4 ++-- sorc/link_workflow.sh | 2 +- versions/fix.ver | 1 + 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/cases/gfsv17/ocnanal.yaml b/ci/cases/gfsv17/ocnanal.yaml index 5f320d23f6..88b4f20867 100644 --- a/ci/cases/gfsv17/ocnanal.yaml +++ b/ci/cases/gfsv17/ocnanal.yaml @@ -17,7 +17,7 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} ocnanal: - SOCA_INPUT_FIX_DIR: /scratch1/NCEPDEV/global/glopara/fix/gdas/soca/20240624/1440x1080x75/soca + SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca CASE_ANL: 'C24' SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 100 diff --git a/ci/cases/yamls/soca_gfs_defaults_ci.yaml b/ci/cases/yamls/soca_gfs_defaults_ci.yaml index d0afe85e85..3d75cc911a 100644 --- a/ci/cases/yamls/soca_gfs_defaults_ci.yaml +++ b/ci/cases/yamls/soca_gfs_defaults_ci.yaml @@ -1,5 +1,5 @@ defaults: !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml base: - HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_JEDIOCNVAR: "YES" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index ebf6aee7ad..da4d587dff 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -45,12 +45,12 @@ snowanl: IO_LAYOUT_Y: 1 ocnanal: - SOCA_INPUT_FIX_DIR: "/scratch1/NCEPDEV/global/glopara/fix/gdas/soca/20240624/72x35x25/soca" + SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 prepoceanobs: - SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. + SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" DMPDIR: "/scratch1/NCEPDEV/global/glopara/data/experimental_obs" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 81af6c21ac..be912292fe 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -237,7 +237,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 [[ ! -d gdas ]] && mkdir -p gdas cd gdas || exit 1 - for gdas_sub in fv3jedi gsibec obs; do + for gdas_sub in fv3jedi gsibec obs soca; do if [[ -d "${gdas_sub}" ]]; then rm -rf "${gdas_sub}" fi diff --git a/versions/fix.ver b/versions/fix.ver index 1d54572c0b..6e35e651cf 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -9,6 +9,7 @@ export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 export gdas_fv3jedi_ver=20220805 +export gdas_soca_ver=20240624 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 export glwu_ver=20220805 From 8a7b148c3c79a51e60e3cf611e2ab814fbb1bc8c Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 11:48:18 -0400 Subject: [PATCH 16/26] Update jobs/JGLOBAL_MARINE_BMAT Co-authored-by: Rahul Mahajan --- jobs/JGLOBAL_MARINE_BMAT | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 74c3e7b2a4..57648338c2 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -2,19 +2,18 @@ source "${HOMEgfs}/ush/preamble.sh" -export DATA=${DATA:-${DATAROOT}/${RUN}marinebmat_${cyc}} -# source config.base, config.ocnanal and config.marinebmat -# and pass marinebmat to ${machine}.env -source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" - -# If ensemble perturbations are used, save them in a DATAenspert directory that will -# exist throughout the DA cycle -if (( 10#${NMEM_ENS:-0} > 0 )); then +if (( 10#${ENSMEM:-0} > 0 )); then export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}" + export DATA="${DATAjob}/${jobid}" + # Create the directory to hold ensemble perturbations export DATAenspert="${DATAjob}/enspert" if [[ ! -d "${DATAenspert}" ]]; then mkdir -p "${DATAenspert}"; fi fi +# source config.base, config.ocnanal and config.marinebmat +# and pass marinebmat to ${machine}.env +source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" + ############################################## # Set variables used in the script ############################################## From 7cf347df4da0283d3a97341182a2d74ee60de1e5 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 11:48:39 -0400 Subject: [PATCH 17/26] Update jobs/JGLOBAL_ARCHIVE Co-authored-by: Rahul Mahajan --- jobs/JGLOBAL_ARCHIVE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 6e9d671285..8240df317f 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -35,8 +35,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ From c8cbba09e3e3c39db13bb65a57cd13bdc04e078d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 10:54:41 -0500 Subject: [PATCH 18/26] cycle on orion/hercules --- env/HERCULES.env | 20 +++++--------------- sorc/link_workflow.sh | 4 ++-- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/env/HERCULES.env b/env/HERCULES.env index 3ce739cce9..f94bae73cc 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -114,23 +114,13 @@ case ${step} in ;; "marinebmat") - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - - nth_max=$((npe_node_max / npe_node_marinebmat)) - - export NTHREADS_OCNANAL=${nth_marinebmat:-${nth_max}} - [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_marinebmat} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEBMAT="${launcher} -n ${npe_marinebmat}" ;; "ocnanalrun") - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - - nth_max=$((npe_node_max / npe_node_ocnanalrun)) - - export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}} - [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} - export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}" + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}" ;; "ocnanalecen") diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index be912292fe..9722f5a2b8 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -71,8 +71,8 @@ ${LINK_OR_COPY} "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/ru case "${machine}" in "wcoss2") FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;; "hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;; - "orion") FIX_DIR="/work/noaa/global/glopara/fix" ;; - "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; + "orion") FIX_DIR="/work/noaa/global/kfriedma/glopara/fix" ;; + "hercules") FIX_DIR="/work/noaa/global/kfriedma/glopara/fix" ;; "jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; "gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;; From 112085f3d413734aa79652cf8ebc66155f235096 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 12:09:08 -0400 Subject: [PATCH 19/26] Update ush/python/pygfs/utils/marine_da_utils.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/utils/marine_da_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index f86f65e005..1b85041fcc 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -21,8 +21,6 @@ def run(exec_cmd): except Exception: raise WorkflowException(f"An error occured during execution of {exec_cmd}") - pass - @logit(logger) def link_executable(task_config, exe_name: str) -> None: From 4e691243f506acc206f8ec1375945fa8faebe7ef Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 11:10:58 -0500 Subject: [PATCH 20/26] renamed ex script --- ci/cases/gfsv17/ocnanal.yaml | 4 ---- jobs/JGLOBAL_MARINE_BMAT | 2 +- .../{exglobal_marine_bmat_run.py => exglobal_marinebmat.py} | 0 3 files changed, 1 insertion(+), 5 deletions(-) rename scripts/{exglobal_marine_bmat_run.py => exglobal_marinebmat.py} (100%) diff --git a/ci/cases/gfsv17/ocnanal.yaml b/ci/cases/gfsv17/ocnanal.yaml index 88b4f20867..a2d7363c18 100644 --- a/ci/cases/gfsv17/ocnanal.yaml +++ b/ci/cases/gfsv17/ocnanal.yaml @@ -18,12 +18,8 @@ base: ocnanal: SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca - CASE_ANL: 'C24' SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 100 - SABER_BLOCKS_YAML: '' - NICAS_RESOL: 1 - NICAS_GRID_SIZE: 15000 prepoceanobs: SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 57648338c2..3dacec9278 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -47,7 +47,7 @@ mkdir -p "${COMOUT_ICE_BMATRIX}" ############################################################### # Run relevant script -EXSCRIPT=${GDASMARINEBMATRUNPY:-${SCRgfs}/exglobal_marine_bmat_run.py} +EXSCRIPT=${GDASMARINEBMATRUNPY:-${SCRgfs}/exglobal_marinebmat.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/scripts/exglobal_marine_bmat_run.py b/scripts/exglobal_marinebmat.py similarity index 100% rename from scripts/exglobal_marine_bmat_run.py rename to scripts/exglobal_marinebmat.py From 4da579182e4abe92d2ae0a9aaa4fdf47d618460d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 13:24:44 -0400 Subject: [PATCH 21/26] Update ush/python/pygfs/utils/marine_da_utils.py Co-authored-by: Rahul Mahajan --- ush/python/pygfs/utils/marine_da_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 1b85041fcc..995e34a382 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -27,7 +27,7 @@ def link_executable(task_config, exe_name: str) -> None: """Link the executable to the DATA directory """ logger.info(f"Link executable {exe_name}") - logger.warn("Linking is not permitted per EE2.") + logger.warn("WARNING: Linking is not permitted per EE2.") exe_src = os.path.join(task_config.EXECgfs, exe_name) exe_dest = os.path.join(task_config.DATA, exe_name) if os.path.exists(exe_dest): From a0a34d9fe5945a23ce4c784b34ee6590e3bfac83 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 18 Jul 2024 12:25:49 -0500 Subject: [PATCH 22/26] ... --- jobs/JGLOBAL_ARCHIVE | 4 ++-- ush/python/pygfs/utils/marine_da_utils.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index 8240df317f..6e9d671285 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -35,8 +35,8 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \ COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \ COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ - COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ + COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \ COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \ COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \ diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 1b85041fcc..ed5f50b0ad 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -9,7 +9,7 @@ @logit(logger) -def run(exec_cmd): +def run(exec_cmd: Executable) -> None: """Run the executable command """ logger.info(f"Executing {exec_cmd}") @@ -23,7 +23,7 @@ def run(exec_cmd): @logit(logger) -def link_executable(task_config, exe_name: str) -> None: +def link_executable(task_config: AttrDict, exe_name: str) -> None: """Link the executable to the DATA directory """ logger.info(f"Link executable {exe_name}") @@ -36,7 +36,7 @@ def link_executable(task_config, exe_name: str) -> None: @logit(logger) -def prep_input_nml(task_config) -> None: +def prep_input_nml(task_config: AttrDict) -> None: """Prepare the input.nml file TODO: Use jinja2 instead of f90nml """ @@ -56,7 +56,7 @@ def prep_input_nml(task_config) -> None: @logit(logger) -def cice_hist2fms(input_filename, output_filename) -> None: +def cice_hist2fms(input_filename: str, output_filename: str) -> None: """ Reformat the CICE history file so it can be read by SOCA/FMS Simple reformatting utility to allow soca/fms to read the CICE history files """ @@ -79,7 +79,7 @@ def cice_hist2fms(input_filename, output_filename) -> None: @logit(logger) -def stage_ens_mem(task_config) -> None: +def stage_ens_mem(task_config: AttrDict) -> None: """ Copy the ensemble members to the DATA directory Copy the ensemble members to the DATA directory and reformat the CICE history files """ From 65eb77525c03e6c70f299a4268933cbbcd952032 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Fri, 19 Jul 2024 12:24:46 -0500 Subject: [PATCH 23/26] fixed missing import --- ush/python/pygfs/task/marine_bmat.py | 1 - ush/python/pygfs/utils/marine_da_utils.py | 18 +++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 7132fa3c11..9d64e621c9 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -12,7 +12,6 @@ parse_j2yaml, logit, Executable, - jinja, Task) logger = getLogger(__name__.split('.')[-1]) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 050285e0d0..016551878b 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -3,7 +3,13 @@ from logging import getLogger import xarray as xr -from wxflow import FileHandler, logit, WorkflowException, AttrDict, parse_j2yaml, jinja +from wxflow import (FileHandler, + logit, + WorkflowException, + AttrDict, + parse_j2yaml, + Executable, + jinja) logger = getLogger(__name__.split('.')[-1]) @@ -84,17 +90,7 @@ def stage_ens_mem(task_config: AttrDict) -> None: Copy the ensemble members to the DATA directory and reformat the CICE history files """ # Copy the ensemble members to the DATA directory -# ensbkgconf = AttrDict() -# keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS', -# 'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL'] -# for key in keys: -# ensbkgconf[key] = task_config[key] -# ensbkgconf.RUN = 'enkfgdas' -# letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) -# FileHandler(letkf_stage_list).sync() - logger.info("---------------- Stage ensemble members") - # make directories and stage ensemble background files ensbkgconf = AttrDict(task_config) ensbkgconf.RUN = task_config.GDUMP_ENS logger.debug(f"{jinja.Jinja(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf).render}") From 4ba7dee171ee2e7e017bb8d0a2f10a4bde5de54d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Mon, 22 Jul 2024 14:22:47 -0400 Subject: [PATCH 24/26] new hash for gdas.cd --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 01a7c4f433..52f41a298b 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 01a7c4f433346581dee172044e0cd3bd0fe8bd71 +Subproject commit 52f41a298b4c6b7bbf6f203b6579516819fbbf36 From 3e85a94d595851e9e90b244af6360e64fbdb601d Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Tue, 23 Jul 2024 14:20:44 -0400 Subject: [PATCH 25/26] snow at 18z and fixed marine da arch j2 --- parm/archive/gdasocean_analysis.yaml.j2 | 16 ++++++++-------- scripts/exglobal_prep_snow_obs.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index ba5a582a1c..b7c057eacf 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -6,8 +6,8 @@ gdasocean_analysis: # analysis and analysis increments - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ocninc.nc' {% for domain in ["ocn", "ice"] %} - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}.incr.nc' + - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}{{domain}}ana.nc' {% endfor %} # static background error @@ -16,12 +16,12 @@ gdasocean_analysis: # ensemble background error {% if NMEM_ENS > 2 %} - - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ice.ens_weights.nc' - - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.ens_weights.nc' - - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.recentering_error.nc' - {% for diag_type in ["ssh_steric_stddev", "ssh_unbal_stddev", "ssh_total_stddev", "steric_explained_variance"] %} - - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.{{ diag_type }}.nc' - {% endfor %} + - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}ice.ens_weights.nc' + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.ens_weights.nc' + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.recentering_error.nc' + {% for diag_type in ["ssh_steric_stddev", "ssh_unbal_stddev", "ssh_total_stddev", "steric_explained_variance"] %} + - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}ocean.{{ diag_type }}.nc' + {% endfor %} {% endif %} # obs space diags diff --git a/scripts/exglobal_prep_snow_obs.py b/scripts/exglobal_prep_snow_obs.py index d4998a7d84..a6a9070151 100755 --- a/scripts/exglobal_prep_snow_obs.py +++ b/scripts/exglobal_prep_snow_obs.py @@ -21,5 +21,5 @@ # Instantiate the snow prepare task SnowAnl = SnowAnalysis(config) SnowAnl.prepare_GTS() - if f"{ SnowAnl.task_config.cyc }" == '18': + if SnowAnl.task_config.cyc == 0: SnowAnl.prepare_IMS() From 6d3704c7c7b422e4c1e38876118637ff6e9ff69f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 24 Jul 2024 12:11:56 +0000 Subject: [PATCH 26/26] Rename marinebmat log file in gdas.yaml.j2 --- parm/archive/gdas.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index b253d27268..fe6a794224 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -22,7 +22,7 @@ gdas: {% if DO_JEDIOCNVAR %} - "logs/{{ cycle_YMDH }}/{{ RUN }}prepoceanobs.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalprep.log" - - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalbmat.log" + - "logs/{{ cycle_YMDH }}/{{ RUN }}marinebmat.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalrun.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalpost.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}ocnanalchkpt.log"