Skip to content

Commit

Permalink
the latests
Browse files Browse the repository at this point in the history
  • Loading branch information
@AndrewEichmann-NOAA
AndrewEichmann-NOAA committed Sep 13, 2024
1 parent 7f30c33 commit 581f1c3
Show file tree
Hide file tree
Showing 7 changed files with 114 additions and 29 deletions.
12 changes: 9 additions & 3 deletions jobs/JGLOBAL_MARINE_ANALYSIS_LETKF
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,18 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")

gPDY=${GDATE:0:8}
gcyc=${GDATE:8:2}
# needed for obspace yamls
OPREFIX="${RUN}.t${cyc}z."
export OPREFIX

YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL

YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL
YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
COMIN_OBS:COM_OBS_TMPL \
COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \
COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL

##############################################
# Begin JOB SPECIFIC work
Expand Down
2 changes: 2 additions & 0 deletions parm/config/gfs/config.com
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,14 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history'
declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart'
declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input'
declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean'
declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf'
declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean'
declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf'
declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2'
declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}'

declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice'
declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf'
declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice'
declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input'
declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history'
Expand Down
89 changes: 72 additions & 17 deletions ush/python/pygfs/task/marine_letkf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@
from pygfs.task.analysis import Analysis
from typing import Dict
from wxflow import (AttrDict,
Executable,
FileHandler,
logit,
parse_j2yaml,
to_timedelta,
to_YMDH)
to_YMDH,
WorkflowException)

logger = getLogger(__name__.split('.')[-1])

Expand Down Expand Up @@ -49,6 +51,7 @@ def __init__(self, config: Dict) -> None:
self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl')
self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml')
self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs')
self.task_config.OPREFIX = f"{self.task_config.RUN}.t{self.task_config.cyc}z."

@logit(logger)
def initialize(self):
Expand All @@ -64,17 +67,37 @@ def initialize(self):
logger.info("initialize")

# make directories and stage ensemble background files
ensbkgconf = AttrDict()
keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS',
'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL']
for key in keys:
ensbkgconf[key] = self.task_config[key]
ensbkgconf.RUN = 'enkfgdas'
soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, ensbkgconf)
FileHandler(soca_ens_bkg_stage_list).sync()
soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
FileHandler(soca_fix_stage_list).sync()
letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config)
self.task_config['NMEM_ENS']=3
stageconf = AttrDict()
keys = ['current_cycle',
'previous_cycle',
'COM_ICE_LETKF_TMPL',
'COM_OCEAN_LETKF_TMPL',
'COM_ICE_HISTORY_TMPL',
'COM_OCEAN_HISTORY_TMPL',
'COMIN_OCEAN_HISTORY_PREV',
'COMIN_ICE_HISTORY_PREV',
'COMOUT_ICE_LETKF',
'COMOUT_OCEAN_LETKF',
'DATA',
'NMEM_ENS',
'PARMgfs',
'ROTDIR',
'WINDOW_BEGIN',
'WINDOW_MIDDLE']
for key in keys:
stageconf[key] = self.task_config[key]
stageconf.RUN = 'enkfgdas'

# stage ensemble background files
soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, stageconf)
print("soca_ens_bkg_stage_list: ",soca_ens_bkg_stage_list)
FileHandler(soca_ens_bkg_stage_list).sync()

# stage letkf-specific files
letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf)
FileHandler(letkf_stage_list).sync()

obs_list = parse_j2yaml(self.task_config.OBS_YAML, self.task_config)
Expand All @@ -83,7 +106,7 @@ def initialize(self):
obs_files = []
for ob in obs_list['observers']:
obs_name = ob['obs space']['name'].lower()
obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc"
obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4"
obs_files.append((obs_filename, ob))

obs_files_to_copy = []
Expand All @@ -102,12 +125,7 @@ def initialize(self):
FileHandler({'copy': obs_files_to_copy}).sync()

# make the letkf.yaml
letkfconf = AttrDict()
keys = ['WINDOW_BEGIN', 'WINDOW_MIDDLE', 'RUN', 'gcyc', 'NMEM_ENS']
for key in keys:
letkfconf[key] = self.task_config[key]
letkfconf.RUN = 'enkfgdas'
letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, letkfconf)
letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf)
letkf_yaml.observations.observers = obs_to_use
letkf_yaml.save(self.task_config.letkf_yaml_file)

Expand All @@ -120,6 +138,7 @@ def initialize(self):
nml['fms_nml']['domains_stack_size'] = int(domain_stack_size)
nml.write(self.task_config.mom_input_nml, force=True) # force to overwrite if necessary


@logit(logger)
def run(self):
"""Method run for ocean and sea ice LETKF task
Expand All @@ -133,6 +152,33 @@ def run(self):

logger.info("run")

exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANALLETKF)
exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC)
exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML)

try:
logger.debug(f"Executing {exec_cmd_gridgen}")
exec_cmd_gridgen()
except OSError:
raise OSError(f"Failed to execute {exec_cmd_gridgen}")
except Exception:
raise WorkflowException(f"An error occured during execution of {exec_cmd_gridgen}")
pass

exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANALLETKF)
for letkf_exec_arg in self.task_config.letkf_exec_args:
exec_cmd_letkf.add_default_arg(letkf_exec_arg)

try:
logger.debug(f"Executing {exec_cmd_letkf}")
exec_cmd_letkf()
except OSError:
raise OSError(f"Failed to execute {exec_cmd_letkf}")
except Exception:
raise WorkflowException(f"An error occured during execution of {exec_cmd_letkf}")
pass


@logit(logger)
def finalize(self):
"""Method finalize for ocean and sea ice LETKF task
Expand All @@ -145,3 +191,12 @@ def finalize(self):
"""

logger.info("finalize")

letkfsaveconf = AttrDict()
keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN',
'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL']
for key in keys:
letkfsaveconf[key] = self.task_config[key]
letkfsaveconf.RUN = 'enkfgdas'
letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf)
FileHandler(letkf_save_list).sync()
7 changes: 1 addition & 6 deletions ush/python/pygfs/utils/marine_da_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,4 @@ def stage_ens_mem(task_config: AttrDict) -> None:
logger.debug(f"{jinja.Jinja(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf).render}")
letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf)
logger.info(f"{letkf_stage_list}")
FileHandler(letkf_stage_list).sync()

# for _, ens_mem_file in letkf_stage_list['copy']:
# print("ens_mem_file: ", ens_mem_file)
# if re.fullmatch(r'ice\.\d+\.nc', os.path.basename(ens_mem_file)):
# cice_hist2fms(ens_mem_file, ens_mem_file)
FileHandler(letkf_stage_list).sync()
4 changes: 2 additions & 2 deletions workflow/applications/gfs_cycled.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def _get_app_configs(self):
if self.do_jediocnvar:
configs += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun']
if self.do_hybvar:
configs += ['ocnanalecen']
configs += ['marineanalletkf', 'ocnanalecen']
configs += ['ocnanalchkpt', 'ocnanalpost']
if self.do_vrfy_oceanda:
configs += ['ocnanalvrfy']
Expand Down Expand Up @@ -147,7 +147,7 @@ def get_task_names(self):
if self.do_jediocnvar:
gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun']
if self.do_hybvar:
gdas_gfs_common_tasks_before_fcst += ['ocnanalecen']
gdas_gfs_common_tasks_before_fcst += ['marineanalletkf', 'ocnanalecen']
gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost']
if self.do_vrfy_oceanda:
gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy']
Expand Down
27 changes: 27 additions & 0 deletions workflow/rocoto/gfs_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,33 @@ def prepoceanobs(self):

return task

def marineanalletkf(self):

ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'})

deps = []
data = f'{ocean_hist_path}/[email protected]'
dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)

resources = self.get_resource('marineanalletkf')
task_name = f'{self.run}marineanalletkf'
task_dict = {'task_name': task_name,
'resources': resources,
'dependency': dependencies,
'envars': self.envars,
'cycledef': self.run.replace('enkf', ''),
'command': f'{self.HOMEgfs}/jobs/rocoto/marineanalletkf.sh',
'job_name': f'{self.pslot}_{task_name}_@H',
'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log',
'maxtries': '&MAXTRIES;'
}

task = rocoto.create_task(task_dict)

return task

def marinebmat(self):

ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'})
Expand Down
2 changes: 1 addition & 1 deletion workflow/rocoto/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class Tasks:
'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup",
'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal',
'prepoceanobs',
'ocnanalprep', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy',
'ocnanalprep', 'marineanalletkf', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy',
'earc', 'ecen', 'echgres', 'ediag', 'efcs',
'eobs', 'eomg', 'epos', 'esfc', 'eupd',
'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal',
Expand Down

0 comments on commit 581f1c3

Please sign in to comment.