From 240667378645469fc0e1e1411e74d3c8e9e2eac0 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:04:05 -0600 Subject: [PATCH 1/8] Removed dependency for ; added new capabilities for the task. --- workflow/rocoto/gfs_tasks.py | 296 +++++++++++++++++++++++------------ 1 file changed, 200 insertions(+), 96 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 56449cb9d5..86352cf785 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -12,7 +12,8 @@ def __init__(self, app_config: AppConfig, cdump: str) -> None: @staticmethod def _is_this_a_gdas_task(cdump, task_name): if cdump != 'enkfgdas': - raise TypeError(f'{task_name} must be part of the "enkfgdas" cycle and not {cdump}') + raise TypeError( + f'{task_name} must be part of the "enkfgdas" cycle and not {cdump}') # Specific Tasks begin here def stage_ic(self): @@ -73,7 +74,8 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('stage_ic') - task = create_wf_task('stage_ic', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('stage_ic', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -82,14 +84,16 @@ def prep(self): dump_suffix = self._base["DUMP_SUFFIX"] gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] - atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'}) + atm_hist_path = self._template_to_rocoto_cycstring( + self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'}) dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSDMP_TMPL"], {'DMPDIR': dmpdir, 'DUMP_SUFFIX': dump_suffix}) gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_cdumps else False deps = [] - dep_dict = {'type': 'metatask', 'name': 'gdaspost', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', + 'name': 'gdaspost', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) data = f'{atm_hist_path}/gdas.t@Hz.atmf009.nc' dep_dict = {'type': 'data', 'data': data, 'offset': '-06:00:00'} @@ -117,11 +121,14 @@ def waveinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dep_dict = {'type': 'cycleexist', + 'condition': 'not', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + dependencies = rocoto.create_dependency( + dep_condition='or', dep=deps) - task = create_wf_task('waveinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveinit', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -133,18 +140,22 @@ def waveprep(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('waveprep') - task = create_wf_task('waveprep', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveprep', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task def aerosol_init(self): - input_path = self._template_to_rocoto_cycstring(self._base['COM_ATMOS_INPUT_TMPL']) - restart_path = self._template_to_rocoto_cycstring(self._base['COM_ATMOS_RESTART_TMPL']) + input_path = self._template_to_rocoto_cycstring( + self._base['COM_ATMOS_INPUT_TMPL']) + restart_path = self._template_to_rocoto_cycstring( + self._base['COM_ATMOS_RESTART_TMPL']) deps = [] # Files from current cycle - files = ['gfs_ctrl.nc'] + [f'gfs_data.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] + files = ['gfs_ctrl.nc'] + \ + [f'gfs_data.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] for file in files: data = f'{input_path}/{file}' dep_dict = {'type': 'data', 'data': data} @@ -161,7 +172,8 @@ def aerosol_init(self): # Files from previous cycle files = [f'@Y@m@d.@H0000.fv_core.res.nc'] + \ [f'@Y@m@d.@H0000.fv_core.res.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] + \ - [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile}.nc' for tile in range(1, self.n_tiles + 1)] + [f'@Y@m@d.@H0000.fv_tracer.res.tile{tile}.nc' for tile in range( + 1, self.n_tiles + 1)] for file in files: data = [f'{restart_path}', file] @@ -182,14 +194,17 @@ def anal(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', + 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency( + dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('anal') - task = create_wf_task('anal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('anal', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -204,12 +219,14 @@ def sfcanl(self): if self.app_config.do_jedilandda: dep_dict = {'type': 'task', 'name': f'{self.cdump}landanl'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency( + dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('sfcanl') - task = create_wf_task('sfcanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('sfcanl', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -224,12 +241,14 @@ def analcalc(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar and self.cdump in ['gdas']: - dep_dict = {'type': 'task', 'name': 'enkfgdasechgres', 'offset': '-06:00:00'} + dep_dict = {'type': 'task', + 'name': 'enkfgdasechgres', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analcalc') - task = create_wf_task('analcalc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('analcalc', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -241,7 +260,8 @@ def analdiag(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('analdiag') - task = create_wf_task('analdiag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('analdiag', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -253,7 +273,8 @@ def prepatmiodaobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('prepatmiodaobs') - task = create_wf_task('prepatmiodaobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('prepatmiodaobs', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -263,9 +284,11 @@ def atmanlinit(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', + 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency( + dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) @@ -290,7 +313,8 @@ def atmanlrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmanlrun') - task = create_wf_task('atmanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('atmanlrun', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -302,7 +326,8 @@ def atmanlfinal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmanlfinal') - task = create_wf_task('atmanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('atmanlfinal', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -314,7 +339,8 @@ def aeroanlinit(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') - task = create_wf_task('aeroanlinit', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('aeroanlinit', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task def aeroanlrun(self): @@ -325,7 +351,8 @@ def aeroanlrun(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('aeroanlrun') - task = create_wf_task('aeroanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('aeroanlrun', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -337,7 +364,8 @@ def aeroanlfinal(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlfinal') - task = create_wf_task('aeroanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('aeroanlfinal', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -349,7 +377,8 @@ def preplandobs(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('preplandobs') - task = create_wf_task('preplandobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('preplandobs', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -361,12 +390,14 @@ def landanl(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('landanl') - task = create_wf_task('landanl', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('landanl', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task def ocnanalprep(self): - ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"]) + ocean_hist_path = self._template_to_rocoto_cycstring( + self._base["COM_OCEAN_HISTORY_TMPL"]) deps = [] data = f'{ocean_hist_path}/gdas.t@Hz.ocnf009.nc' @@ -488,7 +519,8 @@ def _fcst_forecast_only(self): dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave and self.cdump in self.app_config.wave_cdumps: - wave_job = 'waveprep' if self.app_config.model_app in ['ATMW'] else 'waveinit' + wave_job = 'waveprep' if self.app_config.model_app in [ + 'ATMW'] else 'waveinit' dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'} dependencies.append(rocoto.add_dependency(dep_dict)) @@ -503,14 +535,18 @@ def _fcst_forecast_only(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}aerosol_init'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} + dep_dict = {'type': 'cycleexist', + 'condition': 'not', 'offset': offset} deps.append(rocoto.add_dependency(dep_dict)) - dependencies.append(rocoto.create_dependency(dep_condition='or', dep=deps)) + dependencies.append(rocoto.create_dependency( + dep_condition='or', dep=deps)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + dependencies = rocoto.create_dependency( + dep_condition='and', dep=dependencies) resources = self.get_resource('fcst') - task = create_wf_task('fcst', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('fcst', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -536,12 +572,15 @@ def _fcst_cycled(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}landanl'} dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) + dependencies = rocoto.create_dependency( + dep_condition='and', dep=dependencies) if self.cdump in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dep_dict = {'type': 'cycleexist', + 'condition': 'not', 'offset': '-06:00:00'} dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + dependencies = rocoto.create_dependency( + dep_condition='or', dep=dependencies) cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump @@ -554,20 +593,30 @@ def _fcst_cycled(self): def post(self): add_anl_to_post = False if self.app_config.mode in ['cycled']: - add_anl_to_post = True + # TODO: Setting to `False` since it should do nothing. + add_anl_to_post = False return self._post_task('post', add_anl_to_post=add_anl_to_post) def ocnpost(self): - if self.app_config.mode in ['forecast-only']: # TODO: fix ocnpost in cycled mode + # TODO: fix ocnpost in cycled mode + if self.app_config.mode in ['forecast-only']: return self._post_task('ocnpost', add_anl_to_post=False) + def atmanlpost(self): + if self.app_config.mode in ['cycled']: + return self._post_task('atmanlpost') + def _post_task(self, task_name, add_anl_to_post=False): - if task_name not in ['post', 'ocnpost']: + if task_name not in ['atmanlpost', 'post', 'ocnpost']: raise KeyError(f'Invalid post-processing task: {task_name}') + # TODO: These blocks can be removed since the only task + # that will use `anl` is the `atmanlpost` task. if task_name in ['ocnpost']: add_anl_to_post = False + if task_name in ['atmanlpost']: + add_anl_to_post = False def _get_postgroups(cdump, config, add_anl=False): @@ -586,7 +635,8 @@ def _get_postgroups(cdump, config, add_anl=False): fhmax_hf = config['FHMAX_HF_GFS'] fhout_hf = config['FHOUT_HF_GFS'] fhrs_hf = range(fhmin, fhmax_hf + fhout_hf, fhout_hf) - fhrs = list(fhrs_hf) + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + fhrs = list(fhrs_hf) + \ + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) npostgrp = config['NPOSTGRP'] ngrps = npostgrp if len(fhrs) > npostgrp else len(fhrs) @@ -594,20 +644,24 @@ def _get_postgroups(cdump, config, add_anl=False): fhrs = [f'f{fhr:03d}' for fhr in fhrs] fhrs = np.array_split(fhrs, ngrps) fhrs = [fhr.tolist() for fhr in fhrs] - if add_anl: - fhrs.insert(0, ['anl']) + # TODO: Commenting out for now. + # TODO: if add_anl: + # TODO: fhrs.insert(0, ['anl']) - grp = ' '.join(f'_{fhr[0]}-{fhr[-1]}' if len(fhr) > 1 else f'_{fhr[0]}' for fhr in fhrs) + grp = ' '.join(f'_{fhr[0]}-{fhr[-1]}' if len(fhr) + > 1 else f'_{fhr[0]}' for fhr in fhrs) dep = ' '.join([fhr[-1] for fhr in fhrs]) lst = ' '.join(['_'.join(fhr) for fhr in fhrs]) return grp, dep, lst deps = [] - atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"]) - data = f'{atm_hist_path}/{self.cdump}.t@Hz.atm.log#dep#.txt' - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) + atm_hist_path = self._template_to_rocoto_cycstring( + self._base["COM_ATMOS_HISTORY_TMPL"]) + # TODO: Removing `atm` task dependency. + # data = f'{atm_hist_path}/{self.cdump}.t@Hz.atm.log#dep#.txt' + # dep_dict = {'type': 'data', 'data': data} + # deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -620,10 +674,16 @@ def _get_postgroups(cdump, config, add_anl=False): postenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_postgroups(self.cdump, self._configs[task_name], add_anl=add_anl_to_post) + varval1, varval2, varval3 = _get_postgroups( + self.cdump, self._configs[task_name], add_anl=add_anl_to_post) vardict = {varname2: varval2, varname3: varval3} - cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump + if task_name != "atmanlpost": + cycles = 'gdas_half, gdas' + else: + cycles = 'gdas_half' # TODO: Currently only used for the + # `atmanlpost` task. + cycledef = cycles if self.cdump in ['gdas'] else self.cdump resources = self.get_resource(task_name) task = create_wf_task(task_name, resources, cdump=self.cdump, envar=postenvars, dependency=dependencies, @@ -634,14 +694,16 @@ def _get_postgroups(cdump, config, add_anl=False): def wavepostsbs(self): deps = [] for wave_grid in self._configs['wavepostsbs']['waveGRD'].split(): - wave_hist_path = self._template_to_rocoto_cycstring(self._base["COM_WAVE_HISTORY_TMPL"]) + wave_hist_path = self._template_to_rocoto_cycstring( + self._base["COM_WAVE_HISTORY_TMPL"]) data = f'{wave_hist_path}/{self.cdump}wave.out_grd.{wave_grid}.@Y@m@d.@H0000' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostsbs') - task = create_wf_task('wavepostsbs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostsbs', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -652,13 +714,15 @@ def wavepostbndpnt(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavepostbndpnt') - task = create_wf_task('wavepostbndpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostbndpnt', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task def wavepostbndpntbll(self): deps = [] - atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"]) + atmos_hist_path = self._template_to_rocoto_cycstring( + self._base["COM_ATMOS_HISTORY_TMPL"]) data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf180.txt' dep_dict = {'type': 'data', 'data': data} deps.append(rocoto.add_dependency(dep_dict)) @@ -675,12 +739,14 @@ def wavepostpnt(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave_bnd: - dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpntbll'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump}wavepostbndpntbll'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('wavepostpnt') - task = create_wf_task('wavepostpnt', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavepostpnt', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -691,7 +757,8 @@ def wavegempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wavegempak') - task = create_wf_task('wavegempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wavegempak', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -704,7 +771,8 @@ def waveawipsbulls(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('waveawipsbulls') - task = create_wf_task('waveawipsbulls', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('waveawipsbulls', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -736,7 +804,8 @@ def _wafs_task(self, task_name): if task_name not in ['wafs', 'wafsgcip', 'wafsgrib2', 'wafsgrib20p25']: raise KeyError(f'Invalid WAFS task: {task_name}') - wafs_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_WAFS_TMPL"]) + wafs_path = self._template_to_rocoto_cycstring( + self._base["COM_ATMOS_WAFS_TMPL"]) deps = [] fhrlst = [6] + [*range(12, 36 + 3, 3)] @@ -747,7 +816,8 @@ def _wafs_task(self, task_name): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource(task_name) - task = create_wf_task(task_name, resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task(task_name, resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -758,7 +828,8 @@ def wafsblending(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('wafsblending') - task = create_wf_task('wafsblending', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('wafsblending', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -781,7 +852,8 @@ def postsnd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('postsnd') - task = create_wf_task('postsnd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('postsnd', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -808,7 +880,8 @@ def _get_awipsgroups(cdump, config): if fhmax_hf > 240: fhmax_hf = 240 fhrs_hf = list(range(fhmin, fhmax_hf + fhout_hf, fhout_hf)) - fhrs = fhrs_hf + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) + fhrs = fhrs_hf + \ + list(range(fhrs_hf[-1] + fhout, fhmax + fhout, fhout)) nawipsgrp = config['NAWIPSGRP'] ngrps = nawipsgrp if len(fhrs) > nawipsgrp else len(fhrs) @@ -836,7 +909,8 @@ def _get_awipsgroups(cdump, config): awipsenvars.append(rocoto.create_envar(name=key, value=str(value))) varname1, varname2, varname3 = 'grp', 'dep', 'lst' - varval1, varval2, varval3 = _get_awipsgroups(self.cdump, self._configs['awips']) + varval1, varval2, varval3 = _get_awipsgroups( + self.cdump, self._configs['awips']) vardict = {varname2: varval2, varname3: varval3} resources = self.get_resource('awips') @@ -853,7 +927,8 @@ def gempak(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('gempak') - task = create_wf_task('gempak', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('gempak', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -893,9 +968,11 @@ def metp(self): metpenvars = self.envars.copy() if self.app_config.mode in ['cycled']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H")} + metpenvar_dict = {'SDATE_GFS': self._base.get( + 'SDATE_GFS').strftime("%Y%m%d%H")} elif self.app_config.mode in ['forecast-only']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE').strftime("%Y%m%d%H")} + metpenvar_dict = {'SDATE_GFS': self._base.get( + 'SDATE').strftime("%Y%m%d%H")} metpenvar_dict['METPCASE'] = '#metpcase#' for key, value in metpenvar_dict.items(): metpenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -923,10 +1000,12 @@ def arch(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostpnt'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave_bnd: - dep_dict = {'type': 'task', 'name': f'{self.cdump}wavepostbndpnt'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump}wavepostbndpnt'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: - if self.app_config.mode in ['forecast-only']: # TODO: fix ocnpost to run in cycled mode + # TODO: fix ocnpost to run in cycled mode + if self.app_config.mode in ['forecast-only']: dep_dict = {'type': 'metatask', 'name': f'{self.cdump}ocnpost'} deps.append(rocoto.add_dependency(dep_dict)) # If all verification and ocean/wave coupling is off, add the gdas/gfs post metatask as a dependency @@ -957,21 +1036,25 @@ def cleanup(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('cleanup') - task = create_wf_task('cleanup', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('cleanup', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task # Start of ensemble tasks def eobs(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}prep'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump.replace("enkf","")}prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', + 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('eobs') - task = create_wf_task('eobs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('eobs', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -984,7 +1067,8 @@ def eomg(self): eomgenvars = self.envars.copy() eomgenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['eobs']['NMEM_EOMGGRP']) + groups = self._get_hybgroups( + self._base['NMEM_ENS'], self._configs['eobs']['NMEM_EOMGGRP']) resources = self.get_resource('eomg') task = create_wf_task('eomg', resources, cdump=self.cdump, envar=eomgenvars, dependency=dependencies, @@ -999,7 +1083,8 @@ def ediag(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('ediag') - task = create_wf_task('ediag', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('ediag', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -1013,15 +1098,18 @@ def eupd(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('eupd') - task = create_wf_task('eupd', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('eupd', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task def atmensanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}prepatmiodaobs'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump.replace("enkf","")}prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', + 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -1037,12 +1125,14 @@ def atmensanlrun(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} + dep_dict = {'type': 'metatask', + 'name': 'enkfgdasepmn', 'offset': '-06:00:00'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmensanlrun') - task = create_wf_task('atmensanlrun', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('atmensanlrun', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -1054,7 +1144,8 @@ def atmensanlfinal(self): dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('atmensanlfinal') - task = create_wf_task('atmensanlfinal', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('atmensanlfinal', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -1084,7 +1175,8 @@ def _get_ecengroups(): return grp, dep, lst deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediatmens: dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfinal'} @@ -1113,7 +1205,8 @@ def esfc(self): # eupd_cdump = 'gdas' if 'gdas' in self.app_config.eupd_cdumps else 'gfs' deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}analcalc'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump.replace("enkf","")}analcalc'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_jediatmens: dep_dict = {'type': 'task', 'name': f'{self.cdump}atmensanlfinal'} @@ -1123,7 +1216,8 @@ def esfc(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('esfc') - task = create_wf_task('esfc', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies) + task = create_wf_task('esfc', resources, cdump=self.cdump, + envar=self.envars, dependency=dependencies) return task @@ -1135,18 +1229,23 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}esfc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': '-06:00:00'} + dep_dict = {'type': 'cycleexist', + 'condition': 'not', 'offset': '-06:00:00'} dependencies.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) + dependencies = rocoto.create_dependency( + dep_condition='or', dep=dependencies) efcsenvars = self.envars.copy() efcsenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['efcs']['NMEM_EFCSGRP']) + groups = self._get_hybgroups( + self._base['NMEM_ENS'], self._configs['efcs']['NMEM_EFCSGRP']) if self.cdump == "enkfgfs": - groups = self._get_hybgroups(self._base['NMEM_ENS_GFS'], self._configs['efcs']['NMEM_EFCSGRP_GFS']) - cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') + groups = self._get_hybgroups( + self._base['NMEM_ENS_GFS'], self._configs['efcs']['NMEM_EFCSGRP_GFS']) + cycledef = 'gdas_half,gdas' if self.cdump in [ + 'enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('efcs') task = create_wf_task('efcs', resources, cdump=self.cdump, envar=efcsenvars, dependency=dependencies, metatask='efmn', varname='grp', varval=groups, cycledef=cycledef) @@ -1158,13 +1257,15 @@ def echgres(self): self._is_this_a_gdas_task(self.cdump, 'echgres') deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump.replace("enkf","")}fcst'} + dep_dict = {'type': 'task', + 'name': f'{self.cdump.replace("enkf","")}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.cdump}efcs01'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump + cycledef = 'gdas_half,gdas' if self.cdump in [ + 'enkfgdas'] else self.cdump resources = self.get_resource('echgres') task = create_wf_task('echgres', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies, @@ -1211,7 +1312,8 @@ def _get_eposgroups(epos): varval1, varval2, varval3 = _get_eposgroups(self._configs['epos']) vardict = {varname2: varval2, varname3: varval3} - cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') + cycledef = 'gdas_half,gdas' if self.cdump in [ + 'enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('epos') task = create_wf_task('epos', resources, cdump=self.cdump, envar=eposenvars, dependency=dependencies, @@ -1229,9 +1331,11 @@ def earc(self): earcenvars = self.envars.copy() earcenvars.append(rocoto.create_envar(name='ENSGRP', value='#grp#')) - groups = self._get_hybgroups(self._base['NMEM_ENS'], self._configs['earc']['NMEM_EARCGRP'], start_index=0) + groups = self._get_hybgroups( + self._base['NMEM_ENS'], self._configs['earc']['NMEM_EARCGRP'], start_index=0) - cycledef = 'gdas_half,gdas' if self.cdump in ['enkfgdas'] else self.cdump.replace('enkf', '') + cycledef = 'gdas_half,gdas' if self.cdump in [ + 'enkfgdas'] else self.cdump.replace('enkf', '') resources = self.get_resource('earc') task = create_wf_task('earc', resources, cdump=self.cdump, envar=earcenvars, dependency=dependencies, From 734c4fac107a1efccdd87b2cbd393a6abe472fae Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:10:19 -0600 Subject: [PATCH 2/8] Added task. --- workflow/rocoto/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 29ed57daf2..053ce2203f 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -22,7 +22,8 @@ class Tasks: 'fcst', 'post', 'ocnpost', 'vrfy', 'metp', 'postsnd', 'awips', 'gempak', 'waveawipsbulls', 'waveawipsgridded', 'wavegempak', 'waveinit', - 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep'] + 'wavepostbndpnt', 'wavepostbndpntbll', 'wavepostpnt', 'wavepostsbs', 'waveprep', + 'atmanlpost'] def __init__(self, app_config: AppConfig, cdump: str) -> None: @@ -60,7 +61,7 @@ def _set_envars(envar_dict) -> list: @staticmethod def _get_hybgroups(nens: int, nmem_per_group: int, start_index: int = 1): ngrps = nens / nmem_per_group - groups = ' '.join([f'{x:02d}' for x in range(start_index, int(ngrps) + 1)]) + group = [f'{x:02d}' for x in range(start_index, int(ngrps) + 1)]) return groups def _template_to_rocoto_cycstring(self, template: str, subs_dict: dict = {}) -> str: From 7dcefb491aea9e3673a63be6afd1a75d8c69010b Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:27:46 -0600 Subject: [PATCH 3/8] Added atmanlpost task. --- workflow/applications/gfs_cycled.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 6eff929d5f..6f1ced6d00 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -101,7 +101,7 @@ def get_task_names(self): """ gdas_gfs_common_tasks_before_fcst = ['prep'] - gdas_gfs_common_tasks_after_fcst = ['post'] + gdas_gfs_common_tasks_after_fcst = ['post', 'atmanlpost'] # if self.do_ocean: # TODO: uncomment when ocnpost is fixed in cycled mode # gdas_gfs_common_tasks_after_fcst += ['ocnpost'] gdas_gfs_common_tasks_after_fcst += ['vrfy'] From 864ae8cf881f796069b0088084d99acf4d97c1cf Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:35:17 -0600 Subject: [PATCH 4/8] Added jjob. --- jobs/JGLOBAL_ATMOSANL_POST | 105 +++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100755 jobs/JGLOBAL_ATMOSANL_POST diff --git a/jobs/JGLOBAL_ATMOSANL_POST b/jobs/JGLOBAL_ATMOSANL_POST new file mode 100755 index 0000000000..4e5a4b6108 --- /dev/null +++ b/jobs/JGLOBAL_ATMOSANL_POST @@ -0,0 +1,105 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "post" -c "base post" + + +#################################### +# Specify version numbers +#################################### +export crtm_ver=${post_crtm_ver:-v2.2.6} +export gfs_ver=${gfs_ver:-v15.0.0} +export hwrf_ver=${hwrf_ver:-v11.0.5} +export g2tmpl_ver=${g2tmpl_ver:-v1.5.0} + +############################################## +# Set variables used in the exglobal script +############################################## +export CDUMP=${RUN/enkf} + +############################################## +# Begin JOB SPECIFIC work +############################################## +export APRUNP=${APRUN:-${APRUN_NP}} +export RERUN=${RERUN:-NO} +export HOMECRTM=${HOMECRTM:-${PACKAGEROOT}/lib/crtm/${crtm_ver}} +export FIXCRTM=${CRTM_FIX:-${HOMECRTM}/fix} +export PARMpost=${PARMpost:-${HOMEgfs}/parm/post} +export INLINE_POST=${WRITE_DOPOST:-".false."} + +# Construct COM variables from templates +YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_RESTART COM_ATMOS_ANALYSIS COM_ATMOS_HISTORY COM_ATMOS_MASTER +if [[ ! -d ${COM_ATMOS_MASTER} ]]; then mkdir -m 775 -p "${COM_ATMOS_MASTER}"; fi + +if [[ ${GOESF} == "YES" ]]; then + YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_GOES + if [[ ! -d ${COM_ATMOS_GOES} ]]; then mkdir -m 775 -p "${COM_ATMOS_GOES}"; fi +fi + +for grid in '0p25' '0p50' '1p00'; do + prod_dir="COM_ATMOS_GRIB_${grid}" + GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL" + if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi +done + +if [ "${RUN}" = gfs ];then + export FHOUT_PGB=${FHOUT_GFS:-3} #Output frequency of gfs pgb file at 1.0 and 0.5 deg. +fi +if [ "${RUN}" = gdas ]; then + export IGEN_GFS="gfs_avn" + export IGEN_ANL="anal_gfs" + export IGEN_FCST="gfs_avn" + export IGEN_GDAS_ANL="anal_gdas" + export FHOUT_PGB=${FHOUT:-1} #Output frequency of gfs pgb file at 1.0 and 0.5 deg. +fi + +if [ "${GRIBVERSION}" = grib2 ]; then + export IGEN_ANL="anal_gfs" + export IGEN_FCST="gfs_avn" + export IGEN_GFS="gfs_avn" +fi + +####################################### +# Specify Restart File Name to Key Off +####################################### +# TODO Improve the name of this variable +export restart_file=${COM_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf + +#################################### +# Specify Timeout Behavior of Post +# +# SLEEP_TIME - Amount of time to wait for +# a restart file before exiting +# SLEEP_INT - Amount of time to wait between +# checking for restart files +#################################### +export SLEEP_TIME=900 +export SLEEP_INT=5 + + +############################################################### +# Run relevant exglobal script + +"${HOMEgfs}/scripts/ex${RUN}_atmosanl_post.sh" +status=$? +(( status != 0 )) && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [ -e "${pgmout}" ]; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA:-NO}" = "NO" ]] && rm -rf "${DATA}" + + +exit 0 From c1b6ebf5663f838eda93e86c3b27acba3cb874cc Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:35:47 -0600 Subject: [PATCH 5/8] Added Rocoto task script. --- jobs/rocoto/atmanlpost.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 jobs/rocoto/atmanlpost.sh diff --git a/jobs/rocoto/atmanlpost.sh b/jobs/rocoto/atmanlpost.sh new file mode 100644 index 0000000000..79a43278df --- /dev/null +++ b/jobs/rocoto/atmanlpost.sh @@ -0,0 +1,21 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +## NCEP atmanlpost driver script +############################################################### + +# Source FV3GFS workflow modules +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + +export job="atmanlpost" +export jobid="${job}.$$" + +${HOMEgfs}/jobs/JGLOBAL_ATMOSANL_POST +status=$? +[[ ${status} -ne 0 ]] && exit ${status} + +exit 0 From 5dc6849f3234e210a27f237d593bbce020d0fe04 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:41:36 -0600 Subject: [PATCH 6/8] Added question for reviewers. --- workflow/prod.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workflow/prod.yml b/workflow/prod.yml index 64783dd611..5b0bf82a4c 100644 --- a/workflow/prod.yml +++ b/workflow/prod.yml @@ -87,6 +87,10 @@ suites: FHRLST: 'f( )' FHR: 'f( )' HR: '( )' + # TODO: No description; is this for production only + # (i.e., operations); I am going to need explicit + # documentation, in particular `template`. + # jgfs_atmosanl_post_f( 384 ): post_processing: bufr_sounding: tasks: From dd0e72e198311441c150d2ba7de3216b95dcae2e Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Fri, 20 Oct 2023 12:47:17 -0600 Subject: [PATCH 7/8] Added question for reviewers. --- scripts/exgfs_atmosanl_post.sh | 387 +++++++++++++++++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100755 scripts/exgfs_atmosanl_post.sh diff --git a/scripts/exgfs_atmosanl_post.sh b/scripts/exgfs_atmosanl_post.sh new file mode 100755 index 0000000000..03fb2f85e7 --- /dev/null +++ b/scripts/exgfs_atmosanl_post.sh @@ -0,0 +1,387 @@ +#! /usr/bin/env bash + +##################################################################### +# echo "-----------------------------------------------------" +# echo " exgfs_atmosanl_post.sh" +# echo " Oct 23 Winterbottom -- Initial implementation." +# echo "-----------------------------------------------------" +##################################################################### + +# TODO: This script should be identical to `exgfs_atmos_post.sh` but +# is only run for `anl`; can `exgfs_atmos_post.sh` be used rather than +# adding an additional script? + +source "${HOMEgfs}/ush/preamble.sh" + +cd "${DATA}" || exit 1 + +export POSTGPSH=${POSTGPSH:-${USHgfs}/gfs_post.sh} +export GFSDOWNSH=${GFSDOWNSH:-${USHgfs}/fv3gfs_downstream_nems.sh} +export GFSDOWNSHF=${GFSDOWNSHF:-${USHgfs}/inter_flux.sh} +export GFSDWNSH=${GFSDWNSH:-${USHgfs}/fv3gfs_dwn_nems.sh} +export INLINE_POST=${INLINE_POST:-".false."} + +############################################################ +# Define Variables: +# ----------------- +# FH is the current forecast hour. +# SLEEP_TIME is the number of seconds to sleep before exiting with error. +# SLEEP_INT is the number of seconds to sleep between restrt file checks. +# restart_file is the name of the file to key off of to kick off post. +############################################################ +export IO=${LONB:-1440} +export JO=${LATB:-721} +export OUTTYP=${OUTTYP:-4} +export FLXF=${FLXF:-"YES"} +export FLXGF=${FLXGF:-"YES"} +export GOESF=${GOESF:-"YES"} +export PGBF=${PGBF:-"YES"} +export TCYC=${TCYC:-".t${cyc}z."} +export PREFIX=${PREFIX:-${RUN}${TCYC}} +export machine=${machine:-WCOSS2} + +########################### +# Specify Output layers +########################### +export POSTGPVARS="KPO=57,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,40.,30.,20.,15.,10.,7.,5.,3.,2.,1.,0.7,0.4,0.2,0.1,0.07,0.04,0.02,0.01," + +########################################################## +# Specify variable to directly output pgrb2 files for GDAS/GFS +########################################################## +export IDRT=${IDRT:-0} # IDRT=0 is setting for outputting grib files on lat/lon grid + +############################################################ +# Post Analysis Files before starting the Forecast Post +############################################################ +# Process analysis when post_times is 00 +stime="$(echo "${post_times}" | cut -c1-3)" +export stime +export loganl="${COM_ATMOS_ANALYSIS}/${PREFIX}atmanl.nc" + +if [[ "${stime}" = "anl" ]]; then + if [[ -f "${loganl}" ]]; then + # add new environmental variables for running new ncep post + # Validation date + export VDATE=${PDY}${cyc} + # specify output file name from chgres which is input file name to nceppost + # if model already runs gfs io, make sure GFSOUT is linked to the gfsio file + # new imported variable for global_nceppost.sh + export GFSOUT=${PREFIX}gfsioanl + + # specify smaller control file for GDAS because GDAS does not + # produce flux file, the default will be /nwprod/parm/gfs_cntrl.parm + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + # use grib2 nomonic table in product g2tmpl directory as default + export POSTGRB2TBL=${POSTGRB2TBL:-${g2tmpl_ROOT}/share/params_grib2_tbl_new} + export PostFlatFile=${PostFlatFile:-${PARMpost}/postxconfig-NT-GFS-ANL.txt} + export CTLFILE=${PARMpost}/postcntrl_gfs_anl.xml + fi + + [[ -f flxfile ]] && rm flxfile ; [[ -f nemsfile ]] && rm nemsfile + ln -fs "${COM_ATMOS_ANALYSIS}/${PREFIX}atmanl.nc" nemsfile + export NEMSINP=nemsfile + ln -fs "${COM_ATMOS_ANALYSIS}/${PREFIX}sfcanl.nc" flxfile + export FLXINP=flxfile + + export PGBOUT=pgbfile + export PGIOUT=pgifile + export PGBOUT2=pgbfile.grib2 + export PGIOUT2=pgifile.grib2.idx + export IGEN=${IGEN_ANL} + export FILTER=0 + + ${POSTGPSH} + export err=$?; err_chk + + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + mv "${PGBOUT}" "${PGBOUT2}" + fi + + # Process pgb files + if [[ "${PGBF}" = 'YES' ]]; then + export FH=-1 + export downset=${downset:-2} + ${GFSDOWNSH} + export err=$?; err_chk + fi + + export fhr3=anl + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + MASTERANL=${PREFIX}master.grb2${fhr3} + MASTERANLIDX=${PREFIX}master.grb2i${fhr3} + cp "${PGBOUT2}" "${COM_ATMOS_MASTER}/${MASTERANL}" + ${GRB2INDEX} "${PGBOUT2}" "${COM_ATMOS_MASTER}/${MASTERANLIDX}" + fi + + if [[ "${SENDDBN}" = 'YES' ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_MSC_sfcanl "${job}" "${COM_ATMOS_ANALYSIS}/${PREFIX}sfcanl.nc" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_SA "${job}" "${COM_ATMOS_ANALYSIS}/${PREFIX}atmanl.nc" + if [[ "${PGBF}" = 'YES' ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P25 "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.anl" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P25_WIDX "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.anl.idx" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P25 "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.anl" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P25_WIDX "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.anl.idx" + + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P5 "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.anl" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P5_WIDX "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.anl.idx" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P5 "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.anl" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P5_WIDX "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.anl.idx" + + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_1P0 "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.anl" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_1P0_WIDX "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.anl.idx" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_1P0 "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.anl" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_1P0_WIDX "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.anl.idx" + fi + fi + [[ -f pgbfile.grib2 ]] && rm pgbfile.grib2 + # ecflow_client --event release_pgrb2_anl + else + #### atmanl file not found need failing job + echo " *** FATAL ERROR: No model anl file output " + export err=9 + err_chk + fi +else ## not_anl if_stime + SLEEP_LOOP_MAX=$(( SLEEP_TIME / SLEEP_INT )) + + ############################################################ + # Loop Through the Post Forecast Files + ############################################################ + + for fhr3 in ${post_times}; do + echo "Start processing fhr=${fhr3}" + fhr=$(( 10#${fhr3} )) + ############################### + # Start Looping for the + # existence of the restart files + ############################### + export pgm="postcheck" + ic=1 + while (( ic <= SLEEP_LOOP_MAX )); do + if [[ -f "${restart_file}${fhr3}.txt" ]]; then + break + else + ic=$(( ic + 1 )) + sleep "${SLEEP_INT}" + fi + ############################### + # If we reach this point assume + # fcst job never reached restart + # period and error exit + ############################### + if (( ic == SLEEP_LOOP_MAX )); then + echo " *** FATAL ERROR: No model output for f${fhr3} " + export err=9 + err_chk + fi + done + + ############################### + # Put restart files into /nwges + # for backup to start Model Fcst + ############################### + [[ -f flxfile ]] && rm flxfile ; [[ -f nemsfile ]] && rm nemsfile + ln -fs "${COM_ATMOS_HISTORY}/${PREFIX}atmf${fhr3}.nc" nemsfile + export NEMSINP=nemsfile + ln -fs "${COM_ATMOS_HISTORY}/${PREFIX}sfcf${fhr3}.nc" flxfile + export FLXINP=flxfile + + if (( fhr > 0 )); then + export IGEN=${IGEN_FCST} + else + export IGEN=${IGEN_ANL} + fi + + # No shellcheck, NDATE is not a typo + # shellcheck disable=SC2153 + VDATE="$(${NDATE} "+${fhr}" "${PDY}${cyc}")" + # shellcheck disable= + export VDATE + export OUTTYP=${OUTTYP:-4} + export GFSOUT="${PREFIX}gfsio${fhr3}" + + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + export POSTGRB2TBL="${POSTGRB2TBL:-${g2tmpl_ROOT}/share/params_grib2_tbl_new}" + export PostFlatFile="${PostFlatFile:-${PARMpost}/postxconfig-NT-GFS.txt}" + + if [[ "${RUN}" = "gfs" ]]; then + export IGEN=${IGEN_GFS} + if (( fhr > 0 )); then export IGEN=${IGEN_FCST} ; fi + else + export IGEN=${IGEN_GDAS_ANL} + if (( fhr > 0 )); then export IGEN=${IGEN_FCST} ; fi + fi + if [[ "${RUN}" = "gfs" ]]; then + if (( fhr == 0 )); then + export PostFlatFile="${PARMpost}/postxconfig-NT-GFS-F00.txt" + export CTLFILE="${PARMpost}/postcntrl_gfs_f00.xml" + else + export CTLFILE="${CTLFILEGFS:-${PARMpost}/postcntrl_gfs.xml}" + fi + else + if (( fhr == 0 )); then + export PostFlatFile="${PARMpost}/postxconfig-NT-GFS-F00.txt" + export CTLFILE="${CTLFILEGFS:-${PARMpost}/postcntrl_gfs_f00.xml}" + else + export CTLFILE="${CTLFILEGFS:-${PARMpost}/postcntrl_gfs.xml}" + fi + fi + fi + + export FLXIOUT=flxifile + export PGBOUT=pgbfile + export PGIOUT=pgifile + export PGBOUT2=pgbfile.grib2 + export PGIOUT2=pgifile.grib2.idx + export FILTER=0 + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + MASTERFL=${PREFIX}master.grb2f${fhr3} + MASTERFLIDX=${PREFIX}master.grb2if${fhr3} + fi + + if [[ "${INLINE_POST}" = ".false." ]]; then + ${POSTGPSH} + else + cp -p "${COM_ATMOS_MASTER}/${MASTERFL}" "${PGBOUT}" + fi + export err=$?; err_chk + + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + mv "${PGBOUT}" "${PGBOUT2}" + fi + + # Process pgb files + if [[ "${PGBF}" = 'YES' ]]; then + export FH=$(( fhr )) + export downset=${downset:-2} + ${GFSDOWNSH} + export err=$?; err_chk + fi + + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + if [[ "${INLINE_POST}" = ".false." ]]; then + cp "${PGBOUT2}" "${COM_ATMOS_MASTER}/${MASTERFL}" + fi + ${GRB2INDEX} "${PGBOUT2}" "${COM_ATMOS_MASTER}/${MASTERFLIDX}" + fi + + if [[ "${SENDDBN}" = 'YES' ]]; then + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + if [[ "${PGBF}" = 'YES' ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P25 "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.f${fhr3}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P25_WIDX "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2.0p25.f${fhr3}.idx" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P25 "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.f${fhr3}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P25_WIDX "${job}" "${COM_ATMOS_GRIB_0p25}/${PREFIX}pgrb2b.0p25.f${fhr3}.idx" + + if [[ -s "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.f${fhr3}" ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P5 "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.f${fhr3}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_0P5_WIDX "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2.0p50.f${fhr3}.idx" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P5 "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.f${fhr3}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_0P5_WIDX "${job}" "${COM_ATMOS_GRIB_0p50}/${PREFIX}pgrb2b.0p50.f${fhr3}.idx" + fi + + if [[ -s "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.f${fhr3}" ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_1P0 "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.f${fhr3}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2_1P0_WIDX "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2.1p00.f${fhr3}.idx" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_1P0 "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.f${fhr3}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_PGB2B_1P0_WIDX "${job}" "${COM_ATMOS_GRIB_1p00}/${PREFIX}pgrb2b.1p00.f${fhr3}.idx" + fi + fi + fi + fi + + export fhr + "${USHgfs}/gfs_transfer.sh" + [[ -f pgbfile.grib2 ]] && rm pgbfile.grib2 + + + # use post to generate GFS Grib2 Flux file as model generated Flux file + # will be in nemsio format after FY17 upgrade. + if (( OUTTYP == 4 )) && [[ "${FLXF}" == "YES" ]]; then + if (( fhr == 0 )); then + export PostFlatFile="${PARMpost}/postxconfig-NT-GFS-FLUX-F00.txt" + export CTLFILE="${PARMpost}/postcntrl_gfs_flux_f00.xml" + else + export PostFlatFile="${PARMpost}/postxconfig-NT-GFS-FLUX.txt" + export CTLFILE="${PARMpost}/postcntrl_gfs_flux.xml" + fi + export PGBOUT=fluxfile + export FILTER=0 + export FLUXFL=${PREFIX}sfluxgrbf${fhr3}.grib2 + FLUXFLIDX=${PREFIX}sfluxgrbf${fhr3}.grib2.idx + + if [[ "${INLINE_POST}" = ".false." ]]; then + ${POSTGPSH} + export err=$?; err_chk + mv fluxfile "${COM_ATMOS_MASTER}/${FLUXFL}" + fi + ${WGRIB2} -s "${COM_ATMOS_MASTER}/${FLUXFL}" > "${COM_ATMOS_MASTER}/${FLUXFLIDX}" + + #Add extra flux.1p00 file for coupled + if [[ "${FLXGF}" = 'YES' ]]; then + export FH=$(( fhr )) + ${GFSDOWNSHF} + export err=$?; err_chk + fi + + if [[ "${SENDDBN}" = 'YES' ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_SGB_GB2 "${job}" "${COM_ATMOS_MASTER}/${FLUXFL}" + "${DBNROOT}/bin/dbn_alert" MODEL GFS_SGB_GB2_WIDX "${job}" "${COM_ATMOS_MASTER}/${FLUXFLIDX}" + fi + fi + + # process satellite look alike separately so that master pgb gets out in time + # set outtyp to 2 because master post already generates gfs io files + if [[ "${GOESF}" = "YES" ]]; then + export OUTTYP=${OUTTYP:-4} + + # specify output file name from chgres which is input file name to nceppost + # if model already runs gfs io, make sure GFSOUT is linked to the gfsio file + # new imported variable for global_post.sh + + export GFSOUT=${PREFIX}gfsio${fhr3} + + # link satellite coefficients files, use hwrf version as ops crtm 2.0.5 + # does not new coefficient files used by post + export FIXCRTM="${FIXCRTM:-${CRTM_FIX}}" + "${USHgfs}/link_crtm_fix.sh" "${FIXCRTM}" + + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + export PostFlatFile="${PARMpost}/postxconfig-NT-GFS-GOES.txt" + export CTLFILE="${PARMpost}/postcntrl_gfs_goes.xml" + fi + export FLXINP=flxfile + export FLXIOUT=flxifile + export PGBOUT=goesfile + export PGIOUT=goesifile + export FILTER=0 + export IO=0 + export JO=0 + export IGEN=0 + + if [[ "${NET}" = "gfs" ]]; then + ${POSTGPSH} + export err=$?; err_chk + fi + + if [[ "${GRIBVERSION}" = 'grib2' ]]; then + SPECIALFL="${PREFIX}special.grb2" + SPECIALFLIDX="${PREFIX}special.grb2i" + fi + + mv goesfile "${COM_ATMOS_GOES}/${SPECIALFL}f${fhr3}" + mv goesifile "${COM_ATMOS_GOES}/${SPECIALFLIDX}f${fhr3}" + + if [[ "${SENDDBN}" = "YES" ]]; then + "${DBNROOT}/bin/dbn_alert" MODEL GFS_SPECIAL_GB2 "${job}" "${COM_ATMOS_GOES}/${SPECIALFL}f${fhr3}" + fi + fi + # end of satellite processing + done + + #---------------------------------- +fi ## end_if_stime + +exit 0 + +################## END OF SCRIPT ####################### From 7123d337f24edc4cd94c23409ed1121f170be200 Mon Sep 17 00:00:00 2001 From: henrywinterbottom-wxdev Date: Mon, 23 Oct 2023 11:52:06 -0600 Subject: [PATCH 8/8] Updates for spack-stack path. --- docs/source/noaa_csp.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/noaa_csp.rst b/docs/source/noaa_csp.rst index 3af8956293..5d19f303ab 100644 --- a/docs/source/noaa_csp.rst +++ b/docs/source/noaa_csp.rst @@ -190,7 +190,7 @@ global-workflow execution may be loaded as follows. user@host:$ module unuse /opt/cray/craype/default/modulefiles user@host:$ module unuse /opt/cray/modulefiles - user@host:$ module use /contrib/global-workflow/spack-stack/miniconda/modulefiles/miniconda + user@host:$ module use /contrib/emc_static/spack-stack/miniconda/modulefiles/miniconda user@host:$ module load py39_4.12.0 user@host:$ module load rocoto/1.3.3