From 20fb88c057b22991ed3b889c7388a271cc5db99d Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 28 Aug 2024 17:01:06 +0000 Subject: [PATCH 01/19] replace separate JEDI radiance bias correction files with tarball (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 ++ parm/stage/analysis.yaml.j2 | 9 +---- ush/python/pygfs/task/analysis.py | 36 ++++++++++---------- ush/python/pygfs/task/atm_analysis.py | 48 ++++++++++----------------- 4 files changed, 38 insertions(+), 57 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 9d86292065..7a011671be 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -32,6 +32,8 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" + {% else %} + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index d30389644a..4068f1e928 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,17 +10,10 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} {% endfor %} - {% if DO_JEDIATMVAR %} - {% for ftype in ["satbias.nc", "satbias_cov.nc", "tlapse.txt"] %} - {% for file in glob(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z.atms_*." ~ ftype) %} - - ["{{ file }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] - {% endfor %} - {% endfor %} - {% endif %} {% endfor %} # mem loop {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index bf47b9a950..b8e7809263 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -42,8 +42,7 @@ def initialize(self) -> None: FileHandler(obs_dict).sync() # some analyses need to stage bias corrections - bias_dict = self.get_bias_dict() - FileHandler(bias_dict).sync() + self.get_bias() # link jedi executable to run directory self.link_jediexe() @@ -128,25 +127,21 @@ def get_obs_dict(self) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy + def get_bias(self) -> None: + """Stage radiance bias correciton files - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- + Task: GDAS task Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ - logger.info(f"Extracting a list of bias correction files from Jedi config file") + logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -156,17 +151,22 @@ def get_bias_dict(self) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.radbcor" + copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + FileHandler(bias_dict).sync() + + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4e9d37335c..e32dcdf815 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -207,37 +207,23 @@ def finalize(self: Analysis) -> None: } FileHandler(yaml_copy).sync() - # copy bias correction files to ROTDIR - logger.info("Copy bias correction files from DATA/ to COM/") - biasdir = os.path.join(self.task_config.DATA, 'bc') - biasls = os.listdir(biasdir) - biaslist = [] - for bfile in biasls: - src = os.path.join(biasdir, bfile) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - biaslist.append([src, dest]) - - gprefix = f"{self.task_config.GPREFIX}" - gsuffix = f"{to_YMDH(self.task_config.previous_cycle)}" + ".txt" - aprefix = f"{self.task_config.APREFIX}" - asuffix = f"{to_YMDH(self.task_config.current_cycle)}" + ".txt" - - logger.info(f"Copying {gprefix}*{gsuffix} from DATA/ to COM/ as {aprefix}*{asuffix}") - obsdir = os.path.join(self.task_config.DATA, 'obs') - obsls = os.listdir(obsdir) - for ofile in obsls: - if ofile.endswith(".txt"): - src = os.path.join(obsdir, ofile) - tfile = ofile.replace(gprefix, aprefix) - tfile = tfile.replace(gsuffix, asuffix) - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, tfile) - biaslist.append([src, dest]) - - bias_copy = { - 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], - 'copy': biaslist, - } - FileHandler(bias_copy).sync() + # path of output radiance bias correction tarfile + bfile = f"{self.task_config.APREFIX}radbcor" + radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) + + # get lists of radiance bias correction files to put in tarball + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + + # tar radiance bias correction files to ROTDIR + logger.info(f"Creating radiance bias correction tar file {radtar}") + with tarfile.open(radtar, 'w') as radbcor: + for satfile in satlist: + radbcor.add(satfile, arcname=os.path.basename(satfile)) + for tlapfile in tlaplist: + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) + logger.info(f"Add {radbcor.getnames()}") + radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file") From 73aa233f743ccd9232481e34fd5a3c72ace17a2f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 29 Aug 2024 10:26:14 +0000 Subject: [PATCH 02/19] rename JEDI radiance bias correction tarball file to be more self-describing (#2862) --- parm/archive/gdas_restarta.yaml.j2 | 2 +- parm/stage/analysis.yaml.j2 | 2 +- ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 7a011671be..fc5ce9478d 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -33,7 +33,7 @@ gdas_restarta: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}dtfanl.nc" - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}loginc.txt" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radbcor" + - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar" {% endif %} # Snow surface data diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index 4068f1e928..2d4b7d2854 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,7 +10,7 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "radbcor"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc", "rad_varbc_params.tar"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index b8e7809263..ee674f8ef0 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -152,7 +152,7 @@ def get_bias(self) -> None: obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.radbcor" + bfile = f"{prefix}.rad_varbc_params.tar" copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) break diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index e32dcdf815..99c6e600cb 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -208,7 +208,7 @@ def finalize(self: Analysis) -> None: FileHandler(yaml_copy).sync() # path of output radiance bias correction tarfile - bfile = f"{self.task_config.APREFIX}radbcor" + bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) # get lists of radiance bias correction files to put in tarball From 1203866f068eea2a9ccd28ad9d65b4c6c7d62311 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 7 Sep 2024 11:36:35 +0000 Subject: [PATCH 03/19] update gdas.cd hash to include recent commits to GDASApp develop (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index faa95efb18..554c55a6ad 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit faa95efb18f0f52acab2cf09b17f78406f9b48b1 +Subproject commit 554c55a6ad1a4fc7d5868122a9b0147af3b300a2 From 5985e9f718ded5316007f6d38ac16b5abb2c9649 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 9 Sep 2024 14:22:13 +0000 Subject: [PATCH 04/19] update pygfs jedi class and atmospheric tasks to extract JEDI radiance bias correction files from tarball (#2862) --- ush/python/pygfs/jedi/jedi.py | 9 +++-- ush/python/pygfs/task/analysis.py | 45 ------------------------ ush/python/pygfs/task/atm_analysis.py | 16 +++++++++ ush/python/pygfs/task/atmens_analysis.py | 16 +++++++++ 4 files changed, 36 insertions(+), 50 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 62dcb517ca..dace86c4fe 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -216,11 +216,10 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: obfile = ob['obs bias']['input file'] obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-2]) - for file in ['satbias.nc', 'satbias_cov.nc', 'tlapse.txt']: - bfile = f"{prefix}.{file}" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - # TODO: Why is this specific to ATMOS? + prefix = '.'.join(basename.split('.')[:-3]) + bfile = f"{prefix}.rad_varbc_params.tar" + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index d9cd358219..1d8b38483b 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -41,9 +41,6 @@ def initialize(self) -> None: obs_dict = self.get_obs_dict() FileHandler(obs_dict).sync() - # some analyses need to stage bias corrections - self.get_bias() - # link jedi executable to run directory self.link_jediexe() @@ -126,48 +123,6 @@ def get_obs_dict(self) -> Dict[str, Any]: } return obs_dict - @logit(logger) - def get_bias(self) -> None: - """Stage radiance bias correciton files - - This method stages radiance bias correction files in the obs sub-diretory of the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - - logger.info(f"Copy radiance bias correction tarball if Jedi config processes bias corrected radiances") - observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") - - copylist = [] - for ob in observations['observers']: - if 'obs bias' in ob.keys(): - obfile = ob['obs bias']['input file'] - obdir = os.path.dirname(obfile) - basename = os.path.basename(obfile) - prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(self.task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) - break - - bias_dict = { - 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], - 'copy': copylist - } - FileHandler(bias_dict).sync() - - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() - @logit(logger) def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: """Add cubed-sphere increments to cubed-sphere backgrounds diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ad082063bb..36f768fa13 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -143,6 +144,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 55e72702b1..7004bf059c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import re import glob import gzip import tarfile @@ -142,6 +143,21 @@ def initialize_analysis(self) -> None: FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Extract radiance bias correction files from tarball + for action, filelist in bias_dict.items(): + if 'copy' in action: + for sublist in filelist: + if len(sublist) != 2: + raise Exception( + f"List must be of the form ['src', 'dest'], not {sublist}") + src = sublist[0] + if re.search(".tar", src): + radtar = src + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() + # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_dict = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) From 73e56a7fa14c553f0d64003b88c8e9118f088bea Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 10:50:33 +0000 Subject: [PATCH 05/19] move radiance bias correction staging to jedi class (#2868) --- ush/python/pygfs/jedi/jedi.py | 29 ++++++++++++++++-------- ush/python/pygfs/task/atm_analysis.py | 20 +--------------- ush/python/pygfs/task/atmens_analysis.py | 20 +--------------- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index dace86c4fe..4e527460ab 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 import os +import tarfile from logging import getLogger +from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, + FileHandler, chdir, rm_p, parse_j2yaml, logit, @@ -188,14 +191,10 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation bias correction files that are to be copied to the run directory - from the component directory. - TODO: COM_ATMOS_ANALYSIS_PREV is hardwired here and this method is not appropriate in - `analysis.py` and should be implemented in the component where this is applicable. + This method stages radiance bias correction files in the obs sub-diretory of the run directory Parameters ---------- @@ -204,8 +203,7 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: Returns ---------- - bias_dict: Dict - a dictionary containing the list of observation bias files to copy for FileHandler + None """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -218,14 +216,25 @@ def get_bias_dict(self, task_config: AttrDict) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.rad_varbc_params.tar" - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), os.path.join(obdir, bfile)]) + radtar = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) break bias_dict = { 'mkdir': [os.path.join(task_config.DATA, 'bc')], 'copy': copylist } - return bias_dict + + # stage bias corrections + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract radiance bias correction files from tarball + radtar = os.path.join(obdir, bfile) + with tarfile.open(radtar, "r") as radbcor: + radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {radbcor.getnames()}") + radbcor.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 36f768fa13..c1a6558ade 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -140,24 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 7004bf059c..1f3a065133 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import os -import re import glob import gzip import tarfile @@ -139,24 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - bias_dict = self.jedi.get_bias_dict(self.task_config) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # Extract radiance bias correction files from tarball - for action, filelist in bias_dict.items(): - if 'copy' in action: - for sublist in filelist: - if len(sublist) != 2: - raise Exception( - f"List must be of the form ['src', 'dest'], not {sublist}") - src = sublist[0] - if re.search(".tar", src): - radtar = src - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(self.task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + self.jedi.get_bias(self.task_config) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 666d7d6d89c8c17ef9627367a3efdb76454d07f7 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 15:03:43 +0000 Subject: [PATCH 06/19] initial attempt to generalize processing of variational bias correction files using jedi class (#2862) --- ush/python/pygfs/jedi/jedi.py | 6 +++--- ush/python/pygfs/task/atm_analysis.py | 6 ++++-- ush/python/pygfs/task/atmens_analysis.py | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 4e527460ab..2b899df4e3 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -191,7 +191,7 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: + def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: """Compile a dictionary of observation files to copy This method stages radiance bias correction files in the obs sub-diretory of the run directory @@ -215,9 +215,9 @@ def get_bias(self, task_config: AttrDict) -> Dict[str, Any]: obdir = os.path.dirname(obfile) basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) - bfile = f"{prefix}.rad_varbc_params.tar" + bfile = f"{prefix}.{bias_file}" radtar = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.COM_ATMOS_ANALYSIS_PREV, bfile), radtar]) + copylist.append([os.path.join(task_config.VarBcDir, bfile), radtar]) break bias_dict = { diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c1a6558ade..454d557e87 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", } ) @@ -139,7 +140,8 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - self.jedi.get_bias(self.task_config) + bias_file = f"rad_varbc_params.tar" + self.jedi.get_bias(self.task_config, bias_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1f3a065133..01fc4faac0 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -66,7 +66,8 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", } ) @@ -138,7 +139,8 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") - self.jedi.get_bias(self.task_config) + bias_file = f"rad_varbc_params.tar" + self.jedi.get_bias(self.task_config, bias_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 1e4a28434dc475cf533250219383bf20c347977e Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 10 Sep 2024 17:09:42 +0000 Subject: [PATCH 07/19] move VarBcDir inside initialize_analysis method (#2862) --- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 454d557e87..2cc738008a 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,8 +66,7 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications - 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -140,6 +139,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") + self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" self.jedi.get_bias(self.task_config, bias_file) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 01fc4faac0..b57c80e22c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -66,8 +66,7 @@ def __init__(self, config: Dict[str, Any], yaml_name: Optional[str] = None): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications - 'VarBcDir': f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}", + 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) @@ -139,6 +138,7 @@ def initialize_analysis(self) -> None: # stage bias corrections logger.info(f"Staging list of bias correction files generated from JEDI config") + self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" self.jedi.get_bias(self.task_config, bias_file) From b48e126341795a11ad3941e0beb56c95644fac99 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 11 Sep 2024 18:29:24 +0000 Subject: [PATCH 08/19] update sorc/gdas.cd hash to bring in radiance tarball ctest changes (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 554c55a6ad..032b708f6a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 554c55a6ad1a4fc7d5868122a9b0147af3b300a2 +Subproject commit 032b708f6a476ae4726d1533b82feb21fc8daa92 From ba77d4ec1954968de8e88ebc002073ff85ab6953 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 12 Sep 2024 16:05:43 +0000 Subject: [PATCH 09/19] add extract_tar method to jedi class, use extract_tar in atm and atmens analysis scripts (#2862) --- ush/python/pygfs/jedi/jedi.py | 51 +++++++++++++++++------- ush/python/pygfs/task/atm_analysis.py | 8 +++- ush/python/pygfs/task/atmens_analysis.py | 8 +++- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 2b899df4e3..b5fdb42c7c 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -3,7 +3,6 @@ import os import tarfile from logging import getLogger -from pprint import pformat from typing import List, Dict, Any, Optional from jcb import render from wxflow import (AttrDict, @@ -191,19 +190,24 @@ def get_obs_dict(self, task_config: AttrDict) -> Dict[str, Any]: return obs_dict @logit(logger) - def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: + def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: """Compile a dictionary of observation files to copy - This method stages radiance bias correction files in the obs sub-diretory of the run directory + This method extracts 'observers' from the JEDI yaml and determines from that list + if bias correction tar files are to be copied to the run directory + from the component directory. Parameters ---------- task_config: AttrDict Attribute-dictionary of all configuration variables associated with a GDAS task. + bias_file + name of bias correction tar file Returns ---------- - None + bias_dict: Dict + a dictionary containing the list of observation bias files to copy for FileHandler """ observations = find_value_in_nested_dict(self.config, 'observations') @@ -216,8 +220,8 @@ def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: basename = os.path.basename(obfile) prefix = '.'.join(basename.split('.')[:-3]) bfile = f"{prefix}.{bias_file}" - radtar = os.path.join(obdir, bfile) - copylist.append([os.path.join(task_config.VarBcDir, bfile), radtar]) + tar_file = os.path.join(obdir, bfile) + copylist.append([os.path.join(task_config.VarBcDir, bfile), tar_file]) break bias_dict = { @@ -225,16 +229,33 @@ def get_bias(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: 'copy': copylist } - # stage bias corrections - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + return bias_dict + + @logit(logger) + def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: + """Extract files from list of tarfiles + + This method extract bias correction files from tarball(s) + + Parameters + ---------- + task_config: AttrDict + Attribute-dictionary of all configuration variables associated with a GDAS task. + tar_dict + a dictionary containing the list of tar files + + Returns + ---------- + None + """ - # extract radiance bias correction files from tarball - radtar = os.path.join(obdir, bfile) - with tarfile.open(radtar, "r") as radbcor: - radbcor.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {radbcor.getnames()}") - radbcor.close() + # extract bias correction files from tar file + for tar_file in tar_dict['copy']: + if ".tar" in tar_file[1]: + with tarfile.open(tar_file[1], "r") as tarball: + tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {tarball.getnames()}") + tarball.close() @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 2cc738008a..900e78ba1c 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -141,7 +141,13 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + logger.info(f"Extract bias correction files from tarball") + self.jedi.extract_tar(self.task_config, bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b57c80e22c..a02932d8f5 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -140,7 +140,13 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias(self.task_config, bias_file) + FileHandler(bias_dict).sync() + logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + + # extract bias corrections + logger.info(f"Extract bias correction files from tarball") + self.jedi.extract_tar(self.task_config, bias_dict) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From bc3a6ddab05faaea4466719e96bc36133901a199 Mon Sep 17 00:00:00 2001 From: "russ.treadon" Date: Thu, 12 Sep 2024 16:15:21 +0000 Subject: [PATCH 10/19] correct typo in atmens_analysis.py (2862) --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a02932d8f5..8bab631495 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -140,7 +140,7 @@ def initialize_analysis(self) -> None: logger.info(f"Staging list of bias correction files generated from JEDI config") self.task_config.VarBcDir = f"{self.task_config.COM_ATMOS_ANALYSIS_PREV}" bias_file = f"rad_varbc_params.tar" - bias_dict = self.jedi.get_bias(self.task_config, bias_file) + bias_dict = self.jedi.get_bias_dict(self.task_config, bias_file) FileHandler(bias_dict).sync() logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") From 0ab7bdd6083051f5cb9b9bc687d2a4bbbe5e2e34 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:20:47 +0000 Subject: [PATCH 11/19] improve error handling for jedi class tarfile.extractall (#2862) --- ush/python/pygfs/jedi/jedi.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index b5fdb42c7c..08cfbc02af 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -252,10 +252,19 @@ def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: # extract bias correction files from tar file for tar_file in tar_dict['copy']: if ".tar" in tar_file[1]: - with tarfile.open(tar_file[1], "r") as tarball: - tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {tarball.getnames()}") - tarball.close() + try: + with tarfile.open(tar_file[1], "r") as tarball: + tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tarfile[1]): + logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") @logit(logger) From 8b627a159fdef4bc67e5877b8d75f9a696699675 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:23:52 +0000 Subject: [PATCH 12/19] remove unnecessary close following tarfile add (#2862) --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 900e78ba1c..27243159fa 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -287,7 +287,6 @@ def finalize(self) -> None: for tlapfile in tlaplist: radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) logger.info(f"Add {radbcor.getnames()}") - radbcor.close() # Copy FV3 atm increment to comrot directory logger.info("Copy UFS model readable atm increment file") From eb4fc374b83b693ac767587452a8d63aca77675f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 17 Sep 2024 18:45:14 +0000 Subject: [PATCH 13/19] use endswith to clean up scripting (#2862) --- ush/python/pygfs/jedi/jedi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 08cfbc02af..61ad820302 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -251,7 +251,7 @@ def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: # extract bias correction files from tar file for tar_file in tar_dict['copy']: - if ".tar" in tar_file[1]: + if tar_file[1].endswith('.tar'): try: with tarfile.open(tar_file[1], "r") as tarball: tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) From ed52b724ef8762f9c1eea2c347a5276f49c9469b Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 17:13:24 +0000 Subject: [PATCH 14/19] refactor jedi class method extract_tar (#2862) --- ush/python/pygfs/jedi/jedi.py | 43 +++++++++++------------- ush/python/pygfs/task/atm_analysis.py | 5 +-- ush/python/pygfs/task/atmens_analysis.py | 5 +-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 61ad820302..9e69065bb2 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -232,39 +232,36 @@ def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: return bias_dict @logit(logger) - def extract_tar(self, task_config: AttrDict, tar_dict) -> Dict[str, Any]: - """Extract files from list of tarfiles + def extract_tar(self, tar_file: str) -> None: + """Extract bias correction files from a tarball - This method extract bias correction files from tarball(s) + This method extract files from a tarball Parameters ---------- - task_config: AttrDict - Attribute-dictionary of all configuration variables associated with a GDAS task. - tar_dict - a dictionary containing the list of tar files + tar_file + path/name of tarball Returns ---------- None """ - # extract bias correction files from tar file - for tar_file in tar_dict['copy']: - if tar_file[1].endswith('.tar'): - try: - with tarfile.open(tar_file[1], "r") as tarball: - tarball.extractall(path=os.path.join(task_config.DATA, 'obs')) - logger.info(f"Extract {tarball.getnames()}") - except tarfile.ReadError as err: - if tarfile.is_tarfile(tarfile[1]): - logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") - else: - logger.info() - except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except tarfile.ReadError as err: + if tarfile.is_tarfile(tarfile[1]): + logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + else: + logger.info() + except tarfile.ExtractError as err: + logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 27243159fa..badb01a74a 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -146,8 +146,9 @@ def initialize_analysis(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - logger.info(f"Extract bias correction files from tarball") - self.jedi.extract_tar(self.task_config, bias_dict) + tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 8bab631495..4b2f8ebbf4 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -145,8 +145,9 @@ def initialize_analysis(self) -> None: logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") # extract bias corrections - logger.info(f"Extract bias correction files from tarball") - self.jedi.extract_tar(self.task_config, bias_dict) + tar_file = os.path.join(self.task_config.DATA, 'obs', f"{self.task_config.GPREFIX}{bias_file}") + logger.info(f"Extract bias correction files from {tar_file}") + self.jedi.extract_tar(tar_file) # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") From 80305b67cfbf941e40604d71b7f1413106b2d807 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 18:04:24 +0000 Subject: [PATCH 15/19] update sorc/gdas.cd to current head of GDASApp develop (#2862) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 032b708f6a..55e895f1dc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 032b708f6a476ae4726d1533b82feb21fc8daa92 +Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 From 821ee760945bbd38b5a4c6c7db28d9fa08de7fb2 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 18 Sep 2024 18:21:10 +0000 Subject: [PATCH 16/19] make jedi method extract_tar static, clean up comments, correct typo (#2862) --- ush/python/pygfs/jedi/jedi.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 9e69065bb2..415a0a3c08 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -231,9 +231,10 @@ def get_bias_dict(self, task_config: AttrDict, bias_file) -> Dict[str, Any]: return bias_dict + @staticmethod @logit(logger) - def extract_tar(self, tar_file: str) -> None: - """Extract bias correction files from a tarball + def extract_tar(tar_file: str) -> None: + """Extract files from a tarball This method extract files from a tarball @@ -254,14 +255,14 @@ def extract_tar(self, tar_file: str) -> None: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") except tarfile.ReadError as err: - if tarfile.is_tarfile(tarfile[1]): - logger.error(f"FATAL ERROR: {tarfile[1]} could not be read") - raise tarfile.ReadError(f"FATAL ERROR: unable to read {tarfile[1]}") + if tarfile.is_tarfile(tar_file): + logger.error(f"FATAL ERROR: {tar_file} could not be read") + raise tarfile.ReadError(f"FATAL ERROR: unable to read {tar_file}") else: logger.info() except tarfile.ExtractError as err: - logger.exception(f"FATAL ERROR: unable to extract from {tarfile[1]}") - raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tarfile[1]}") + logger.exception(f"FATAL ERROR: unable to extract from {tar_file}") + raise tarfile.ExtractError("FATAL ERROR: unable to extract from {tar_file}") @logit(logger) From 042ec5094cd62dff9d5abc1e4770dd2451284d0c Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 19 Sep 2024 18:17:17 +0000 Subject: [PATCH 17/19] correctly prefix tlapse radiance bias correction files for cycling (#2862) --- ush/python/pygfs/task/atm_analysis.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index badb01a74a..5f67ea9d72 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -276,9 +276,21 @@ def finalize(self) -> None: bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" radtar = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, bfile) - # get lists of radiance bias correction files to put in tarball + # rename and copy tlapse radiance bias correction files from obs to bc + tlapobs = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + copylist = [] + for tlapfile in tlapobs: + obsfile = os.path.basename(tlapfile).split('.', 2) + newfile = f"{self.task_config.APREFIX}{obsfile[2]}" + copylist.append([tlapfile, os.path.join(self.task_config.DATA, 'bc', newfile)]) + tlapse_dict = { + 'copy': copylist + } + FileHandler(tlapse_dict).sync() + + # get lists of radiance bias correction files to add to tarball satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) - tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*tlapse.txt')) # tar radiance bias correction files to ROTDIR logger.info(f"Creating radiance bias correction tar file {radtar}") From 5306b069727777d814af4433c9b57d46c47f2269 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 20 Sep 2024 10:01:29 +0000 Subject: [PATCH 18/19] extend duration of C96C48_ufs_hybatmDA CI by one cycle to 2024022406 (#2862) --- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 0b5aa7b6ac..b1566d77a0 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -11,7 +11,7 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610 idate: 2024022318 - edate: 2024022400 + edate: 2024022406 nens: 2 gfs_cyc: 1 start: warm From 43c8075b39e9134c0567dd7a4c06c9879a0010a1 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 27 Sep 2024 13:10:12 +0000 Subject: [PATCH 19/19] correct ORION.env typo, adjust JEDI ORION job configurations (#2862) --- env/ORION.env | 2 +- parm/config/gfs/config.resources.ORION | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/env/ORION.env b/env/ORION.env index 1bc7eb60d4..3b8053d060 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -142,7 +142,7 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then export NTHREADS_OCNANAL=${NTHREADSmax} - export APRUN_MARINEANLCHKPT="${APRUN} --cpus-per-task=${NTHREADS_OCNANAL}" + export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}" elif [[ "${step}" = "ocnanalecen" ]]; then diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION index d761df7b73..461b6f14f7 100644 --- a/parm/config/gfs/config.resources.ORION +++ b/parm/config/gfs/config.resources.ORION @@ -23,6 +23,16 @@ case ${step} in # Remove this block once the GSI issue is resolved. export walltime="00:45:00" ;; + "atmanlvar") + # Run on 8 nodes for memory requirement + export tasks_per_node=8 + export walltime="00:45:00" + ;; + "atmensanlobs") + # Run on 8 nodes for memory requirement + export tasks_per_node=8 + export walltime="00:45:00" + ;; *) ;; esac