From bdd67fc6472edefc47faef3006e9c06a2ad243b8 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 12 Aug 2023 13:15:42 -0400 Subject: [PATCH 01/10] Shift window_begin 1 second backward Copy the generated YAML file from initialize to the COM Copy the increment files to the COM/ --- ush/python/pygfs/task/land_analysis.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 0ac5c19a8d..920ef5b669 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -34,6 +34,7 @@ def __init__(self, config): _res = int(self.config['CASE'][1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) + _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"1S") ) _letkfoi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") # Create a local dictionary that is repeatedly used across this class @@ -323,8 +324,10 @@ def execute(self) -> None: def finalize(self) -> None: """Performs closing actions of the Land analysis task This method: - - copies analysis back to COM/ from DATA/ - - tar and gzips the JEDI diagnostic files + - tarring up output diag files and place in COM/ + - copying the generated YAML file from initialize to the COM/ + - copying the analysis files to the COM/ + - copying the increment files to the COM/ Parameters ---------- @@ -336,6 +339,15 @@ def finalize(self) -> None: statfile = os.path.join(self.task_config.COM_LAND_ANALYSIS, f"{self.task_config.APREFIX}landstat.tgz") self.tgz_diags(statfile, self.task_config.DATA) + logger.info("Copy full YAML to COM") + src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") + dest = os.path.join(self.task_config.COM_LAND_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") + yaml_copy = { + 'mkdir': [self.task_config.COM_LAND_ANALYSIS], + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() + logger.info("Copy analysis to COM") template = f'{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' anllist = [] @@ -346,6 +358,16 @@ def finalize(self) -> None: anllist.append([src, dest]) FileHandler({'copy': anllist}).sync() + logger.info('Copy increments to COM') + template = f'landinc.{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' + inclist = [] + for itile in range(1, self.task_config.ntiles + 1): + filename = template.format(tilenum=itile) + src = os.path.join(self.task_config.DATA, 'anl', filename) + dest = os.path.join(self.task_config.COM_LAND_ANALYSIS, filename) + inclist.append([src, dest]) + FileHandler({'copy': inclist}).sync() + @staticmethod @logit(logger) def get_bkg_dict(config: Dict) -> Dict[str, List[str]]: From 5dd9539beef87836a91339b1b108dadbc373b7d5 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 12 Aug 2023 13:38:42 -0400 Subject: [PATCH 02/10] Fix python norm issue --- ush/python/pygfs/task/land_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 920ef5b669..65b4098571 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -34,7 +34,7 @@ def __init__(self, config): _res = int(self.config['CASE'][1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"1S") ) + _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"1S")) _letkfoi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") # Create a local dictionary that is repeatedly used across this class From d38d22788751afc6629caf0a200cb9096791ae0b Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Mon, 14 Aug 2023 21:11:44 -0400 Subject: [PATCH 03/10] Add 1S to the wind lenght and remove a space. --- ush/python/pygfs/task/land_analysis.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 65b4098571..1ca3de26ce 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -45,7 +45,7 @@ def __init__(self, config): 'npz_ges': self.config.LEVS - 1, 'npz': self.config.LEVS - 1, 'LAND_WINDOW_BEGIN': _window_begin, - 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", + 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H1S", 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'jedi_yaml': _letkfoi_yaml @@ -324,10 +324,10 @@ def execute(self) -> None: def finalize(self) -> None: """Performs closing actions of the Land analysis task This method: - - tarring up output diag files and place in COM/ - - copying the generated YAML file from initialize to the COM/ - - copying the analysis files to the COM/ - - copying the increment files to the COM/ + - tar and gzip the output diag files and place in COM/ + - copy the generated YAML file from initialize to the COM/ + - copy the analysis files to the COM/ + - copy the increment files to the COM/ Parameters ---------- @@ -339,7 +339,7 @@ def finalize(self) -> None: statfile = os.path.join(self.task_config.COM_LAND_ANALYSIS, f"{self.task_config.APREFIX}landstat.tgz") self.tgz_diags(statfile, self.task_config.DATA) - logger.info("Copy full YAML to COM") + logger.info("Copy full YAML to COM") src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") dest = os.path.join(self.task_config.COM_LAND_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") yaml_copy = { From 0060e3529da57ad83c526a3aa90ebf3ac30e493a Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 15 Aug 2023 13:06:09 -0400 Subject: [PATCH 04/10] Set the window lenght 6 hours + 2 seconds to make DA window centering at the cycle time. --- ush/python/pygfs/task/land_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 1ca3de26ce..5493aedd16 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -34,7 +34,7 @@ def __init__(self, config): _res = int(self.config['CASE'][1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"1S")) + _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"2S")) _letkfoi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") # Create a local dictionary that is repeatedly used across this class From a56a8349e8a1ca206aad907eee1b3b1f2c10be5e Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 15 Aug 2023 13:15:38 -0400 Subject: [PATCH 05/10] Change the window length. --- ush/python/pygfs/task/land_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 5493aedd16..6ede68cb3a 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -34,7 +34,7 @@ def __init__(self, config): _res = int(self.config['CASE'][1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"2S")) + _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"1S")) _letkfoi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") # Create a local dictionary that is repeatedly used across this class @@ -45,7 +45,7 @@ def __init__(self, config): 'npz_ges': self.config.LEVS - 1, 'npz': self.config.LEVS - 1, 'LAND_WINDOW_BEGIN': _window_begin, - 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H1S", + 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H2S", 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'jedi_yaml': _letkfoi_yaml From 35104d923ebbfb58b45a5b91f3163053d5e29020 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 16 Aug 2023 09:57:18 -0400 Subject: [PATCH 06/10] Update GDASApp commit hash. --- Externals.cfg | 2 +- sorc/checkout.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 64255ded09..d46bebfd78 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -57,7 +57,7 @@ protocol = git required = False [GDASApp] -hash = 09757ce +hash = 2774a10 local_path = sorc/gdas.cd repo_url = https://github.com/NOAA-EMC/GDASApp.git protocol = git diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 9373bc7f30..57bdae649f 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -160,7 +160,7 @@ if [[ ${checkout_gsi} == "YES" ]]; then fi if [[ ${checkout_gdas} == "YES" ]]; then - checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "09757ce"; errs=$((errs + $?)) + checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "2774a10"; errs=$((errs + $?)) fi if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then From 81cd817d396ff17c2159c16e2c725feec4b4ab68 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 16 Aug 2023 10:04:15 -0400 Subject: [PATCH 07/10] Update the window length to 6 hr 1 sec. --- ush/python/pygfs/task/land_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 6ede68cb3a..1ca3de26ce 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -45,7 +45,7 @@ def __init__(self, config): 'npz_ges': self.config.LEVS - 1, 'npz': self.config.LEVS - 1, 'LAND_WINDOW_BEGIN': _window_begin, - 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H2S", + 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H1S", 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'jedi_yaml': _letkfoi_yaml From 4e22b5a388dc312634cf8b1faaef7d17c5334af6 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 18 Aug 2023 09:43:54 -0400 Subject: [PATCH 08/10] create a COM_CONF_TMPL and copy YAML file into COM_CONF. --- jobs/JGLOBAL_LAND_ANALYSIS | 4 ++-- parm/config/gfs/config.com | 1 + ush/python/pygfs/task/land_analysis.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jobs/JGLOBAL_LAND_ANALYSIS b/jobs/JGLOBAL_LAND_ANALYSIS index f66892f29f..d463ca4f87 100755 --- a/jobs/JGLOBAL_LAND_ANALYSIS +++ b/jobs/JGLOBAL_LAND_ANALYSIS @@ -17,12 +17,12 @@ GDUMP="gdas" # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_LAND_ANALYSIS +YMD=${PDY} HH=${cyc} generate_com -rx COM_OBS COM_LAND_ANALYSIS COM_CONF RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} generate_com -rx \ COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL -mkdir -m 775 -p "${COM_LAND_ANALYSIS}" +mkdir -m 775 -p "${COM_LAND_ANALYSIS}" "${COM_CONF}" ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 788a672a3d..dbbd9afb32 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -48,6 +48,7 @@ COM_BASE='${ROTDIR}/${RUN}.${YMD}/${HH}/${MEMDIR}' declare -rx COM_TOP_TMPL='${ROTDIR}/${RUN}.${YMD}/${HH}' +declare -rx COM_CONF_TMPL=${COM_BASE}'/conf' declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model_data/atmos/input' declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model_data/atmos/restart' declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos' diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 1ca3de26ce..79625eeecc 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -341,9 +341,9 @@ def finalize(self) -> None: logger.info("Copy full YAML to COM") src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") - dest = os.path.join(self.task_config.COM_LAND_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") + dest = os.path.join(self.task_config.COM_CONF, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") yaml_copy = { - 'mkdir': [self.task_config.COM_LAND_ANALYSIS], + 'mkdir': [self.task_config.COM_CONF], 'copy': [[src, dest]] } FileHandler(yaml_copy).sync() From 696fbf15feeb223e236b477de60f428884e194bf Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 18 Aug 2023 10:39:32 -0400 Subject: [PATCH 09/10] Address the reviewer comments. --- ush/python/pygfs/task/land_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 79625eeecc..2de3e6933d 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -340,8 +340,8 @@ def finalize(self) -> None: self.tgz_diags(statfile, self.task_config.DATA) logger.info("Copy full YAML to COM") - src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") - dest = os.path.join(self.task_config.COM_CONF, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") + src = os.path.join(self.task_config['DATA'], f"{self.task_config.APREFIX}letkfoi.yaml") + dest = os.path.join(self.task_config.COM_CONF, f"{self.task_config.APREFIX}letkfoi.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CONF], 'copy': [[src, dest]] From c32b1b46d64d5bb795bf88de334ab3cff5a77616 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 22 Aug 2023 20:32:01 -0400 Subject: [PATCH 10/10] Revert the changes for window manipulation. --- ush/python/pygfs/task/land_analysis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 2de3e6933d..7ef74626ef 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -34,7 +34,6 @@ def __init__(self, config): _res = int(self.config['CASE'][1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _window_begin = add_to_datetime(_window_begin, -to_timedelta(f"1S")) _letkfoi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") # Create a local dictionary that is repeatedly used across this class @@ -45,7 +44,7 @@ def __init__(self, config): 'npz_ges': self.config.LEVS - 1, 'npz': self.config.LEVS - 1, 'LAND_WINDOW_BEGIN': _window_begin, - 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H1S", + 'LAND_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", 'jedi_yaml': _letkfoi_yaml