Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update UFS to latest #1889

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Externals.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ protocol = git
required = True

[UFS]
tag = 4d05445
tag = f7a94ce
local_path = sorc/ufs_model.fd
repo_url = https://github.com/ufs-community/ufs-weather-model.git
protocol = git
Expand Down
3 changes: 3 additions & 0 deletions modulefiles/module_base.wcoss2.lua
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ load(pathJoin("udunits", os.getenv("udunits_ver")))
load(pathJoin("gsl", os.getenv("gsl_ver")))
load(pathJoin("cdo", os.getenv("cdo_ver")))

prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/compiler/intel/19.1.3.304")
prepend_path("MODULEPATH", "/apps/test/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.4.2/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.12")

load(pathJoin("hdf5", os.getenv("hdf5_ver")))
load(pathJoin("netcdf", os.getenv("netcdf_ver")))

Expand Down
2 changes: 2 additions & 0 deletions parm/ufs/chem/CAP.rc
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ REPORT_THROUGHPUT: .false.

USE_SHMEM: 0

GridType: Cubed-Sphere

MAPL_ENABLE_TIMERS: NO
MAPL_ENABLE_MEMUTILS: NO
PRINTSPEC: 0 # (0: OFF, 1: IMPORT & EXPORT, 2: IMPORT, 3: EXPORT)
Expand Down
2 changes: 2 additions & 0 deletions parm/ufs/chem/GOCART2G_GridComp.rc
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,5 @@ PASSIVE_INSTANCES_NI:
aerosol_monochromatic_optics_wavelength_in_nm_from_LUT: 470 550 670 870
wavelengths_for_profile_aop_in_nm: 470 550 670 870 # must be included in LUT
wavelengths_for_vertically_integrated_aop_in_nm: 470 550 670 870 # must be included in LUT

use_threads: .TRUE.
11 changes: 8 additions & 3 deletions parm/ufs/nems.configure.cpld.IN
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ runSeq::
MED med_phases_post_atm
ICE -> MED :remapMethod=redist
MED med_phases_post_ice
MED med_phases_ocnalb_run
MED med_phases_prep_ocn_accum
@
OCN -> MED :remapMethod=redist
Expand All @@ -89,12 +90,10 @@ MED_attributes::
ATM_model = @[atm_model]
ICE_model = @[ice_model]
OCN_model = @[ocn_model]
history_n = 0
history_option = nhours
history_ymd = -999
coupling_mode = @[CPLMODE]
history_tile_atm = @[ATMTILESIZE]
pio_rearranger = box
ocean_albedo_limit = @[ocean_albedo_limit]
::
ALLCOMP_attributes::
ScalarFieldCount = 2
Expand All @@ -114,4 +113,10 @@ ALLCOMP_attributes::
stop_n = @[FHMAX]
stop_option = nhours
stop_ymd = -999
orb_eccen = 1.e36
orb_iyear = 2000
orb_iyear_align = 2000
orb_mode = fixed_year
orb_mvelp = 1.e36
orb_obliq = 1.e36
::
11 changes: 8 additions & 3 deletions parm/ufs/nems.configure.cpld_aero.IN
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ runSeq::
MED med_phases_post_atm
ICE -> MED :remapMethod=redist
MED med_phases_post_ice
MED med_phases_ocnalb_run
MED med_phases_prep_ocn_accum
@
OCN -> MED :remapMethod=redist
Expand All @@ -101,12 +102,10 @@ MED_attributes::
ATM_model = @[atm_model]
ICE_model = @[ice_model]
OCN_model = @[ocn_model]
history_n = 0
history_option = nhours
history_ymd = -999
coupling_mode = @[CPLMODE]
history_tile_atm = @[ATMTILESIZE]
pio_rearranger = box
ocean_albedo_limit = @[ocean_albedo_limit]
::
ALLCOMP_attributes::
ScalarFieldCount = 2
Expand All @@ -126,4 +125,10 @@ ALLCOMP_attributes::
stop_n = @[FHMAX]
stop_option = nhours
stop_ymd = -999
orb_eccen = 1.e36
orb_iyear = 2000
orb_iyear_align = 2000
orb_mode = fixed_year
orb_mvelp = 1.e36
orb_obliq = 1.e36
::
11 changes: 8 additions & 3 deletions parm/ufs/nems.configure.cpld_aero_outerwave.IN
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ runSeq::
MED med_phases_post_atm
ICE -> MED :remapMethod=redist
MED med_phases_post_ice
MED med_phases_ocnalb_run
MED med_phases_prep_ocn_accum
MED med_phases_prep_wav_accum
@
Expand All @@ -121,12 +122,10 @@ MED_attributes::
ICE_model = @[ice_model]
OCN_model = @[ocn_model]
WAV_model = @[wav_model]
history_n = 0
history_option = nhours
history_ymd = -999
coupling_mode = @[CPLMODE]
history_tile_atm = @[ATMTILESIZE]
pio_rearranger = box
ocean_albedo_limit = @[ocean_albedo_limit]
::
ALLCOMP_attributes::
ScalarFieldCount = 2
Expand All @@ -146,4 +145,10 @@ ALLCOMP_attributes::
stop_n = @[FHMAX]
stop_option = nhours
stop_ymd = -999
orb_eccen = 1.e36
orb_iyear = 2000
orb_iyear_align = 2000
orb_mode = fixed_year
orb_mvelp = 1.e36
orb_obliq = 1.e36
::
12 changes: 9 additions & 3 deletions parm/ufs/nems.configure.cpld_aero_wave.IN
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ runSeq::
MED med_phases_post_ice
WAV -> MED :remapMethod=redist
MED med_phases_post_wav
MED med_phases_ocnalb_run
MED med_phases_prep_ocn_accum
@
OCN -> MED :remapMethod=redist
Expand All @@ -121,11 +122,10 @@ MED_attributes::
ICE_model = @[ice_model]
OCN_model = @[ocn_model]
WAV_model = @[wav_model]
history_n = 0
history_option = nhours
history_ymd = -999
coupling_mode = @[CPLMODE]
history_tile_atm = @[ATMTILESIZE]
pio_rearranger = box
ocean_albedo_limit = @[ocean_albedo_limit]
::
ALLCOMP_attributes::
ScalarFieldCount = 2
Expand All @@ -145,4 +145,10 @@ ALLCOMP_attributes::
stop_n = @[FHMAX]
stop_option = nhours
stop_ymd = -999
orb_eccen = 1.e36
orb_iyear = 2000
orb_iyear_align = 2000
orb_mode = fixed_year
orb_mvelp = 1.e36
orb_obliq = 1.e36
::
11 changes: 8 additions & 3 deletions parm/ufs/nems.configure.cpld_outerwave.IN
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ runSeq::
MED med_phases_post_atm
ICE -> MED :remapMethod=redist
MED med_phases_post_ice
MED med_phases_ocnalb_run
MED med_phases_prep_ocn_accum
MED med_phases_prep_wav_accum
@
Expand All @@ -109,12 +110,10 @@ MED_attributes::
ICE_model = @[ice_model]
OCN_model = @[ocn_model]
WAV_model = @[wav_model]
history_n = 0
history_option = nhours
history_ymd = -999
coupling_mode = @[CPLMODE]
history_tile_atm = @[ATMTILESIZE]
pio_rearranger = box
ocean_albedo_limit = @[ocean_albedo_limit]
::
ALLCOMP_attributes::
ScalarFieldCount = 2
Expand All @@ -134,4 +133,10 @@ ALLCOMP_attributes::
stop_n = @[FHMAX]
stop_option = nhours
stop_ymd = -999
orb_eccen = 1.e36
orb_iyear = 2000
orb_iyear_align = 2000
orb_mode = fixed_year
orb_mvelp = 1.e36
orb_obliq = 1.e36
::
11 changes: 8 additions & 3 deletions parm/ufs/nems.configure.cpld_wave.IN
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ runSeq::
MED med_phases_post_ice
WAV -> MED :remapMethod=redist
MED med_phases_post_wav
MED med_phases_ocnalb_run
MED med_phases_prep_ocn_accum
@
OCN -> MED :remapMethod=redist
Expand All @@ -109,12 +110,10 @@ MED_attributes::
ICE_model = @[ice_model]
OCN_model = @[ocn_model]
WAV_model = @[wav_model]
history_n = 0
history_option = nhours
history_ymd = -999
coupling_mode = @[CPLMODE]
history_tile_atm = @[ATMTILESIZE]
pio_rearranger = box
ocean_albedo_limit = @[ocean_albedo_limit]
::
ALLCOMP_attributes::
ScalarFieldCount = 2
Expand All @@ -134,4 +133,10 @@ ALLCOMP_attributes::
stop_n = @[FHMAX]
stop_option = nhours
stop_ymd = -999
orb_eccen = 1.e36
orb_iyear = 2000
orb_iyear_align = 2000
orb_mode = fixed_year
orb_mvelp = 1.e36
orb_obliq = 1.e36
::
2 changes: 1 addition & 1 deletion sorc/checkout.sh
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ errs=0
checkout "wxflow" "https://github.com/NOAA-EMC/wxflow" "528f5ab" ; errs=$((errs + $?))
checkout "gfs_utils.fd" "https://github.com/NOAA-EMC/gfs-utils" "a283262" ; errs=$((errs + $?))
checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "72a0471" ; errs=$((errs + $?))
checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-4d05445}" ; errs=$((errs + $?))
checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-f7a94ce}" ; errs=$((errs + $?))
checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?))

if [[ ${checkout_gsi} == "YES" ]]; then
Expand Down
4 changes: 2 additions & 2 deletions ush/forecast_postdet.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1038,7 +1038,7 @@ GOCART_postdet() {
rm -f "${COM_CHEM_HISTORY}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4"
fi

${NLN} "${COM_CHEM_HISTORY}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4" \
"${DATA}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4"
#${NLN} "${COM_CHEM_HISTORY}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4" \
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@WalterKolczynski-NOAA @bbakernoaa @lipan-NOAA @rmontuoro --- I think i have other things working in my other branch so I'm coming back to other issues including this. Is it anticipated that if I link these files that the model wouldn't run? Barry helped me figure out that this was causing some earlier crashes. I can re-test to see if this is still an issue, but want to see if there's some known issue with this or suggested workaround. My other branch is : https://github.com/JessicaMeixner-NOAA/global-workflow/tree/feature/updateufsstack15

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without the links, the output will never get into a permanent location. I think what needs to happen is any existing files at the target need to be deleted. GOCART seems to be okay with the links as long as the target doesn't exist, otherwise we would be seeing more problems.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay I can try again without this commented out and see if other issuses I was having was partially disguised as this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Uncommenting this line resulted in a failure still (even from a completely clean run):

nid001088.dogwood.wcoss2.ncep.noaa.gov 0: UFS Aerosols: Advancing from 2021-03-23T17:40:00 to 2021-03-23T18:00:00
nid001088.dogwood.wcoss2.ncep.noaa.gov 0:
nid001088.dogwood.wcoss2.ncep.noaa.gov 0:  Writing:     28 Slices to File:  gocart.inst_aod.20210323_1800z.nc4
nid001088.dogwood.wcoss2.ncep.noaa.gov 0: pe=00000 FAIL at line=00187    NetCDF4_FileFormatter.F90                <status=13>
pe=00000 FAIL at line=00061    HistoryCollection.F90                    <status=13>
pe=00000 FAIL at line=00790    ServerThread.F90                         <status=13>
pe=00000 FAIL at line=00138    BaseServer.F90                           <status=13>
pe=00000 FAIL at line=00981    ServerThread.F90                         <status=13>
pe=00000 FAIL at line=00094    MessageVisitor.F90                       <status=13>
pe=00000 FAIL at line=00113    AbstractMessage.F90                      <status=13>
pe=00000 FAIL at line=00107    SimpleSocket.F90                         <status=13>
pe=00000 FAIL at line=00429    ClientThread.F90                         <status=13>
pe=00000 FAIL at line=00363    ClientManager.F90                        <status=13>
pe=00000 FAIL at line=03524    MAPL_HistoryGridComp.F90                 <status=13>
pe=00000 FAIL at line=01818    MAPL_Generic.F90                         <status=13>
pe=00000 FAIL at line=01284    MAPL_CapGridComp.F90                     <status=13>
pe=00000 FAIL at line=01213    MAPL_CapGridComp.F90                     <status=13>
pe=00000 FAIL at line=01159    MAPL_CapGridComp.F90                     <status=13>
pe=00000 FAIL at line=00827    MAPL_CapGridComp.F90                     <status=13>
pe=00000 FAIL at line=00967    MAPL_CapGridComp.F90                     <status=13>
nid001088.dogwood.wcoss2.ncep.noaa.gov 0: MPICH ERROR [Rank 0] [job id 2794f105-9f31-4db4-b1dc-56a1883195f6] [Fri Oct 13 12:32:25 2023] [nid001088] - Abort(1) (rank 0 in comm 496): application called MPI_Abort(comm=0x84000002, 1) - process 0

nid001088.dogwood.wcoss2.ncep.noaa.gov 0: forrtl: severe (174): SIGSEGV, segmentation fault occurred
Image              PC                Routine            Line        Source
ufs_model.x        000000000641185A  Unknown               Unknown  Unknown
libpthread-2.31.s  00001456DAA868C0  Unknown               Unknown  Unknown
libmpi_intel.so.1  00001456DCBE3BFA  Unknown               Unknown  Unknown
libmpi_intel.so.1  00001456DCABC05F  Unknown               Unknown  Unknown
libmpi_intel.so.1  00001456DB1C9DA8  MPI_Abort             Unknown  Unknown
ufs_model.x        00000000013E79C4  _ZN5ESMCI3VMK5abo         757  ESMCI_VMKernel.C
ufs_model.x        00000000013C6B57  _ZN5ESMCI2VM5abor        3597  ESMCI_VM.C
ufs_model.x        0000000000BE8E83  c_esmc_vmabort_          1190  ESMCI_VM_F.C
ufs_model.x        000000000054D279  esmf_vmmod_mp_esm        9431  ESMF_VM.F90
ufs_model.x        00000000006CEFCF  esmf_initmod_mp_e        1226  ESMF_Init.F90
ufs_model.x        000000000042B7B0  MAIN__                    403  UFS.F90
ufs_model.x        000000000042A292  Unknown               Unknown  Unknown
libc-2.31.so       00001456DA69124D  __libc_start_main     Unknown  Unknown

Full log file: /lfs/h2/emc/couple/noscrub/jessica.meupdatemodel/s2swc48t03/COMROOT/s2swc48t03/logs/2021032312/gfsfcst.log.0

It does seem that GOCART has a problem with this, unless I'm missing something. At this point I'm ready to do a fresh round of low res testing + a high res spot check and open a new PR to update the model. But this seems to likely be a sticking point. I'm hoping someone who works on the aerosols component can chime in on this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now, comment it out and add a GOCART_out() to match the others that copies the files to COM_CHEM_HISTORY at the end of the forecast.

I'd also like to know what changed that this no longer works and if there is anyone working to change it back.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The error from: /scratch1/NCEPDEV/climate/Jessica.Meixner/HR3/updatemodel02/s2swc48t02/COMROOT/s2swc48t02/logs/2021032312/gfsfcst.log.0
is:

SUB GOCART_out: Copying output data for GOCART
+ forecast_postdet.sh[1052]: for fhr in '${FV3_OUTPUT_FH}'
++ forecast_postdet.sh[1053]: date --utc -d '20210323 12 + 0 hours' +%Y%m%d%H
+ forecast_postdet.sh[1053]: local vdate=2021032312
+ forecast_postdet.sh[1054]: /bin/cp -p /scratch1/NCEPDEV/climate/Jessica.Meixner/HR3/updatemodel02/s2swc48t02/RUNDIRS/s2swc48t02/fcst.123448/gocart.inst_aod.20210323_1200z.nc4 /scratch1/NCEPDEV/climate/Jessica.Meixner/HR3/updatemodel02/s2swc48t02/COMROOT/s2swc48t02/gfs.20210323/12//model_data/chem/history/gocart.inst_aod.20210323_1200z.nc4
/bin/cp: cannot stat '/scratch1/NCEPDEV/climate/Jessica.Meixner/HR3/updatemodel02/s2swc48t02/RUNDIRS/s2swc48t02/fcst.123448/gocart.inst_aod.20210323_1200z.nc4': No such file or directory

This was the code: JessicaMeixner-NOAA@6a61d8a

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand wanting to copy explicit lists but we should really be general as possible here as the inst_aod is just one of the output files that are possible.

It would really be better if we copied or linked all of the gocart.*.nc4 files to the chem directory as there are lots of possible diagnostics available

We can't link them or the run dies. If what I'm trying now works, we could try to make it slightly more general as long as it doesn't conflict with other linking statements.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you need to skip the f000 one.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There might be something in there that we just need to add to the AERO_HISTORY.rc file.

It needs to be added at the top of the file

Allow_Overwrite: true

Testing now

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How'd your tests go @bbakernoaa ? Mine did not go well. The changes I tried are here: https://github.com/JessicaMeixner-NOAA/global-workflow/tree/trygocartfix with the allow overwrite and back tracking the other changes.

I also haven't had good luck copying files at the end of the run, I keep getting errors, that branch is here: https://github.com/JessicaMeixner-NOAA/global-workflow/tree/updateUFS101223

# "${DATA}/gocart.inst_aod.${vdate:0:8}_${vdate:8:2}00z.nc4"
done
}
1 change: 1 addition & 0 deletions ush/nems_configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ if [[ "${cplflx}" = ".true." ]]; then
local coupling_interval_fast_sec="${CPL_FAST}"
local RESTART_N="${restart_interval}"
local ATMTILESIZE="${CASE:2:}"
local ocean_albedo_limit=0.06
fi

if [[ "${cplice}" = ".true." ]]; then
Expand Down
12 changes: 0 additions & 12 deletions ush/parsing_namelists_FV3.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,18 +74,6 @@ cat > input.nml <<EOF
${diag_manager_nml:-}
/

&fms_io_nml
checksum_required = .false.
max_files_r = 100
max_files_w = 100
${fms_io_nml:-}
/

&mpp_io_nml
shuffle=${shuffle:-1}
deflate_level=${deflate_level:-1}
/

&fms_nml
clock_grain = 'ROUTINE'
domains_stack_size = ${domains_stack_size:-3000000}
Expand Down
8 changes: 4 additions & 4 deletions versions/build.wcoss2.ver
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
export PrgEnv_intel_ver=8.1.0
export intel_ver=19.1.3.304
export craype_ver=2.7.13
export cray_mpich_ver=8.1.9
export cray_mpich_ver=8.1.12

export cmake_ver=3.20.2

Expand All @@ -14,8 +14,8 @@ export esmf_ver=8.3.0b09
export pio_ver=2.5.2
export fms_ver=2022.04

export hdf5_ver=1.10.6
export netcdf_ver=4.7.4
export hdf5_ver=1.14.0
export netcdf_ver=4.9.2

export bacio_ver=2.4.1
export w3nco_ver=2.4.1
Expand All @@ -29,7 +29,7 @@ export ip_ver=3.3.3
export wrf_io_ver=1.2.0
export ncio_ver=1.1.2
export ncdiag_ver=1.0.0
export g2tmpl_ver=1.10.0
export g2tmpl_ver=1.10.2
export crtm_ver=2.4.0
export wgrib2_ver=2.0.8

Expand Down
10 changes: 5 additions & 5 deletions versions/run.wcoss2.ver
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export prod_util_ver=${prod_util_ver:-2.0.9} # Allow override from ops ecflow
export PrgEnv_intel_ver=8.1.0
export intel_ver=19.1.3.304
export craype_ver=2.7.13
export cray_mpich_ver=8.1.9
export cray_mpich_ver=8.1.12
export cray_pals_ver=1.0.17
export cfp_ver=2.0.4

Expand All @@ -28,14 +28,14 @@ export zlib_ver=1.2.11
export libpng_ver=1.6.37
export cdo_ver=1.9.8

export hdf5_ver=1.10.6
export netcdf_ver=4.7.4
export hdf5_ver=1.14.0
export netcdf_ver=4.9.2

export nco_ver=4.7.9
export nco_ver=5.0.6
export grib_util_ver=1.2.3
export bufr_dump_ver=1.0.0
export util_shared_ver=1.4.0
export g2tmpl_ver=1.10.0
export g2tmpl_ver=1.10.2
export ncdiag_ver=1.0.0
export crtm_ver=2.4.0
export wgrib2_ver=2.0.8
Expand Down