Update submodules and use spack-stack FMS (#1221)

Update JEDI submodules to the latest develop as of today (7/12/2024) and remove FMS and MOM6 from the submodule list and ecbuild_bundle. FMS is now assumed to be a library loaded as a module and MOM6 is inside of SOCA already. Code compiles on Hera and WCOSS, have not tried to run any tests yet. --------- Co-authored-by: cory martin <[email protected]> Co-authored-by: Rahul Mahajan <[email protected]> Co-authored-by: RussTreadon-NOAA <[email protected]>
NOAA-EMC · Jul 17, 2024 · ca18845 · ca18845
1 parent b2ca01e
commit ca18845
Show file tree

Hide file tree

Showing 24 changed files with 64 additions and 38 deletions.
diff --git a/.gitignore b/.gitignore
@@ -183,4 +183,5 @@ install*/
 /sorc/gdas
 /sorc/gdas-utils
 !/bundle/CMakeLists.txt
+!/bundle/fv3-interface.cmake
 /bundle/*
diff --git a/.gitmodules b/.gitmodules
@@ -49,10 +49,6 @@
   path = sorc/icepack
   url = https://github.com/JCSDA-internal/Icepack.git
   branch = feature/ecbuild-new
-[submodule "sorc/mom6"]
-  path = sorc/mom6
-  url = https://github.com/jcsda-internal/MOM6.git
-  branch = main-ecbuild
 [submodule "sorc/soca"]
   path = sorc/soca
   url = https://github.com/jcsda-internal/soca.git
@@ -73,10 +69,6 @@
   path = sorc/crtm
   url = https://github.com/jcsda/crtm.git
   branch = release/crtm_jedi_v2.4.1
-[submodule "sorc/fms"]
-  path = sorc/fms
-  url = https://github.com/jcsda/fms.git
-  branch = release-stable
 [submodule "sorc/jcb"]
   path = sorc/jcb
   url = https://github.com/noaa-emc/jcb.git

diff --git a/bundle/CMakeLists.txt b/bundle/CMakeLists.txt
@@ -36,6 +36,19 @@ option(BUILD_GDASBUNDLE "Build GDAS Bundle" ON)
 option(CLONE_JCSDADATA "Clone JCSDA test data repositories" OFF)
 option(WORKFLOW_TESTS "Include global-workflow dependent tests" OFF)
 
+# Depend path for non-ecbuild packages
+set(DEPEND_LIB_ROOT ${CMAKE_CURRENT_BINARY_DIR}/Depends)
+list(APPEND CMAKE_PREFIX_PATH ${DEPEND_LIB_ROOT})
+
+# Library path for non-ecbuild packages
+link_directories(${CMAKE_CURRENT_BINARY_DIR}/lib)
+
+include( GNUInstallDirs )
+if(APPLE)
+  list( APPEND CMAKE_INSTALL_RPATH $ENV{llvm_openmp_ROOT}/lib )
+endif()
+list( APPEND CMAKE_INSTALL_RPATH ${CMAKE_CURRENT_BINARY_DIR}/fv3 )
+
 # Initialize bundle
 # -----------------
 ecbuild_bundle_initialize()
@@ -84,8 +97,8 @@ if(BUILD_GDASBUNDLE)
   option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
   ecbuild_bundle( PROJECT ufo SOURCE "../sorc/ufo" )
 
-# FMS and FV3 dynamical core
-  ecbuild_bundle( PROJECT fms SOURCE "../sorc/fms" )
+# FV3 dynamical core
+  include(fv3-interface.cmake)
   ecbuild_bundle( PROJECT fv3 SOURCE "../sorc/fv3" )
 
 # fv3-jedi and associated repositories
@@ -100,7 +113,6 @@ if(BUILD_GDASBUNDLE)
   if ( BUILD_ICEPACK )
     ecbuild_bundle( PROJECT icepack SOURCE "../sorc/icepack" )
   endif()
-  ecbuild_bundle( PROJECT mom6 SOURCE "../sorc/mom6" )
   ecbuild_bundle( PROJECT soca SOURCE "../sorc/soca" )
 
   # Build JEDI/DA or other peripherals

diff --git a/bundle/fv3-interface.cmake b/bundle/fv3-interface.cmake
@@ -0,0 +1,18 @@
+# FV3 dycore
+set(OPENMP                ON )
+set(32BIT                 OFF)
+set(DEBUG                 OFF)
+set(MOVING_NEST           OFF)
+set(MULTI_GASES           OFF)
+set(USE_GFSL63            ON )
+set(NO_PHYS               ON )
+set(GFS_PHYS              OFF)
+set(GFS_TYPES             OFF)
+set(use_WRTCOMP           OFF)
+set(INTERNAL_FILE_NML     ON )
+set(ENABLE_QUAD_PRECISION ON )
+
+add_library(fv3dycore IMPORTED SHARED)
+set_target_properties(fv3dycore PROPERTIES IMPORTED_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/fv3/libfv3.${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+message(WARN "WE ARE USING fv3-interface.cmake")
diff --git a/modulefiles/GDAS/gaea.intel.lua b/modulefiles/GDAS/gaea.intel.lua
@@ -43,6 +43,7 @@ load("fftw/3.3.10")
 load("fckit/0.11.0")
 load("fiat/1.2.0")
 load("ectrans/1.2.0")
+load("fms/2023.04")
 load("atlas/0.35.1")
 load("sp/2.5.0")
 load("gsl-lite/0.37.0")

diff --git a/modulefiles/GDAS/hera.intel.lua b/modulefiles/GDAS/hera.intel.lua
@@ -43,6 +43,7 @@ load("fftw/3.3.10")
 load("fckit/0.11.0")
 load("fiat/1.2.0")
 --load("ectrans/1.2.0")
+load("fms/2023.04")
 load("atlas/0.35.1")
 load("sp/2.5.0")
 load("gsl-lite/0.37.0")

diff --git a/modulefiles/GDAS/hercules.intel.lua b/modulefiles/GDAS/hercules.intel.lua
@@ -40,6 +40,7 @@ load("fftw/3.3.10")
 load("fckit/0.11.0")
 load("fiat/1.2.0")
 load("ectrans/1.2.0")
+load("fms/2023.04")
 load("atlas/0.35.1")
 load("sp/2.5.0")
 load("gsl-lite/0.37.0")

diff --git a/modulefiles/GDAS/noaacloud.intel.lua b/modulefiles/GDAS/noaacloud.intel.lua
@@ -41,6 +41,7 @@ load("fftw/3.3.10")
 load("fckit/0.11.0")
 load("fiat/1.2.0")
 load("ectrans/1.2.0")
+load("fms/2023.04")
 load("atlas/0.35.1")
 load("sp/2.5.0")
 load("gsl-lite/0.37.0")

diff --git a/modulefiles/GDAS/orion.intel.lua b/modulefiles/GDAS/orion.intel.lua
@@ -40,6 +40,7 @@ load("fftw/3.3.10")
 load("fckit/0.11.0")
 load("fiat/1.2.0")
 load("ectrans/1.2.0")
+load("fms/2023.04")
 load("atlas/0.35.1")
 load("sp/2.5.0")
 load("gsl-lite/0.37.0")

diff --git a/modulefiles/GDAS/wcoss2.intel.lua b/modulefiles/GDAS/wcoss2.intel.lua
@@ -33,13 +33,17 @@ load("nco/5.0.6")
 load("gsl/2.7")
 load("prod_util/2.0.14")
 load("bufr/12.0.1")
+load("fms-C/2023.04")
 
 -- hack for pybind11
 setenv("pybind11_ROOT", "/apps/spack/python/3.8.6/intel/19.1.3.304/pjn2nzkjvqgmjw4hmyz43v5x4jbxjzpk/lib/python3.8/site-packages/pybind11/share/cmake/pybind11")
 
 -- hack for git-lfs
 prepend_path("PATH", "/apps/spack/git-lfs/2.11.0/gcc/11.2.0/m6b6nl5kfqngfteqbggydc7kflxere3s/bin")
 
+-- hack for FMS
+setenv('fms_ROOT', '/apps/prod/hpc-stack/i-19.1.3.304__m-8.1.12__h-1.14.0__n-4.9.2__p-2.5.10__e-8.6.0pnetcdf/intel-19.1.3.304/cray-mpich-8.1.12/fms/2023.04')
+
 local mpiexec = '/pe/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/bin/mpirun'
 local mpinproc = '-n'
 setenv('MPIEXEC_EXEC', mpiexec)

diff --git a/sorc/fms b/sorc/fms
diff --git a/sorc/fv3 b/sorc/fv3
diff --git a/sorc/fv3-jedi b/sorc/fv3-jedi
diff --git a/sorc/fv3-jedi-lm b/sorc/fv3-jedi-lm
diff --git a/sorc/ioda b/sorc/ioda
diff --git a/sorc/iodaconv b/sorc/iodaconv
diff --git a/sorc/land-imsproc b/sorc/land-imsproc
diff --git a/sorc/mom6 b/sorc/mom6
diff --git a/sorc/oops b/sorc/oops
diff --git a/sorc/saber b/sorc/saber
diff --git a/sorc/soca b/sorc/soca
diff --git a/sorc/ufo b/sorc/ufo
diff --git a/sorc/vader b/sorc/vader
diff --git a/test/atm/global-workflow/3dvar.ref b/test/atm/global-workflow/3dvar.ref
@@ -1,8 +1,8 @@
 CostJb   : Nonlinear Jb = 0.0000000000000000e+00
-CostJo   : Nonlinear Jo(AMSUA N19) = 3.9490015258330794e+04, nobs = 73667, Jo/n = 5.3606112992697941e-01, err = 6.2697697603629345e+00
-CostJo   : Nonlinear Jo(sondes) = 1.0738490911289638e+04, nobs = 4255, Jo/n = 2.5237346442513839e+00, err = 1.0981085630539836e+01
-CostFunction: Nonlinear J = 5.0228506169620436e+04
-DRPCGMinimizer: reduction in residual norm = 9.6305250910882700e-01
+CostJo   : Nonlinear Jo(AMSUA N19) = 3.9490015258335385e+04, nobs = 73667, Jo/n = 5.3606112992704180e-01, err = 6.2697697603632934e+00
+CostJo   : Nonlinear Jo(sondes) = 1.0738490911289642e+04, nobs = 4255, Jo/n = 2.5237346442513848e+00, err = 1.0981085630539836e+01
+CostFunction: Nonlinear J = 5.0228506169625027e+04
+DRPCGMinimizer: reduction in residual norm = 9.6305250910880502e-01
 CostFunction::addIncrement: Analysis: 
 ----------------------------------------------------------------------------------------------------
 State print | number of fields = 22 | cube sphere face size: C48
@@ -11,7 +11,7 @@ northward_wind                               | Min:-7.3712421229093422e+01 Max:+
 air_temperature                              | Min:+1.7482158077318448e+02 Max:+3.1467235233685437e+02 RMS:+2.4978424883092669e+02
 air_pressure_thickness                       | Min:+6.0599999874109178e-01 Max:+1.7760098554198755e+03 RMS:+9.5680601590204810e+02
 surface_pressure                             | Min:+5.3297486514404744e+04 Max:+1.0397927292902800e+05 RMS:+9.8896232762674656e+04
-specific_humidity                            | Min:+0.0000000000000000e+00 Max:+2.0041369334954085e-02 RMS:+4.7792636816418408e-03
+specific_humidity                            | Min:+0.0000000000000000e+00 Max:+2.0041369334954085e-02 RMS:+4.7792636816418417e-03
 cloud_liquid_ice                             | Min:+0.0000000000000000e+00 Max:+1.2024392024662985e-03 RMS:+1.4396968866569995e-05
 cloud_liquid_water                           | Min:+0.0000000000000000e+00 Max:+1.5935686618789048e-03 RMS:+4.1931345590543333e-05
 ozone_mass_mixing_ratio                      | Min:+1.1998327271379204e-08 Max:+1.7808431790670831e-05 RMS:+4.4947917149861064e-06
@@ -44,10 +44,10 @@ sensorScanAngle_order_2:  Min= -5.1097259521664853,  Max= 2.8632600308292808,  N
 
 
 CostJb   : Nonlinear Jb = 0.0000003247124839
-CostJo   : Nonlinear Jo(AMSUA N19) = 39487.6033905208023498, nobs = 73667, Jo/n = 0.5360283897881114, err = 6.2697697603629345
-CostJo   : Nonlinear Jo(sondes) = 10687.3021478507725988, nobs = 4255, Jo/n = 2.5117043825736247, err = 10.9810856305398357
-CostFunction: Nonlinear J = 50174.9055386962863849
-DRPCGMinimizer: reduction in residual norm = 1.0067413227955442
+CostJo   : Nonlinear Jo(AMSUA N19) = 39487.6033905211515957, nobs = 73667, Jo/n = 0.5360283897881162, err = 6.2697697603632934
+CostJo   : Nonlinear Jo(sondes) = 10687.3021478507798747, nobs = 4255, Jo/n = 2.5117043825736265, err = 10.9810856305398357
+CostFunction: Nonlinear J = 50174.9055386966429069
+DRPCGMinimizer: reduction in residual norm = 1.0067413227975508
 CostFunction::addIncrement: Analysis: 
 ----------------------------------------------------------------------------------------------------
 State print | number of fields = 22 | cube sphere face size: C48
@@ -56,7 +56,7 @@ northward_wind                               | Min:-7.3712421229093422e+01 Max:+
 air_temperature                              | Min:+1.7482158077318232e+02 Max:+3.1467235233685437e+02 RMS:+2.4978424883219918e+02
 air_pressure_thickness                       | Min:+6.0599999874109178e-01 Max:+1.7760098554198755e+03 RMS:+9.5680601590204810e+02
 surface_pressure                             | Min:+5.3297486514404744e+04 Max:+1.0397927292902800e+05 RMS:+9.8896232762674656e+04
-specific_humidity                            | Min:+0.0000000000000000e+00 Max:+2.0041369334954085e-02 RMS:+4.7792637405692116e-03
+specific_humidity                            | Min:+0.0000000000000000e+00 Max:+2.0041369334954085e-02 RMS:+4.7792637405692090e-03
 cloud_liquid_ice                             | Min:+0.0000000000000000e+00 Max:+1.2024392024662985e-03 RMS:+1.4396968866569995e-05
 cloud_liquid_water                           | Min:+0.0000000000000000e+00 Max:+1.5935686618789048e-03 RMS:+4.1931345590543333e-05
 ozone_mass_mixing_ratio                      | Min:+1.1998327271379204e-08 Max:+1.7808431790670831e-05 RMS:+4.4947917149861064e-06
@@ -88,7 +88,3 @@ sensorScanAngle_order_2:  Min= -5.1097259522434877,  Max= 2.8632600312042698,  N
 ---------------------------------------------------------------
 
 
-CostJb   : Nonlinear Jb = 0.0000033199535370
-CostJo   : Nonlinear Jo(AMSUA N19) = 39476.8510820809169672, nobs = 73667, Jo/n = 0.5358824315104581, err = 6.2697697603629345
-CostJo   : Nonlinear Jo(sondes) = 10639.1482810630404856, nobs = 4255, Jo/n = 2.5003873751029473, err = 10.9810856305398357
-CostFunction: Nonlinear J = 50115.9993664639114286