From 75dc7da5767c7e61b947c162888e271ad403abfc Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Tue, 19 Dec 2023 19:40:47 +0000 Subject: [PATCH 01/10] ww3_shel.inp: remove output parameter "EF" --- regtests/ww3_tp2.22/input/ww3_shel.inp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regtests/ww3_tp2.22/input/ww3_shel.inp b/regtests/ww3_tp2.22/input/ww3_shel.inp index 6b946943c..8a32e5584 100644 --- a/regtests/ww3_tp2.22/input/ww3_shel.inp +++ b/regtests/ww3_tp2.22/input/ww3_shel.inp @@ -18,7 +18,7 @@ $ 19680606 000000 3600 19680608 000000 $ N - HS EF T01 DIR SPR USS USSH + HS T01 DIR SPR USS USSH $ 19680606 000000 21600 19680608 000000 0.0 0.0 'LEFT' From ef8e0389408daf0aaf99c29a4c98bb2c655aa172 Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Tue, 19 Dec 2023 19:41:30 +0000 Subject: [PATCH 02/10] ww3_outf.inp: remove output parameter "EF" --- regtests/ww3_tp2.22/input/ww3_outf.inp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regtests/ww3_tp2.22/input/ww3_outf.inp b/regtests/ww3_tp2.22/input/ww3_outf.inp index 3bc3e4908..2ee125996 100644 --- a/regtests/ww3_tp2.22/input/ww3_outf.inp +++ b/regtests/ww3_tp2.22/input/ww3_outf.inp @@ -3,7 +3,7 @@ $ ----------------------------------------- 19680606 000000 3600. 12 $ N - HS EF T01 DIR SPR USS USSH + HS T01 DIR SPR USS USSH $ 1 0 $ From 586450e0ddc439c80e833c46d717b75be9bf890a Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Tue, 19 Dec 2023 19:42:48 +0000 Subject: [PATCH 03/10] ww3_ounf.inp: remove output parameter "EF" --- regtests/ww3_tp2.22/input/ww3_ounf.inp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regtests/ww3_tp2.22/input/ww3_ounf.inp b/regtests/ww3_tp2.22/input/ww3_ounf.inp index 37703605d..6d1998f36 100644 --- a/regtests/ww3_tp2.22/input/ww3_ounf.inp +++ b/regtests/ww3_tp2.22/input/ww3_ounf.inp @@ -13,7 +13,7 @@ $ file for a full documentation of field output options. Namelist type $ selection is used here (for alternative F/T flags, see ww3_shel.inp). $ N - HS EF T01 DIR SPR USS USSH + HS T01 DIR SPR USS USSH $ $--------------------------------------------------------------------- $ $ netCDF version [3,4] From 4ffc47e10e3d3f3bbee50251aacb28b7e0165b92 Mon Sep 17 00:00:00 2001 From: Denise Worthen Date: Fri, 19 Jan 2024 13:25:20 -0500 Subject: [PATCH 04/10] prevent division by 0 in appendtail and add timers to meshcap (#1163) --- model/src/cmake/src_list.cmake | 1 + model/src/w3fld1md.F90 | 18 ++++- model/src/wav_comp_nuopc.F90 | 46 +++++++++++-- model/src/wav_wrapper_mod.F90 | 119 +++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 model/src/wav_wrapper_mod.F90 diff --git a/model/src/cmake/src_list.cmake b/model/src/cmake/src_list.cmake index 2152b697e..dcab88a09 100644 --- a/model/src/cmake/src_list.cmake +++ b/model/src/cmake/src_list.cmake @@ -66,6 +66,7 @@ set(nuopc_mesh_cap_src wav_shel_inp.F90 wav_comp_nuopc.F90 wav_import_export.F90 + wav_wrapper_mod.F90 ) set(esmf_multi_cap_src diff --git a/model/src/w3fld1md.F90 b/model/src/w3fld1md.F90 index 960fd185a..ad94a12ea 100644 --- a/model/src/w3fld1md.F90 +++ b/model/src/w3fld1md.F90 @@ -1116,7 +1116,11 @@ SUBROUTINE APPENDTAIL(INSPC, WN2, NKT, KA1, KA2, KA3, WNDDIR,SAT) DO K=KA1, KA2-1 AVG=SUM(INSPC(K,:))/MAX(REAL(NTH),1.) DO T=1,NTH - INSPC(K,T)=BT(K)*INSPC(K,T)/TPI/(WN2(K)**3.0)/AVG + if (avg /= 0.0) then + INSPC(K,T)=BT(K)*INSPC(K,T)/TPI/(WN2(K)**3.0)/AVG + else + inspc(k,t) = 0.0 + end if ENDDO ENDDO !----------------------------------------------------------- @@ -1134,7 +1138,11 @@ SUBROUTINE APPENDTAIL(INSPC, WN2, NKT, KA1, KA2, KA3, WNDDIR,SAT) ENDDO AVG=SUM(NORMSPC)/MAX(REAL(NTH),1.) DO T=1, NTH - INSPC(K,T) = SAT * NORMSPC(T)/TPI/(WN2(K)**3.0)/AVG + if (avg /= 0.0) then + INSPC(K,T) = SAT * NORMSPC(T)/TPI/(WN2(K)**3.0)/AVG + else + inspc(k,t) = 0.0 + end if ENDDO ENDDO DO T=1, NTH @@ -1148,7 +1156,11 @@ SUBROUTINE APPENDTAIL(INSPC, WN2, NKT, KA1, KA2, KA3, WNDDIR,SAT) AVG=SUM(NORMSPC)/MAX(REAL(NTH),1.)!1./4. DO K=KA3+1, NKT DO T=1, NTH - INSPC(K,T)=NORMSPC(T)*(SAT)/TPI/(WN2(K)**3.0)/AVG + if (avg /= 0.0) then + INSPC(K,T)=NORMSPC(T)*(SAT)/TPI/(WN2(K)**3.0)/AVG + else + inspc(k,t) = 0.0 + end if ENDDO ENDDO DEALLOCATE(ANGLE1) diff --git a/model/src/wav_comp_nuopc.F90 b/model/src/wav_comp_nuopc.F90 index cec62b55d..6f3eeef5a 100644 --- a/model/src/wav_comp_nuopc.F90 +++ b/model/src/wav_comp_nuopc.F90 @@ -48,6 +48,7 @@ module wav_comp_nuopc use w3odatmd , only : user_netcdf_grdout use w3odatmd , only : time_origin, calendar_name, elapsed_secs use wav_shr_mod , only : casename, multigrid, inst_suffix, inst_index, unstr_mesh + use wav_wrapper_mod , only : ufs_settimer, ufs_logtimer, ufs_file_setlogunit, wtime #ifndef W3_CESMCOUPLED use wmwavemd , only : wmwave use wmupdtmd , only : wmupd2 @@ -99,9 +100,12 @@ module wav_comp_nuopc !! using ESMF. If restart_option is present as config !! option, user_restalarm will be true and will be !! set using restart_option, restart_n and restart_ymd - integer :: time0(2) - integer :: timen(2) - + integer :: ymd !< current year-month-day + integer :: tod !< current time of day (sec) + integer :: time0(2) !< start time stored as yyyymmdd,hhmmss + integer :: timen(2) !< end time stored as yyyymmdd,hhmmss + integer :: nu_timer !< simple timer log, unused except by UFS + logical :: runtimelog = .false. !< logical flag for writing runtime log files character(*), parameter :: modName = "(wav_comp_nuopc)" !< the name of this module character(*), parameter :: u_FILE_u = & !< a character string for an ESMF log message __FILE__ @@ -238,6 +242,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) character(len=*), parameter :: subname=trim(modName)//':(InitializeAdvertise) ' !------------------------------------------------------------------------------- + call ufs_settimer(wtime) rc = ESMF_SUCCESS call ESMF_LogWrite(trim(subname)//' called', ESMF_LOGMSG_INFO) @@ -369,6 +374,15 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) write(logmsg,'(A,l)') trim(subname)//': Wave wav_coupling_to_cice setting is ',wav_coupling_to_cice call ESMF_LogWrite(trim(logmsg), ESMF_LOGMSG_INFO) + ! Determine Runtime logging + call NUOPC_CompAttributeGet(gcomp, name="RunTimeLog", value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) runtimelog=(trim(cvalue)=="true") + write(logmsg,*) runtimelog + call ESMF_LogWrite('WW3_cap:RunTimeLog = '//trim(logmsg), ESMF_LOGMSG_INFO) + if (runtimelog) then + call ufs_file_setLogUnit('./log.ww3.timer',nu_timer,runtimelog) + end if call advertise_fields(importState, exportState, flds_scalar_name, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -475,6 +489,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) rc = ESMF_SUCCESS if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' called', ESMF_LOGMSG_INFO) + call ufs_settimer(wtime) !-------------------------------------------------------------------- ! Set up data structures !-------------------------------------------------------------------- @@ -871,6 +886,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) enddo end if #endif + if (root_task) call ufs_logtimer(nu_timer,time,start_tod,'InitializeRealize time: ',runtimelog,wtime) if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) @@ -1000,8 +1016,6 @@ subroutine ModelAdvance(gcomp, rc) type(ESMF_Time) :: currTime, nextTime, startTime, stopTime integer :: yy,mm,dd,hh,ss integer :: imod - integer :: ymd ! current year-month-day - integer :: tod ! current time of day (sec) integer :: shrlogunit ! original log unit and level character(ESMF_MAXSTR) :: msgString character(len=*),parameter :: subname = '(wav_comp_nuopc:ModelAdvance) ' @@ -1041,6 +1055,8 @@ subroutine ModelAdvance(gcomp, rc) if ( root_task ) then write(nds(1),'(a,3i4,i10)') 'ymd2date currTime wav_comp_nuopc hh,mm,ss,ymd', hh,mm,ss,ymd end if + if (root_task) call ufs_logtimer(nu_timer,time,tod,'ModelAdvance time since last step: ',runtimelog,wtime) + call ufs_settimer(wtime) ! use next time; the NUOPC clock is not updated ! until the end of the time interval @@ -1138,6 +1154,8 @@ subroutine ModelAdvance(gcomp, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) + if (root_task) call ufs_logtimer(nu_timer,time,tod,'ModelAdvance time: ',runtimelog,wtime) + call ufs_settimer(wtime) end subroutine ModelAdvance @@ -1357,6 +1375,7 @@ subroutine ModelFinalize(gcomp, rc) end if call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) + if(root_task) call ufs_logtimer(nu_timer,timen,tod,'ModelFinalize time: ',runtimelog,wtime) end subroutine ModelFinalize @@ -1575,6 +1594,7 @@ subroutine waveinit_ufs( gcomp, ntrace, mpi_comm, mds, rc) ! Initialize ww3 for ufs (called from InitializeRealize) use w3odatmd , only : fnmpre + use w3gdatmd , only : dtcfl, dtcfli, dtmax, dtmin use w3initmd , only : w3init use wav_shel_inp , only : read_shel_config use wav_shel_inp , only : npts, odat, iprt, x, y, pnames, prtfrm @@ -1591,6 +1611,7 @@ subroutine waveinit_ufs( gcomp, ntrace, mpi_comm, mds, rc) character(len=CL) :: logmsg logical :: isPresent, isSet character(len=CL) :: cvalue + integer :: dt_in(4) character(len=*), parameter :: subname = '(wav_comp_nuopc:wavinit_ufs)' ! ------------------------------------------------------------------- @@ -1638,6 +1659,21 @@ subroutine waveinit_ufs( gcomp, ntrace, mpi_comm, mds, rc) call w3init ( 1, .false., 'ww3', mds, ntrace, odat, flgrd, flgr2, flgd, flg2, & npts, x, y, pnames, iprt, prtfrm, mpi_comm ) + write(logmsg,'(A,4f10.2)') trim(subname)//': mod_def timesteps file ',dtmax,dtcfl,dtcfli,dtmin + call ESMF_LogWrite(trim(logmsg), ESMF_LOGMSG_INFO) + call NUOPC_CompAttributeGet(gcomp, name='dt_in', isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + call NUOPC_CompAttributeGet(gcomp, name='dt_in', value=cvalue, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + read(cvalue,*)dt_in + dtmax = real(dt_in(1),4) + dtcfl = real(dt_in(2),4) + dtcfli = real(dt_in(3),4) + dtmin = real(dt_in(4),4) + write(logmsg,'(A,4f10.2)') trim(subname)//': mod_def timesteps reset ',dtmax,dtcfl,dtcfli,dtmin + call ESMF_LogWrite(trim(logmsg), ESMF_LOGMSG_INFO) + end if if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) end subroutine waveinit_ufs diff --git a/model/src/wav_wrapper_mod.F90 b/model/src/wav_wrapper_mod.F90 new file mode 100644 index 000000000..dd2465829 --- /dev/null +++ b/model/src/wav_wrapper_mod.F90 @@ -0,0 +1,119 @@ +!> @file wav_wrapper_mod +!! +!> A wrapper module for log functionality in UFS +!! +!> @details Contains public logging routines for UFS and +!! stub routines for CESM +!! +!> Denise.Worthen@noaa.gov +!> @date 01-08-2024 +module wav_wrapper_mod + + use wav_kind_mod , only : r8 => shr_kind_r8, r4 => shr_kind_r4, i4 => shr_kind_i4 + use wav_kind_mod , only : CL => shr_kind_cl, CS => shr_kind_cs + + implicit none + + real(r8) :: wtime = 0.0 + +#ifdef CESMCOUPLED +contains + ! Define stub routines that do nothing - they are just here to avoid + ! having cppdefs in the main program + subroutine ufs_settimer(timevalue) + real(r8), intent(inout) :: timevalue + end subroutine ufs_settimer + subroutine ufs_logtimer(nunit,times,tod,string,runtimelog,wtime0) + integer, intent(in) :: nunit + integer(i4), intent(in) :: times(2), tod + character(len=*), intent(in) :: string + logical, intent(in) :: runtimelog + real(r8), intent(in) :: wtime0 + end subroutine ufs_logtimer + subroutine ufs_file_setLogUnit(filename,nunit,runtimelog) + character(len=*), intent(in) :: filename + logical, intent(in) :: runtimelog + integer, intent(out) :: nunit + end subroutine ufs_file_setLogUnit + subroutine ufs_logfhour(msg,hour) + character(len=*), intent(in) :: msg + real(r8), intent(in) :: hour + end subroutine ufs_logfhour +#else +contains + subroutine ufs_settimer(timevalue) + !> Set a time value + !! @param[inout] timevalue a MPI time value + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + + real(r8), intent(inout) :: timevalue + real(r8) :: MPI_Wtime + timevalue = MPI_Wtime() + end subroutine ufs_settimer + + subroutine ufs_logtimer(nunit,times,tod,string,runtimelog,wtime0) + !> Log a time interval + !! @param[in] nunit the log file unit + !! @param[in] times the ymd,hms time values + !! @param[in] tod the elapsed seconds in the day + !! @param[in] string a message string to log + !! @param[in] runtimelog a logical to control the log function + !! @param[in] wtime0 an initial MPI time + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + integer, intent(in) :: nunit + integer(i4), intent(in) :: times(2),tod + character(len=*), intent(in) :: string + logical, intent(in) :: runtimelog + real(r8), intent(in) :: wtime0 + real(r8) :: MPI_Wtime, timevalue + if (.not. runtimelog) return + if (wtime0 > 0.) then + timevalue = MPI_Wtime()-wtime0 + write(nunit,'(3i8,a,g14.7)')times,tod,' WW3 '//trim(string),timevalue + end if + end subroutine ufs_logtimer + + subroutine ufs_file_setLogUnit(filename,nunit,runtimelog) + !> Create a log unit + !! @param[in] filename the log filename + !! @param[in] runtimelog a logical to control the log function + !! @param[out] nunit the log file unit + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + + character(len=*), intent(in) :: filename + logical, intent(in) :: runtimelog + integer, intent(out) :: nunit + if (.not. runtimelog) return + open (newunit=nunit, file=trim(filename)) + end subroutine ufs_file_setLogUnit + + subroutine ufs_logfhour(msg,hour) + !> Log the completion of model output + !! @param[in] msg the log message + !! @param[in] hour the forecast hour + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + + character(len=*), intent(in) :: msg + real(r8), intent(in) :: hour + + character(len=CS) :: filename + integer(r4) :: nunit + + write(filename,'(a,i3.3)')'log.ww3.f',int(hour) + open(newunit=nunit,file=trim(filename)) + write(nunit,'(a)')'completed: ww3' + write(nunit,'(a,f10.3)')'forecast hour:',hour + write(nunit,'(a)')'valid time: '//trim(msg) + close(nunit) + end subroutine ufs_logfhour +#endif + +end module wav_wrapper_mod From ea361a58635c4cfe22065571c44abcb7dc249b9b Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Thu, 22 Feb 2024 19:10:13 +0000 Subject: [PATCH 05/10] matrix_cmake_ncep: update modules for spack-stack --- regtests/bin/matrix_cmake_ncep | 57 +++++++++++++++------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep index 771b6f96e..7d0d26bec 100755 --- a/regtests/bin/matrix_cmake_ncep +++ b/regtests/bin/matrix_cmake_ncep @@ -41,16 +41,18 @@ EOF # Convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" -# Module Versions from HPC-Stack that are common for all platforms - modnetcdf='netcdf/4.7.4' - modjasper='jasper/2.0.25' - modzlib='zlib/1.2.11' +# Module Versions from spack-stack that are common for all platforms + modnetcdfc='netcdf-c/4.9.2' + modnetcdff='netcdf-fortran/4.6.0' + modjasper='jasper/2.0.32' + modzlib='zlib/1.2.13' modpng='libpng/1.6.37' - modhdf5='hdf5/1.10.6' + modhdf5='hdf5/1.14.0' modbacio='bacio/2.4.1' modg2='g2/3.4.5' - modw3emc='w3emc/2.9.2' - modesmf='esmf/8.3.0b09' + modw3emc='w3emc/2.10.0' + modesmf='esmf/8.4.2' + modscotch='scotch/7.0.4' # Set batchq queue, choose modules and other custom variables to fit system and # to define headers etc (default to original version if empty) @@ -60,27 +62,19 @@ EOF then # If no other h, assuming Hera batchq='slurm' - basemodcomp='intel/2022.1.2' - basemodmpi='impi/2022.1.2' - hpcstackpath='/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack' - hpcstackversion='hpc/1.2.0' - modcomp='hpc-intel/2022.1.2' - modmpi='hpc-impi/2022.1.2' - scotchpath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hpc-stack/scotch-v7.0.3/install' - metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hpc-stack/parmetis-4.0.3/install' - modcmake='cmake/3.20.1' + spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core' + modcomp='stack-intel/2021.5.0' + modmpi='stack-intel-oneapi-mpi/2021.5.1' + metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' elif [ $isorion ] then batchq='slurm' - basemodcomp='intel/2022.1.2' - basemodmpi='impi/2022.1.2' - hpcstackpath='/work/noaa/epic-ps/role-epic-ps/hpc-stack/libs/intel-2022.1.2/modulefiles/stack' - hpcstackversion='hpc/1.2.0' - modcomp='hpc-intel/2022.1.2' - modmpi='hpc-impi/2022.1.2' - scotchpath='/work2/noaa/marine/mmasarik/waves/opt/hpc-stack/scotch-v7.0.3/install' - metispath='/work2/noaa/marine/mmasarik/waves/opt/hpc-stack/parmetis-4.0.3/install' - modcmake='cmake/3.22.1' + spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core' + modcomp='stack-intel/2022.0.2' + modmpi='stack-intel-oneapi-mpi/2021.5.1' + metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install' + modcmake='cmake/3.23.1' else batchq= fi @@ -96,7 +90,7 @@ EOF # 1.a Computer/ user dependent set up - echo '#!/bin/sh --login' > matrix.head + echo '#!/bin/sh' > matrix.head echo ' ' >> matrix.head if [ $batchq = "slurm" ] && [ $isorion ] then @@ -139,29 +133,28 @@ EOF # Netcdf, Parmetis and SCOTCH modules & variables echo " module purge" >> matrix.head - echo " module load $modcmake" >> matrix.head if [ ! -z $basemodcomp ]; then echo " module load $basemodcomp" >> matrix.head fi if [ ! -z $basemodmpi ]; then echo " module load $basemodmpi" >> matrix.head fi - echo " module use $hpcstackpath" >> matrix.head - echo " module load $hpcstackversion" >> matrix.head + echo " module use $spackstackpath" >> matrix.head echo " module load $modcomp" >> matrix.head echo " module load $modmpi" >> matrix.head + echo " module load $modcmake" >> matrix.head echo " module load $modpng" >> matrix.head echo " module load $modzlib" >> matrix.head echo " module load $modjasper" >> matrix.head echo " module load $modhdf5" >> matrix.head - echo " module load $modnetcdf" >> matrix.head + echo " module load $modnetcdfc" >> matrix.head + echo " module load $modnetcdff" >> matrix.head echo " module load $modbacio" >> matrix.head echo " module load $modg2" >> matrix.head echo " module load $modw3emc" >> matrix.head echo " module load $modesmf" >> matrix.head - + echo " module load $modscotch" >> matrix.head echo " export METIS_PATH=${metispath}" >> matrix.head - echo " export SCOTCH_PATH=${scotchpath}" >> matrix.head echo " export path_build_root=$(dirname $main_dir)/regtests/buildmatrix" >> matrix.head echo ' [[ -d ${path_build_root} ]] && rm -rf ${path_build_root}' >> matrix.head echo ' ' From 3caad4e1f861a2df245f7ca999bda00b323f50b0 Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Fri, 23 Feb 2024 15:11:03 +0000 Subject: [PATCH 06/10] intel.yml: update cache key --- .github/workflows/intel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 700553cea..ef7f6721e 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -8,7 +8,7 @@ concurrency: # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran env: - cache_key: intel10 + cache_key: intel11 CC: icc FC: ifort CXX: icpc From 6e7b1adc2343a559db12e33030810f80f1f1bb1c Mon Sep 17 00:00:00 2001 From: Edward Hartnett <38856240+edwardhartnett@users.noreply.github.com> Date: Tue, 26 Dec 2023 13:16:55 -0700 Subject: [PATCH 07/10] Updated intel workflow to install oneapi compilers from new location. (#1157) --- .github/workflows/intel.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index ef7f6721e..54a388c55 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -1,3 +1,8 @@ +# This is a GitHub actions workflow for WW3. +# +# This workflow builds with the Intel compilers. +# +# Matt Masarik, Alex Richert, Ed Hartnett name: Intel Linux Build on: [push, pull_request, workflow_dispatch] @@ -8,7 +13,7 @@ concurrency: # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran env: - cache_key: intel11 + cache_key: intel10-3 CC: icc FC: ifort CXX: icpc @@ -51,7 +56,7 @@ jobs: sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1 intel-oneapi-openmp # Build WW3 spack environment - name: install-dependencies-with-spack From a10573cb44a2128c061280055f40031ab1baa284 Mon Sep 17 00:00:00 2001 From: Alex Richert <82525672+AlexanderRichert-NOAA@users.noreply.github.com> Date: Thu, 11 Jan 2024 14:12:05 -0500 Subject: [PATCH 08/10] Update Intel CI (relocate /usr/local; ensure intel-oneapi-mpi; use ubuntu-latest) (#1161) --- .github/workflows/intel.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 54a388c55..f5de65dcd 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -26,7 +26,7 @@ env: jobs: setup: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: @@ -62,6 +62,7 @@ jobs: - name: install-dependencies-with-spack if: steps.cache-env.outputs.cache-hit != 'true' run: | + sudo mv /usr/local /usr/local_mv # Install NetCDF, ESMF, g2, etc using Spack . /opt/intel/oneapi/setvars.sh git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git @@ -72,7 +73,7 @@ jobs: spack compiler find sudo apt install cmake spack external find - spack add intel-oneapi-mpi + spack config add "packages:mpi:require:'intel-oneapi-mpi'" spack config add "packages:all:require:['%intel']" spack concretize spack install --dirty -v --fail-fast @@ -97,7 +98,7 @@ jobs: strategy: matrix: switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: checkout-ww3 @@ -118,6 +119,8 @@ jobs: - name: build-ww3 run: | + sudo mv /usr/local /usr/local_mv + sudo apt install cmake . /opt/intel/oneapi/setvars.sh source spack/share/spack/setup-env.sh spack env activate ww3-intel From 30729521bb8550b00504ac6f033e749578bf3d30 Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Fri, 23 Feb 2024 15:46:03 +0000 Subject: [PATCH 09/10] intel.yml: update cache key --- .github/workflows/intel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index f5de65dcd..1159e8599 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -13,7 +13,7 @@ concurrency: # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran env: - cache_key: intel10-3 + cache_key: intel12 CC: icc FC: ifort CXX: icpc From 6944915c718daf194080028a9951d09f31900a20 Mon Sep 17 00:00:00 2001 From: Matthew Masarik Date: Fri, 23 Feb 2024 19:57:43 +0000 Subject: [PATCH 10/10] matrix_cmake_ncep: remove temp module updates used for internal testing --- regtests/bin/matrix_cmake_ncep | 57 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep index 4e2b3504d..8d1ca00e5 100755 --- a/regtests/bin/matrix_cmake_ncep +++ b/regtests/bin/matrix_cmake_ncep @@ -41,18 +41,16 @@ EOF # Convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" -# Module Versions from spack-stack that are common for all platforms - modnetcdfc='netcdf-c/4.9.2' - modnetcdff='netcdf-fortran/4.6.0' - modjasper='jasper/2.0.32' - modzlib='zlib/1.2.13' +# Module Versions from HPC-Stack that are common for all platforms + modnetcdf='netcdf/4.7.4' + modjasper='jasper/2.0.25' + modzlib='zlib/1.2.11' modpng='libpng/1.6.37' - modhdf5='hdf5/1.14.0' + modhdf5='hdf5/1.10.6' modbacio='bacio/2.4.1' modg2='g2/3.4.5' - modw3emc='w3emc/2.10.0' - modesmf='esmf/8.4.2' - modscotch='scotch/7.0.4' + modw3emc='w3emc/2.9.2' + modesmf='esmf/8.3.0b09' # Set batchq queue, choose modules and other custom variables to fit system and # to define headers etc (default to original version if empty) @@ -62,19 +60,27 @@ EOF then # If no other h, assuming Hera batchq='slurm' - spackstackpath='/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core' - modcomp='stack-intel/2021.5.0' - modmpi='stack-intel-oneapi-mpi/2021.5.1' - metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install' - modcmake='cmake/3.23.1' + basemodcomp='intel/2022.1.2' + basemodmpi='impi/2022.1.2' + hpcstackpath='/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/modulefiles/stack' + hpcstackversion='hpc/1.2.0' + modcomp='hpc-intel/2022.1.2' + modmpi='hpc-impi/2022.1.2' + scotchpath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hpc-stack/scotch-v7.0.3/install' + metispath='/scratch1/NCEPDEV/climate/Matthew.Masarik/waves/opt/hpc-stack/parmetis-4.0.3/install' + modcmake='cmake/3.20.1' elif [ $isorion ] then batchq='slurm' - spackstackpath='/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core' - modcomp='stack-intel/2022.0.2' - modmpi='stack-intel-oneapi-mpi/2021.5.1' - metispath='/work/noaa/marine/Matthew.Masarik/waves/opt/spack-stack/1.5.0/parmetis-4.0.3/install' - modcmake='cmake/3.23.1' + basemodcomp='intel/2022.1.2' + basemodmpi='impi/2022.1.2' + hpcstackpath='/work/noaa/epic-ps/role-epic-ps/hpc-stack/libs/intel-2022.1.2/modulefiles/stack' + hpcstackversion='hpc/1.2.0' + modcomp='hpc-intel/2022.1.2' + modmpi='hpc-impi/2022.1.2' + scotchpath='/work2/noaa/marine/mmasarik/waves/opt/hpc-stack/scotch-v7.0.3/install' + metispath='/work2/noaa/marine/mmasarik/waves/opt/hpc-stack/parmetis-4.0.3/install' + modcmake='cmake/3.22.1' else batchq= fi @@ -90,7 +96,7 @@ EOF # 1.a Computer/ user dependent set up - echo '#!/bin/sh' > matrix.head + echo '#!/bin/sh --login' > matrix.head echo ' ' >> matrix.head if [ $batchq = "slurm" ] && [ $isorion ] then @@ -133,28 +139,29 @@ EOF # Netcdf, Parmetis and SCOTCH modules & variables echo " module purge" >> matrix.head + echo " module load $modcmake" >> matrix.head if [ ! -z $basemodcomp ]; then echo " module load $basemodcomp" >> matrix.head fi if [ ! -z $basemodmpi ]; then echo " module load $basemodmpi" >> matrix.head fi - echo " module use $spackstackpath" >> matrix.head + echo " module use $hpcstackpath" >> matrix.head + echo " module load $hpcstackversion" >> matrix.head echo " module load $modcomp" >> matrix.head echo " module load $modmpi" >> matrix.head - echo " module load $modcmake" >> matrix.head echo " module load $modpng" >> matrix.head echo " module load $modzlib" >> matrix.head echo " module load $modjasper" >> matrix.head echo " module load $modhdf5" >> matrix.head - echo " module load $modnetcdfc" >> matrix.head - echo " module load $modnetcdff" >> matrix.head + echo " module load $modnetcdf" >> matrix.head echo " module load $modbacio" >> matrix.head echo " module load $modg2" >> matrix.head echo " module load $modw3emc" >> matrix.head echo " module load $modesmf" >> matrix.head - echo " module load $modscotch" >> matrix.head + echo " export METIS_PATH=${metispath}" >> matrix.head + echo " export SCOTCH_PATH=${scotchpath}" >> matrix.head echo " export path_build_root=$(dirname $main_dir)/regtests/buildmatrix" >> matrix.head echo ' [[ -d ${path_build_root} ]] && rm -rf ${path_build_root}' >> matrix.head echo ' '