diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 700553cea..1159e8599 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -1,3 +1,8 @@ +# This is a GitHub actions workflow for WW3. +# +# This workflow builds with the Intel compilers. +# +# Matt Masarik, Alex Richert, Ed Hartnett name: Intel Linux Build on: [push, pull_request, workflow_dispatch] @@ -8,7 +13,7 @@ concurrency: # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran env: - cache_key: intel10 + cache_key: intel12 CC: icc FC: ifort CXX: icpc @@ -21,7 +26,7 @@ env: jobs: setup: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: @@ -51,12 +56,13 @@ jobs: sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update - sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic + sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-compiler-fortran-2023.2.1 intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.2.1 intel-oneapi-openmp # Build WW3 spack environment - name: install-dependencies-with-spack if: steps.cache-env.outputs.cache-hit != 'true' run: | + sudo mv /usr/local /usr/local_mv # Install NetCDF, ESMF, g2, etc using Spack . /opt/intel/oneapi/setvars.sh git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git @@ -67,7 +73,7 @@ jobs: spack compiler find sudo apt install cmake spack external find - spack add intel-oneapi-mpi + spack config add "packages:mpi:require:'intel-oneapi-mpi'" spack config add "packages:all:require:['%intel']" spack concretize spack install --dirty -v --fail-fast @@ -92,7 +98,7 @@ jobs: strategy: matrix: switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: checkout-ww3 @@ -113,6 +119,8 @@ jobs: - name: build-ww3 run: | + sudo mv /usr/local /usr/local_mv + sudo apt install cmake . /opt/intel/oneapi/setvars.sh source spack/share/spack/setup-env.sh spack env activate ww3-intel diff --git a/model/src/cmake/src_list.cmake b/model/src/cmake/src_list.cmake index 2152b697e..dcab88a09 100644 --- a/model/src/cmake/src_list.cmake +++ b/model/src/cmake/src_list.cmake @@ -66,6 +66,7 @@ set(nuopc_mesh_cap_src wav_shel_inp.F90 wav_comp_nuopc.F90 wav_import_export.F90 + wav_wrapper_mod.F90 ) set(esmf_multi_cap_src diff --git a/model/src/w3fld1md.F90 b/model/src/w3fld1md.F90 index 960fd185a..ad94a12ea 100644 --- a/model/src/w3fld1md.F90 +++ b/model/src/w3fld1md.F90 @@ -1116,7 +1116,11 @@ SUBROUTINE APPENDTAIL(INSPC, WN2, NKT, KA1, KA2, KA3, WNDDIR,SAT) DO K=KA1, KA2-1 AVG=SUM(INSPC(K,:))/MAX(REAL(NTH),1.) DO T=1,NTH - INSPC(K,T)=BT(K)*INSPC(K,T)/TPI/(WN2(K)**3.0)/AVG + if (avg /= 0.0) then + INSPC(K,T)=BT(K)*INSPC(K,T)/TPI/(WN2(K)**3.0)/AVG + else + inspc(k,t) = 0.0 + end if ENDDO ENDDO !----------------------------------------------------------- @@ -1134,7 +1138,11 @@ SUBROUTINE APPENDTAIL(INSPC, WN2, NKT, KA1, KA2, KA3, WNDDIR,SAT) ENDDO AVG=SUM(NORMSPC)/MAX(REAL(NTH),1.) DO T=1, NTH - INSPC(K,T) = SAT * NORMSPC(T)/TPI/(WN2(K)**3.0)/AVG + if (avg /= 0.0) then + INSPC(K,T) = SAT * NORMSPC(T)/TPI/(WN2(K)**3.0)/AVG + else + inspc(k,t) = 0.0 + end if ENDDO ENDDO DO T=1, NTH @@ -1148,7 +1156,11 @@ SUBROUTINE APPENDTAIL(INSPC, WN2, NKT, KA1, KA2, KA3, WNDDIR,SAT) AVG=SUM(NORMSPC)/MAX(REAL(NTH),1.)!1./4. DO K=KA3+1, NKT DO T=1, NTH - INSPC(K,T)=NORMSPC(T)*(SAT)/TPI/(WN2(K)**3.0)/AVG + if (avg /= 0.0) then + INSPC(K,T)=NORMSPC(T)*(SAT)/TPI/(WN2(K)**3.0)/AVG + else + inspc(k,t) = 0.0 + end if ENDDO ENDDO DEALLOCATE(ANGLE1) diff --git a/model/src/wav_comp_nuopc.F90 b/model/src/wav_comp_nuopc.F90 index 8b9ff5a5a..c7bb14ef2 100644 --- a/model/src/wav_comp_nuopc.F90 +++ b/model/src/wav_comp_nuopc.F90 @@ -48,6 +48,7 @@ module wav_comp_nuopc use w3odatmd , only : user_netcdf_grdout use w3odatmd , only : time_origin, calendar_name, elapsed_secs use wav_shr_mod , only : casename, multigrid, inst_suffix, inst_index, unstr_mesh + use wav_wrapper_mod , only : ufs_settimer, ufs_logtimer, ufs_file_setlogunit, wtime #ifndef W3_CESMCOUPLED use wmwavemd , only : wmwave use wmupdtmd , only : wmupd2 @@ -99,9 +100,12 @@ module wav_comp_nuopc !! using ESMF. If restart_option is present as config !! option, user_restalarm will be true and will be !! set using restart_option, restart_n and restart_ymd - integer :: time0(2) - integer :: timen(2) - + integer :: ymd !< current year-month-day + integer :: tod !< current time of day (sec) + integer :: time0(2) !< start time stored as yyyymmdd,hhmmss + integer :: timen(2) !< end time stored as yyyymmdd,hhmmss + integer :: nu_timer !< simple timer log, unused except by UFS + logical :: runtimelog = .false. !< logical flag for writing runtime log files character(*), parameter :: modName = "(wav_comp_nuopc)" !< the name of this module character(*), parameter :: u_FILE_u = & !< a character string for an ESMF log message __FILE__ @@ -238,6 +242,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) character(len=*), parameter :: subname=trim(modName)//':(InitializeAdvertise) ' !------------------------------------------------------------------------------- + call ufs_settimer(wtime) rc = ESMF_SUCCESS call ESMF_LogWrite(trim(subname)//' called', ESMF_LOGMSG_INFO) @@ -369,6 +374,15 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc) write(logmsg,'(A,l)') trim(subname)//': Wave wav_coupling_to_cice setting is ',wav_coupling_to_cice call ESMF_LogWrite(trim(logmsg), ESMF_LOGMSG_INFO) + ! Determine Runtime logging + call NUOPC_CompAttributeGet(gcomp, name="RunTimeLog", value=cvalue, isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) runtimelog=(trim(cvalue)=="true") + write(logmsg,*) runtimelog + call ESMF_LogWrite('WW3_cap:RunTimeLog = '//trim(logmsg), ESMF_LOGMSG_INFO) + if (runtimelog) then + call ufs_file_setLogUnit('./log.ww3.timer',nu_timer,runtimelog) + end if call advertise_fields(importState, exportState, flds_scalar_name, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -475,6 +489,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) rc = ESMF_SUCCESS if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' called', ESMF_LOGMSG_INFO) + call ufs_settimer(wtime) !-------------------------------------------------------------------- ! Set up data structures !-------------------------------------------------------------------- @@ -872,6 +887,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) enddo end if #endif + if (root_task) call ufs_logtimer(nu_timer,time,start_tod,'InitializeRealize time: ',runtimelog,wtime) if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) @@ -1010,8 +1026,6 @@ subroutine ModelAdvance(gcomp, rc) type(ESMF_Time) :: currTime, nextTime, startTime, stopTime integer :: yy,mm,dd,hh,ss integer :: imod - integer :: ymd ! current year-month-day - integer :: tod ! current time of day (sec) integer :: shrlogunit ! original log unit and level character(ESMF_MAXSTR) :: msgString character(len=*),parameter :: subname = '(wav_comp_nuopc:ModelAdvance) ' @@ -1051,6 +1065,8 @@ subroutine ModelAdvance(gcomp, rc) if ( root_task ) then write(nds(1),'(a,3i4,i10)') 'ymd2date currTime wav_comp_nuopc hh,mm,ss,ymd', hh,mm,ss,ymd end if + if (root_task) call ufs_logtimer(nu_timer,time,tod,'ModelAdvance time since last step: ',runtimelog,wtime) + call ufs_settimer(wtime) ! use next time; the NUOPC clock is not updated ! until the end of the time interval @@ -1148,6 +1164,8 @@ subroutine ModelAdvance(gcomp, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) + if (root_task) call ufs_logtimer(nu_timer,time,tod,'ModelAdvance time: ',runtimelog,wtime) + call ufs_settimer(wtime) end subroutine ModelAdvance @@ -1367,6 +1385,7 @@ subroutine ModelFinalize(gcomp, rc) end if call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) + if(root_task) call ufs_logtimer(nu_timer,timen,tod,'ModelFinalize time: ',runtimelog,wtime) end subroutine ModelFinalize @@ -1585,6 +1604,7 @@ subroutine waveinit_ufs( gcomp, ntrace, mpi_comm, mds, rc) ! Initialize ww3 for ufs (called from InitializeRealize) use w3odatmd , only : fnmpre + use w3gdatmd , only : dtcfl, dtcfli, dtmax, dtmin use w3initmd , only : w3init use wav_shel_inp , only : read_shel_config use wav_shel_inp , only : npts, odat, iprt, x, y, pnames, prtfrm @@ -1601,6 +1621,7 @@ subroutine waveinit_ufs( gcomp, ntrace, mpi_comm, mds, rc) character(len=CL) :: logmsg logical :: isPresent, isSet character(len=CL) :: cvalue + integer :: dt_in(4) character(len=*), parameter :: subname = '(wav_comp_nuopc:wavinit_ufs)' ! ------------------------------------------------------------------- @@ -1648,6 +1669,21 @@ subroutine waveinit_ufs( gcomp, ntrace, mpi_comm, mds, rc) call w3init ( 1, .false., 'ww3', mds, ntrace, odat, flgrd, flgr2, flgd, flg2, & npts, x, y, pnames, iprt, prtfrm, mpi_comm ) + write(logmsg,'(A,4f10.2)') trim(subname)//': mod_def timesteps file ',dtmax,dtcfl,dtcfli,dtmin + call ESMF_LogWrite(trim(logmsg), ESMF_LOGMSG_INFO) + call NUOPC_CompAttributeGet(gcomp, name='dt_in', isPresent=isPresent, isSet=isSet, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + if (isPresent .and. isSet) then + call NUOPC_CompAttributeGet(gcomp, name='dt_in', value=cvalue, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + read(cvalue,*)dt_in + dtmax = real(dt_in(1),4) + dtcfl = real(dt_in(2),4) + dtcfli = real(dt_in(3),4) + dtmin = real(dt_in(4),4) + write(logmsg,'(A,4f10.2)') trim(subname)//': mod_def timesteps reset ',dtmax,dtcfl,dtcfli,dtmin + call ESMF_LogWrite(trim(logmsg), ESMF_LOGMSG_INFO) + end if if (dbug_flag > 5) call ESMF_LogWrite(trim(subname)//' done', ESMF_LOGMSG_INFO) end subroutine waveinit_ufs diff --git a/model/src/wav_wrapper_mod.F90 b/model/src/wav_wrapper_mod.F90 new file mode 100644 index 000000000..dd2465829 --- /dev/null +++ b/model/src/wav_wrapper_mod.F90 @@ -0,0 +1,119 @@ +!> @file wav_wrapper_mod +!! +!> A wrapper module for log functionality in UFS +!! +!> @details Contains public logging routines for UFS and +!! stub routines for CESM +!! +!> Denise.Worthen@noaa.gov +!> @date 01-08-2024 +module wav_wrapper_mod + + use wav_kind_mod , only : r8 => shr_kind_r8, r4 => shr_kind_r4, i4 => shr_kind_i4 + use wav_kind_mod , only : CL => shr_kind_cl, CS => shr_kind_cs + + implicit none + + real(r8) :: wtime = 0.0 + +#ifdef CESMCOUPLED +contains + ! Define stub routines that do nothing - they are just here to avoid + ! having cppdefs in the main program + subroutine ufs_settimer(timevalue) + real(r8), intent(inout) :: timevalue + end subroutine ufs_settimer + subroutine ufs_logtimer(nunit,times,tod,string,runtimelog,wtime0) + integer, intent(in) :: nunit + integer(i4), intent(in) :: times(2), tod + character(len=*), intent(in) :: string + logical, intent(in) :: runtimelog + real(r8), intent(in) :: wtime0 + end subroutine ufs_logtimer + subroutine ufs_file_setLogUnit(filename,nunit,runtimelog) + character(len=*), intent(in) :: filename + logical, intent(in) :: runtimelog + integer, intent(out) :: nunit + end subroutine ufs_file_setLogUnit + subroutine ufs_logfhour(msg,hour) + character(len=*), intent(in) :: msg + real(r8), intent(in) :: hour + end subroutine ufs_logfhour +#else +contains + subroutine ufs_settimer(timevalue) + !> Set a time value + !! @param[inout] timevalue a MPI time value + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + + real(r8), intent(inout) :: timevalue + real(r8) :: MPI_Wtime + timevalue = MPI_Wtime() + end subroutine ufs_settimer + + subroutine ufs_logtimer(nunit,times,tod,string,runtimelog,wtime0) + !> Log a time interval + !! @param[in] nunit the log file unit + !! @param[in] times the ymd,hms time values + !! @param[in] tod the elapsed seconds in the day + !! @param[in] string a message string to log + !! @param[in] runtimelog a logical to control the log function + !! @param[in] wtime0 an initial MPI time + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + integer, intent(in) :: nunit + integer(i4), intent(in) :: times(2),tod + character(len=*), intent(in) :: string + logical, intent(in) :: runtimelog + real(r8), intent(in) :: wtime0 + real(r8) :: MPI_Wtime, timevalue + if (.not. runtimelog) return + if (wtime0 > 0.) then + timevalue = MPI_Wtime()-wtime0 + write(nunit,'(3i8,a,g14.7)')times,tod,' WW3 '//trim(string),timevalue + end if + end subroutine ufs_logtimer + + subroutine ufs_file_setLogUnit(filename,nunit,runtimelog) + !> Create a log unit + !! @param[in] filename the log filename + !! @param[in] runtimelog a logical to control the log function + !! @param[out] nunit the log file unit + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + + character(len=*), intent(in) :: filename + logical, intent(in) :: runtimelog + integer, intent(out) :: nunit + if (.not. runtimelog) return + open (newunit=nunit, file=trim(filename)) + end subroutine ufs_file_setLogUnit + + subroutine ufs_logfhour(msg,hour) + !> Log the completion of model output + !! @param[in] msg the log message + !! @param[in] hour the forecast hour + !! + !> Denise.Worthen@noaa.gov + !> @date 01-08-2024 + + character(len=*), intent(in) :: msg + real(r8), intent(in) :: hour + + character(len=CS) :: filename + integer(r4) :: nunit + + write(filename,'(a,i3.3)')'log.ww3.f',int(hour) + open(newunit=nunit,file=trim(filename)) + write(nunit,'(a)')'completed: ww3' + write(nunit,'(a,f10.3)')'forecast hour:',hour + write(nunit,'(a)')'valid time: '//trim(msg) + close(nunit) + end subroutine ufs_logfhour +#endif + +end module wav_wrapper_mod diff --git a/regtests/ww3_tp2.22/input/ww3_ounf.inp b/regtests/ww3_tp2.22/input/ww3_ounf.inp index 37703605d..6d1998f36 100644 --- a/regtests/ww3_tp2.22/input/ww3_ounf.inp +++ b/regtests/ww3_tp2.22/input/ww3_ounf.inp @@ -13,7 +13,7 @@ $ file for a full documentation of field output options. Namelist type $ selection is used here (for alternative F/T flags, see ww3_shel.inp). $ N - HS EF T01 DIR SPR USS USSH + HS T01 DIR SPR USS USSH $ $--------------------------------------------------------------------- $ $ netCDF version [3,4] diff --git a/regtests/ww3_tp2.22/input/ww3_outf.inp b/regtests/ww3_tp2.22/input/ww3_outf.inp index 3bc3e4908..2ee125996 100644 --- a/regtests/ww3_tp2.22/input/ww3_outf.inp +++ b/regtests/ww3_tp2.22/input/ww3_outf.inp @@ -3,7 +3,7 @@ $ ----------------------------------------- 19680606 000000 3600. 12 $ N - HS EF T01 DIR SPR USS USSH + HS T01 DIR SPR USS USSH $ 1 0 $ diff --git a/regtests/ww3_tp2.22/input/ww3_shel.inp b/regtests/ww3_tp2.22/input/ww3_shel.inp index 6b946943c..8a32e5584 100644 --- a/regtests/ww3_tp2.22/input/ww3_shel.inp +++ b/regtests/ww3_tp2.22/input/ww3_shel.inp @@ -18,7 +18,7 @@ $ 19680606 000000 3600 19680608 000000 $ N - HS EF T01 DIR SPR USS USSH + HS T01 DIR SPR USS USSH $ 19680606 000000 21600 19680608 000000 0.0 0.0 'LEFT'