From 379177ecca0fe5b07e3caec4f8d22adde5f73291 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 24 May 2023 19:26:14 +0000 Subject: [PATCH 01/10] add fv3 rum time info --- fv3_cap.F90 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 7b3ab2eba..c367eb168 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -75,6 +75,8 @@ module fv3gfs_cap_mod integer :: dbug = 0 integer :: frestart(999) = -1 + real(kind=8) :: timers, timere + !----------------------------------------------------------------------- contains @@ -1003,6 +1005,9 @@ subroutine InitializeRealize(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return + timers = 0. + timere = 0. + end subroutine InitializeRealize !----------------------------------------------------------------------------- @@ -1012,9 +1017,13 @@ subroutine ModelAdvance(gcomp, rc) type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc + real(kind=8) :: MPI_Wtime + !----------------------------------------------------------------------------- rc = ESMF_SUCCESS + timers = MPI_Wtime() + print *,'in FV3 cap, time between fv3 run step=', timers-timere if (profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance: ") @@ -1024,6 +1033,9 @@ subroutine ModelAdvance(gcomp, rc) call ModelAdvance_phase2(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + timere = MPI_Wtime() + print *,'in FV3 cap, time in fv3 run step=', timers-timere + if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance: ") end subroutine ModelAdvance From 9d81068f3485590bf474ca4994a21bb8b8aafdb9 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 31 May 2023 19:33:57 +0000 Subject: [PATCH 02/10] update missing value --- .gitmodules | 6 ++++-- atmos_cubed_sphere | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 22c723ac1..6ba00ebee 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "atmos_cubed_sphere"] path = atmos_cubed_sphere - url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere - branch = dev/emc + #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere + #branch = dev/emc + url = https://github.com/junwang-noaa/GFDL_atmos_cubed_sphere + branch = missingvalue [submodule "ccpp/framework"] path = ccpp/framework url = https://github.com/NCAR/ccpp-framework diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 4285e3f3a..b2b148e5e 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 4285e3f3a0bf6c054f8a08fc03469dee6b65e428 +Subproject commit b2b148e5ef2df2c880a21c103611e60423b3c145 From 3d9515853e19a41e9d67e688ba341c1fe7487e04 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Fri, 2 Jun 2023 19:10:01 +0000 Subject: [PATCH 03/10] adding timing information --- fv3_cap.F90 | 46 ++++++++++++++++++++++--------------- io/module_wrt_grid_comp.F90 | 2 +- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index c367eb168..d2232c8b5 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -70,13 +70,15 @@ module fv3gfs_cap_mod logical, allocatable :: is_moving_FB(:) logical :: profile_memory = .true. + logical :: lprint = .false. integer :: mype = -1 integer :: dbug = 0 integer :: frestart(999) = -1 real(kind=8) :: timers, timere - + real(kind=8) :: timep1rs, timep1re + real(kind=8) :: timep2rs, timep2re !----------------------------------------------------------------------- contains @@ -213,8 +215,6 @@ subroutine InitializeAdvertise(gcomp, rc) logical :: use_saved_routehandles, rh_file_exist logical :: fieldbundle_is_restart = .false. - integer :: sloc - type(ESMF_StaggerLoc) :: staggerloc ! !------------------------------------------------------------------------ ! @@ -349,6 +349,7 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return first_kdt = 1 + if( mype == 0) lprint = .true. ! !####################################################################### ! set up fcst grid component @@ -488,6 +489,7 @@ subroutine InitializeAdvertise(gcomp, rc) enddo k = k + wrttasks_per_group_from_parent last_wrttask(i) = k - 1 + if( mype == lead_wrttask(i) ) lprint = .true. ! if(mype==0)print *,'af wrtComp(i)=',i,'k=',k ! prepare name of the wrtComp(i) @@ -638,12 +640,7 @@ subroutine InitializeAdvertise(gcomp, rc) dstGrid(j,i) = grid ! loop over all the mirror fields and set the balanced mirror Grid do ii=1, fieldCount - call ESMF_InfoGetFromHost(fieldList(ii), info=info, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - call ESMF_InfoGet(info, key="staggerloc", value=sloc, rc=rc) - if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - staggerloc = sloc ! convert integer into StaggerLoc_Flag - call ESMF_FieldEmptySet(fieldList(ii), grid=grid, staggerloc=staggerloc, rc=rc) + call ESMF_FieldEmptySet(fieldList(ii), grid=grid, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo ! clean-up @@ -972,8 +969,7 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - if(mype==0) print *,'in fv3_cap, aft import, export fields in atmos' - if(mype==0) print *,'in fv3_cap, init time=',MPI_Wtime()-timeis + if(lprint) print *,'in fv3_cap, init time=',MPI_Wtime()-timeis,mype !----------------------------------------------------------------------- ! end subroutine InitializeAdvertise @@ -990,7 +986,10 @@ subroutine InitializeRealize(gcomp, rc) type(ESMF_State) :: importState, exportState integer :: urc + real(8) :: MPI_Wtime, timeirs + rc = ESMF_SUCCESS + timeirs = MPI_Wtime() ! query for importState and exportState call NUOPC_ModelGet(gcomp, driverClock=clock, importState=importState, exportState=exportState, rc=rc) @@ -1008,6 +1007,8 @@ subroutine InitializeRealize(gcomp, rc) timers = 0. timere = 0. + if(lprint) print *,'in fv3_cap, initirealz time=',MPI_Wtime()-timeirs,mype + end subroutine InitializeRealize !----------------------------------------------------------------------------- @@ -1016,14 +1017,13 @@ subroutine ModelAdvance(gcomp, rc) type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc - real(kind=8) :: MPI_Wtime !----------------------------------------------------------------------------- rc = ESMF_SUCCESS timers = MPI_Wtime() - print *,'in FV3 cap, time between fv3 run step=', timers-timere + if(lprint) print *,'in FV3 cap, time between fv3 run step=', timers-timere,mype if (profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance: ") @@ -1033,11 +1033,11 @@ subroutine ModelAdvance(gcomp, rc) call ModelAdvance_phase2(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return - timere = MPI_Wtime() - print *,'in FV3 cap, time in fv3 run step=', timers-timere - if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance: ") + timere = MPI_Wtime() + if(lprint) print *,'in FV3 cap, time in fv3 run step=', timere-timers, mype + end subroutine ModelAdvance !----------------------------------------------------------------------------- @@ -1052,10 +1052,13 @@ subroutine ModelAdvance_phase1(gcomp, rc) logical :: fcstpe character(len=*),parameter :: subname='(fv3_cap:ModelAdvance_phase1)' character(240) :: msgString + real(kind=8) :: MPI_Wtime !----------------------------------------------------------------------------- rc = ESMF_SUCCESS + timep1rs = MPI_Wtime() + if(lprint) print *,'in fv3_cap,modeladvance phase1 time ', timep1rs-timep2re,mype if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase1: ") @@ -1085,6 +1088,8 @@ subroutine ModelAdvance_phase1(gcomp, rc) call diagnose_cplFields(gcomp, clock, fcstpe, cplprint_flag, dbug, 'import') endif + timep1re = MPI_Wtime() + if(lprint) print *,'in fv3_cap,modeladvance phase1 time ', timep1re-timep1rs,mype if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance_phase1: ") end subroutine ModelAdvance_phase1 @@ -1110,9 +1115,12 @@ subroutine ModelAdvance_phase2(gcomp, rc) type(ESMF_Clock) :: clock, clock_out + real(kind=8) :: MPI_Wtime + !----------------------------------------------------------------------------- rc = ESMF_SUCCESS + timep2rs = MPI_Wtime() if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase2: ") @@ -1211,6 +1219,8 @@ subroutine ModelAdvance_phase2(gcomp, rc) call diagnose_cplFields(gcomp, clock_out, fcstpe, cplprint_flag, dbug, 'export') end if + timep2re = MPI_Wtime() + if(lprint) print *,'in fv3_cap,modeladvance phase2 time ', timep2re-timep2rs, mype if(profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance_phase2: ") end subroutine ModelAdvance_phase2 @@ -1385,8 +1395,8 @@ subroutine ModelFinalize(gcomp, rc) !----------------------------------------------------------------------------- !*** finialize forecast - timeffs = MPI_Wtime() rc = ESMF_SUCCESS + timeffs = MPI_Wtime() ! call ESMF_GridCompGet(gcomp,vm=vm,rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -1419,7 +1429,7 @@ subroutine ModelFinalize(gcomp, rc) call ESMF_GridCompDestroy(fcstComp, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - if(mype==0)print *,' wrt grid comp destroy time=',MPI_Wtime()-timeffs + if(lprint)print *,'in fv3_cap, finalize time=',MPI_Wtime()-timeffs, mype end subroutine ModelFinalize ! diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index 94e568073..d6577c208 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -2158,7 +2158,7 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) endif call mpi_bcast(kchunk3d(grid_id),1,mpi_integer,0,wrt_mpi_comm,rc) endif - if (wrt_int_state%mype == 0) then + if (lprnt) then print *,'ichunk2d,jchunk2d',ichunk2d(grid_id),jchunk2d(grid_id) print *,'ichunk3d,jchunk3d,kchunk3d',ichunk3d(grid_id),jchunk3d(grid_id),kchunk3d(grid_id) endif From 3157091e2164a571442f6a5d7e6d2571a6a55071 Mon Sep 17 00:00:00 2001 From: "Jun.Wang" Date: Thu, 6 Jul 2023 02:36:11 +0000 Subject: [PATCH 04/10] fixes from reviewers --- fv3_cap.F90 | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 0fea4be12..7400c03fa 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -215,6 +215,8 @@ subroutine InitializeAdvertise(gcomp, rc) logical :: use_saved_routehandles, rh_file_exist logical :: fieldbundle_is_restart = .false. + integer :: sloc + type(ESMF_StaggerLoc) :: staggerloc ! !------------------------------------------------------------------------ ! @@ -640,7 +642,12 @@ subroutine InitializeAdvertise(gcomp, rc) dstGrid(j,i) = grid ! loop over all the mirror fields and set the balanced mirror Grid do ii=1, fieldCount - call ESMF_FieldEmptySet(fieldList(ii), grid=grid, rc=rc) + call ESMF_InfoGetFromHost(fieldList(ii), info=info, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + call ESMF_InfoGet(info, key="staggerloc", value=sloc, rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + staggerloc = sloc ! convert integer into StaggerLoc_Flag + call ESMF_FieldEmptySet(fieldList(ii), grid=grid, staggerloc=staggerloc, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return enddo ! clean-up @@ -1023,7 +1030,7 @@ subroutine ModelAdvance(gcomp, rc) rc = ESMF_SUCCESS timers = MPI_Wtime() - if(lprint) print *,'in FV3 cap, time between fv3 run step=', timers-timere,mype + if(lprint) print *,'in fv3_cap, time between fv3 run step=', timers-timere,mype if (profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance: ") @@ -1036,7 +1043,7 @@ subroutine ModelAdvance(gcomp, rc) if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance: ") timere = MPI_Wtime() - if(lprint) print *,'in FV3 cap, time in fv3 run step=', timere-timers, mype + if(lprint) print *,'in fv3_cap, time in fv3 run step=', timere-timers, mype end subroutine ModelAdvance @@ -1058,7 +1065,7 @@ subroutine ModelAdvance_phase1(gcomp, rc) rc = ESMF_SUCCESS timep1rs = MPI_Wtime() - if(lprint) print *,'in fv3_cap,modeladvance phase1 time ', timep1rs-timep2re,mype + if(lprint) print *,'in fv3_cap, time between fv3 run phase2 and phase1 ', timep1rs-timep2re,mype if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase1: ") From a7b0ff010fabc788a715336546015244ea74c1e6 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 6 Jul 2023 09:53:00 -0500 Subject: [PATCH 05/10] add fix for time between run phase --- fv3_cap.F90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 7400c03fa..0e48c8101 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -1013,6 +1013,7 @@ subroutine InitializeRealize(gcomp, rc) timers = 0. timere = 0. + timep2re = 0. if(lprint) print *,'in fv3_cap, initirealz time=',MPI_Wtime()-timeirs,mype @@ -1030,7 +1031,7 @@ subroutine ModelAdvance(gcomp, rc) rc = ESMF_SUCCESS timers = MPI_Wtime() - if(lprint) print *,'in fv3_cap, time between fv3 run step=', timers-timere,mype + if(lprint .and. timere>0.) print *,'in fv3_cap, time between fv3 run step=', timers-timere,mype if (profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance: ") @@ -1065,7 +1066,7 @@ subroutine ModelAdvance_phase1(gcomp, rc) rc = ESMF_SUCCESS timep1rs = MPI_Wtime() - if(lprint) print *,'in fv3_cap, time between fv3 run phase2 and phase1 ', timep1rs-timep2re,mype + if(lprint .and. timep2re>0.) print *,'in fv3_cap, time between fv3 run phase2 and phase1 ', timep1rs-timep2re,mype if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase1: ") From 8be94cf4db84c0188ec5e7048e0b31a90ecdf771 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Thu, 6 Jul 2023 12:31:57 -0500 Subject: [PATCH 06/10] set local vars --- fv3_cap.F90 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 0e48c8101..401262a62 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -76,9 +76,7 @@ module fv3atm_cap_mod integer :: dbug = 0 integer :: frestart(999) = -1 - real(kind=8) :: timers, timere - real(kind=8) :: timep1rs, timep1re - real(kind=8) :: timep2rs, timep2re + real(kind=8) :: timere, timep2re !----------------------------------------------------------------------- contains @@ -1011,7 +1009,6 @@ subroutine InitializeRealize(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - timers = 0. timere = 0. timep2re = 0. @@ -1025,7 +1022,7 @@ subroutine ModelAdvance(gcomp, rc) type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc - real(kind=8) :: MPI_Wtime + real(kind=8) :: MPI_Wtime, timers !----------------------------------------------------------------------------- @@ -1060,7 +1057,7 @@ subroutine ModelAdvance_phase1(gcomp, rc) logical :: fcstpe character(len=*),parameter :: subname='(fv3_cap:ModelAdvance_phase1)' character(240) :: msgString - real(kind=8) :: MPI_Wtime + real(kind=8) :: MPI_Wtime, timep1rs, timep1re !----------------------------------------------------------------------------- @@ -1123,7 +1120,7 @@ subroutine ModelAdvance_phase2(gcomp, rc) type(ESMF_Clock) :: clock, clock_out - real(kind=8) :: MPI_Wtime + real(kind=8) :: MPI_Wtime, timep2rs !----------------------------------------------------------------------------- From bce5ad6fa820b361f0b6f334a0d25446c5e84ca7 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Mon, 10 Jul 2023 14:24:51 -0500 Subject: [PATCH 07/10] fix logfile format --- io/module_wrt_grid_comp.F90 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/io/module_wrt_grid_comp.F90 b/io/module_wrt_grid_comp.F90 index a167ccb5c..b75ac7dd3 100644 --- a/io/module_wrt_grid_comp.F90 +++ b/io/module_wrt_grid_comp.F90 @@ -2024,7 +2024,9 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) if (mype == lead_write_task) then !** write out inline post log file open(newunit=nolog,file='log.atm.inlinepost.f'//trim(cfhour),form='FORMATTED') - write(nolog,"(' completed fv3atm fhour=',f10.3,2x,6(i4,2x))") nfhour, idate(1:6) + write(nolog,"('completed: fv3atm')") + write(nolog,"('forecast hour: ',f10.3)") nfhour + write(nolog,"('valid time: ',6(i4,2x))") wrt_int_state%fdate(1:6) close(nolog) endif if (lprnt) then @@ -2327,7 +2329,9 @@ subroutine wrt_run(wrt_comp, imp_state_write, exp_state_write,clock,rc) if (out_phase == 1 .and. mype == lead_write_task) then !** write out log file open(newunit=nolog,file='log.atm.f'//trim(cfhour),form='FORMATTED') - write(nolog,"(' completed fv3atm fhour=',f10.3,2x,6(i4,2x))") nfhour, idate(1:6) + write(nolog,"('completed: fv3atm')") + write(nolog,"('forecast hour: ',f10.3)") nfhour + write(nolog,"('valid time: ',6(i4,2x))") wrt_int_state%fdate(1:6) close(nolog) endif enddo two_phase_loop From e72c5a9a80c9db33cf12f543216314f11c660336 Mon Sep 17 00:00:00 2001 From: "Jun.Wang" Date: Tue, 22 Aug 2023 18:01:37 +0000 Subject: [PATCH 08/10] add write_runtimelog option --- fv3_cap.F90 | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index 3f62d0823..f309d16b1 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -70,6 +70,7 @@ module fv3atm_cap_mod logical, allocatable :: is_moving_FB(:) logical :: profile_memory = .true. + logical :: write_runtimelog = .false. logical :: lprint = .false. integer :: mype = -1 @@ -248,6 +249,11 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return profile_memory = (trim(value)/="false") + call ESMF_AttributeGet(gcomp, name="RunTimeLog", value=value, defaultValue="false", & + convention="NUOPC", purpose="Instance", rc=rc) + if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return + write_runtimelog = (trim(value)=="true") + call ESMF_AttributeGet(gcomp, name="DumpFields", value=value, defaultValue="false", & convention="NUOPC", purpose="Instance", rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return @@ -975,7 +981,7 @@ subroutine InitializeAdvertise(gcomp, rc) if (ESMF_LogFoundError(rcToCheck=urc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__, rcToReturn=rc)) return - if(lprint) print *,'in fv3_cap, init time=',MPI_Wtime()-timeis,mype + if(write_runtimelog .and. lprint) print *,'in fv3_cap, init time=',MPI_Wtime()-timeis,mype !----------------------------------------------------------------------- ! end subroutine InitializeAdvertise @@ -1013,7 +1019,7 @@ subroutine InitializeRealize(gcomp, rc) timere = 0. timep2re = 0. - if(lprint) print *,'in fv3_cap, initirealz time=',MPI_Wtime()-timeirs,mype + if(write_runtimelog .and. lprint) print *,'in fv3_cap, initirealz time=',MPI_Wtime()-timeirs,mype end subroutine InitializeRealize @@ -1029,7 +1035,7 @@ subroutine ModelAdvance(gcomp, rc) rc = ESMF_SUCCESS timers = MPI_Wtime() - if(lprint .and. timere>0.) print *,'in fv3_cap, time between fv3 run step=', timers-timere,mype + if(write_runtimelog .and. timere>0. .and. lprint) print *,'in fv3_cap, time between fv3 run step=', timers-timere,mype if (profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance: ") @@ -1042,7 +1048,7 @@ subroutine ModelAdvance(gcomp, rc) if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance: ") timere = MPI_Wtime() - if(lprint) print *,'in fv3_cap, time in fv3 run step=', timere-timers, mype + if(write_runtimelog .and. lprint) print *,'in fv3_cap, time in fv3 run step=', timere-timers, mype end subroutine ModelAdvance @@ -1064,7 +1070,7 @@ subroutine ModelAdvance_phase1(gcomp, rc) rc = ESMF_SUCCESS timep1rs = MPI_Wtime() - if(lprint .and. timep2re>0.) print *,'in fv3_cap, time between fv3 run phase2 and phase1 ', timep1rs-timep2re,mype + if(write_runtimelog .and. timep2re>0. .and. lprint) print *,'in fv3_cap, time between fv3 run phase2 and phase1 ', timep1rs-timep2re,mype if(profile_memory) call ESMF_VMLogMemInfo("Entering FV3 ModelAdvance_phase1: ") @@ -1095,7 +1101,7 @@ subroutine ModelAdvance_phase1(gcomp, rc) endif timep1re = MPI_Wtime() - if(lprint) print *,'in fv3_cap,modeladvance phase1 time ', timep1re-timep1rs,mype + if(write_runtimelog .and. lprint) print *,'in fv3_cap,modeladvance phase1 time ', timep1re-timep1rs,mype if (profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance_phase1: ") end subroutine ModelAdvance_phase1 @@ -1232,7 +1238,7 @@ subroutine ModelAdvance_phase2(gcomp, rc) end if timep2re = MPI_Wtime() - if(lprint) print *,'in fv3_cap,modeladvance phase2 time ', timep2re-timep2rs, mype + if(write_runtimelog .and. lprint) print *,'in fv3_cap,modeladvance phase2 time ', timep2re-timep2rs, mype if(profile_memory) call ESMF_VMLogMemInfo("Leaving FV3 ModelAdvance_phase2: ") end subroutine ModelAdvance_phase2 @@ -1441,7 +1447,7 @@ subroutine ModelFinalize(gcomp, rc) call ESMF_GridCompDestroy(fcstComp, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - if(lprint)print *,'in fv3_cap, finalize time=',MPI_Wtime()-timeffs, mype + if(write_runtimelog .and. lprnt) print *,'in fv3_cap, finalize time=',MPI_Wtime()-timeffs, mype end subroutine ModelFinalize ! From ae2c2d81f3e123e53af99f2ab38ed8cc2405cc0a Mon Sep 17 00:00:00 2001 From: "Jun.Wang" Date: Thu, 24 Aug 2023 13:04:00 +0000 Subject: [PATCH 09/10] update FV3 --- fv3_cap.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fv3_cap.F90 b/fv3_cap.F90 index f309d16b1..fa8a549d6 100644 --- a/fv3_cap.F90 +++ b/fv3_cap.F90 @@ -1447,7 +1447,7 @@ subroutine ModelFinalize(gcomp, rc) call ESMF_GridCompDestroy(fcstComp, rc=rc) if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return ! - if(write_runtimelog .and. lprnt) print *,'in fv3_cap, finalize time=',MPI_Wtime()-timeffs, mype + if(write_runtimelog .and. lprint) print *,'in fv3_cap, finalize time=',MPI_Wtime()-timeffs, mype end subroutine ModelFinalize ! From 8194b908d9a4a544699675fbb44f53f91cee2fa3 Mon Sep 17 00:00:00 2001 From: "Jun.Wang" Date: Tue, 29 Aug 2023 16:59:56 +0000 Subject: [PATCH 10/10] update upp and not include dycore updates that change results --- .gitmodules | 6 ++---- atmos_cubed_sphere | 2 +- io/post_fv3.F90 | 28 +++++++++++++++++++--------- upp | 2 +- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6ba00ebee..22c723ac1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "atmos_cubed_sphere"] path = atmos_cubed_sphere - #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere - #branch = dev/emc - url = https://github.com/junwang-noaa/GFDL_atmos_cubed_sphere - branch = missingvalue + url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere + branch = dev/emc [submodule "ccpp/framework"] path = ccpp/framework url = https://github.com/NCAR/ccpp-framework diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index bf56ee624..49f15ecbb 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit bf56ee624b5095119da2cea770313bf902769245 +Subproject commit 49f15ecbbc16405025fae8d672dced19c2073d9e diff --git a/io/post_fv3.F90 b/io/post_fv3.F90 index 696a6b026..2026b67d9 100644 --- a/io/post_fv3.F90 +++ b/io/post_fv3.F90 @@ -93,7 +93,7 @@ subroutine post_run_fv3(wrt_int_state,grid_id,mype,mpicomp,lead_write, & its = wrt_int_state%out_grid_info(grid_id)%i_start !<-- Starting I of this write task's subsection ite = wrt_int_state%out_grid_info(grid_id)%i_end !<-- Ending I of this write task's subsection - if(mype==0) print *,'in post_run,jts=',jts,'jte=',jte,'nwtpg=',nwtpg, & + if(mype==0) print *,'in post_run, numx=',numx,'its=',its,'ite=',ite,'nwtpg=',nwtpg, & 'jts=',jts,'jte=',jte,'maptype=',maptype,'wrt_int_state%FBCount=',wrt_int_state%FBCount ! @@ -508,7 +508,7 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) use vrbls3d, only: t, q, uh, vh, wh, alpint, dpres, zint, zmid, o3, & qqr, qqs, cwm, qqi, qqw, qqg, omga, cfr, pmid, & q2, rlwtt, rswtt, tcucn, tcucns, train, el_pbl, & - pint, exch_h, ref_10cm, qqni, qqnr, qqnwfa, & + pint, exch_h, ref_10cm, qqni, qqnr, qqnw, qqnwfa, & qqnifa, effri, effrl, effrs, aextc55, taod5503d, & duem, dusd, dudp, duwt, dusv, ssem, sssd, ssdp, & sswt, sssv, bcem, bcsd, bcdp, bcwt, bcsv, ocem, & @@ -3642,8 +3642,8 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) endif if(imp_physics == 8) then - ! model level rain number - if(trim(fieldname)=='ncrain') then + ! model level rain water number + if(trim(fieldname)=='rain_nc') then !$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,qqnr,arrayr43d,spval,fillvalue) do l=1,lm do j=jsta,jend @@ -3655,8 +3655,8 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif - ! model level rain number - if(trim(fieldname)=='ncice') then + ! model level cloud ice number + if(trim(fieldname)=='nicp') then !$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,qqni,arrayr43d,spval,fillvalue) do l=1,lm do j=jsta,jend @@ -3668,6 +3668,19 @@ subroutine set_postvars_fv3(wrt_int_state,grid_id,mype,mpicomp) enddo endif + ! model level cloud water number + if(trim(fieldname)=='water_nc') then + !$omp parallel do default(none) private(i,j,l) shared(lm,jsta,jend,ista,iend,qqnw,arrayr43d,spval,fillvalue) + do l=1,lm + do j=jsta,jend + do i=ista, iend + qqnw(i,j,l)=arrayr43d(i,j,l) + if(abs(arrayr43d(i,j,l)-fillvalue)