Skip to content

Commit

Permalink
Merge pull request #12 from RussTreadon-NOAA/intel2022
Browse files Browse the repository at this point in the history 
use stack ncdiag, update hera.gnu hpc-stack, update to head of develop
  • Loading branch information
@hu5970
hu5970 committed Jul 10, 2023
2 parents c077a1c + 3d47231 commit 06ad860
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 215 deletions.
3 changes: 0 additions & 3 deletions modulefiles/gsi_cheyenne.gnu.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@ load("hpc/1.2.0")
load("hpc-gnu/10.1.0")
load("hpc-mpt/2.22")

-- Preload w3nco to work around nemsio "find_dependency(w3nco)" hpc-stack bug
load("w3nco/2.4.1")

load("gsi_common")

local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
Expand Down
6 changes: 1 addition & 5 deletions modulefiles/gsi_common.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ load(pathJoin("netcdf", netcdf_ver))
load(pathJoin("bufr", bufr_ver))
load(pathJoin("bacio", bacio_ver))
load(pathJoin("w3emc", w3emc_ver))
load(pathJoin("w3nco", w3nco_ver))
load(pathJoin("sp", sp_ver))
load(pathJoin("ip", ip_ver))
load(pathJoin("sigio", sigio_ver))
Expand All @@ -31,7 +30,4 @@ load(pathJoin("nemsio", nemsio_ver))
load(pathJoin("wrf_io", wrf_io_ver))
load(pathJoin("ncio", ncio_ver))
load(pathJoin("crtm", crtm_ver))
--load(pathJoin("ncdiag",ncdiag_ver))

setenv("ncdiag_ROOT", "/scratch1/NESDIS/nesdis-rdo2/David.Huber/GSI/ncdiag/1.1.1")
setenv("ncdiag_VERSION", "1.1.1")
load(pathJoin("ncdiag",ncdiag_ver))
4 changes: 2 additions & 2 deletions modulefiles/gsi_hera.gnu.lua
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
help([[
]])

prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/gnu-9.2/modulefiles/stack")

local hpc_ver=os.getenv("hpc_ver") or "1.1.0"
local hpc_ver=os.getenv("hpc_ver") or "1.2.0"
local hpc_gnu_ver=os.getenv("hpc_gnu_ver") or "9.2.0"
local hpc_mpich_ver=os.getenv("hpc_mpich_ver") or "3.3.2"
local cmake_ver=os.getenv("cmake_ver") or "3.20.1"
Expand Down
72 changes: 21 additions & 51 deletions src/enkf/gridio_fv3reg.f90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ module gridio
use params, only: nlevs, cliptracers, datapath, arw, nmm, datestring
use params, only: nx_res,ny_res,nlevs,ntiles,l_fv3reg_filecombined,&
fv3_io_layout_nx,fv3_io_layout_ny,nanals
use params, only: pseudo_rh, l_use_enkf_directZDA
use params, only: pseudo_rh
use mpeu_util, only: getindex
use read_fv3regional_restarts,only:read_fv3_restart_data1d,read_fv3_restart_data2d
use read_fv3regional_restarts,only:read_fv3_restart_data3d,read_fv3_restart_data4d
Expand Down Expand Up @@ -694,7 +694,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid

ps_ind = getindex(vars2d, 'ps') ! Ps (2D)


clip=tiny(clip)
!----------------------------------------------------------------------
if (nbackgrounds > 1) then
write(6,*)'gridio/writegriddata: writing multiple backgrounds not yet supported'
Expand Down Expand Up @@ -853,6 +853,8 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
varstrname = 'sphum'
call fv3lamfile%get_idfn(varstrname,file_id,fv3filename)
call read_fv3_restart_data3d(varstrname,fv3filename,file_id,qworkvar3d)
!enforce lower positive bound (clip) to replace negative hydrometers
if ( cliptracers ) where (qworkvar3d < clip) qworkvar3d = clip
tvworkvar3d=tsenworkvar3d*(one+fv*qworkvar3d)
tvworkvar3d=tvworkvar3d+workinc3d
if(q_ind > 0) then
Expand All @@ -866,6 +868,8 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
qworkvar3d=qworkvar3d+workinc3d
!enforce lower positive bound (clip) to replace negative q
if ( cliptracers ) where (qworkvar3d < clip) qworkvar3d = clip
endif
tsenworkvar3d=tvworkvar3d/(one+fv*qworkvar3d)
varstrname = 'T'
Expand Down Expand Up @@ -932,10 +936,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
call write_fv3_restart_data3d(varstrname,fv3filename,file_id,workvar3d)

endif
Expand All @@ -955,10 +956,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
call write_fv3_restart_data3d(varstrname,fv3filename,file_id,workvar3d)

endif
Expand All @@ -978,10 +976,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
call write_fv3_restart_data3d(varstrname,fv3filename,file_id,workvar3d)

endif
Expand All @@ -1001,10 +996,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
call write_fv3_restart_data3d(varstrname,fv3filename,file_id,workvar3d)

endif
Expand All @@ -1024,10 +1016,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
call write_fv3_restart_data3d(varstrname,fv3filename,file_id,workvar3d)

endif
Expand All @@ -1046,10 +1035,7 @@ subroutine writegriddata(nanal1,nanal2,vars3d,vars2d,n3d,n2d,levels,ndim,vargrid
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
call write_fv3_restart_data3d(varstrname,fv3filename,file_id,workvar3d)

endif
Expand Down Expand Up @@ -1888,7 +1874,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat

ps_ind = getindex(vars2d, 'ps') ! Ps (2D)


clip=tiny(clip)
allocate(my_neb(4))
!----------------------------------------------------------------------
if (nbackgrounds > 1) then
Expand Down Expand Up @@ -2173,6 +2159,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo

if(iope ==0 ) then
if ( cliptracers ) where (qworkvar3d < clip) qworkvar3d = clip
tvworkvar3d=tsenworkvar3d*(one+fv*qworkvar3d)
tvworkvar3d=tvworkvar3d+workinc3d
if(q_ind > 0) then
Expand All @@ -2186,6 +2173,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
qworkvar3d=qworkvar3d+workinc3d
if ( cliptracers ) where (qworkvar3d < clip) qworkvar3d = clip
endif
tsenworkvar3d=tvworkvar3d/(one+fv*qworkvar3d)
endif
Expand Down Expand Up @@ -2281,10 +2269,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
endif
do k=1,nlevs
call mpi_scatterv(workvar3d(:,:,k),recvcounts2d,displs2d,mpi_real4,&
Expand Down Expand Up @@ -2315,10 +2300,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
endif
do k=1,nlevs
call mpi_scatterv(workvar3d(:,:,k),recvcounts2d,displs2d,mpi_real4,&
Expand Down Expand Up @@ -2349,10 +2331,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
endif
do k=1,nlevs
call mpi_scatterv(workvar3d(:,:,k),recvcounts2d,displs2d,mpi_real4,&
Expand Down Expand Up @@ -2383,10 +2362,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
endif
do k=1,nlevs
call mpi_scatterv(workvar3d(:,:,k),recvcounts2d,displs2d,mpi_real4,&
Expand Down Expand Up @@ -2417,10 +2393,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
endif
do k=1,nlevs
call mpi_scatterv(workvar3d(:,:,k),recvcounts2d,displs2d,mpi_real4,&
Expand Down Expand Up @@ -2450,10 +2423,7 @@ subroutine writegriddata_pnc(vars3d,vars2d,n3d,n2d,levels,ndim,vargrid,no_inflat
enddo
enddo
workvar3d=workvar3d+workinc3d
if ( l_use_enkf_directZDA .and. cliptracers ) then ! set cliptracers to remove negative hydrometers
clip = tiny(workvar3d(1,1,1))
where (workvar3d < clip) workvar3d = clip
end if
if ( cliptracers ) where (workvar3d < clip) workvar3d = clip
endif
do k=1,nlevs
call mpi_scatterv(workvar3d(:,:,k),recvcounts2d,displs2d,mpi_real4,&
Expand Down
Loading

0 comments on commit 06ad860

Please sign in to comment.