Merge branch 'production_DE_openacc_test' of github.com:FESOM/fesom2 …

…into production_DE_openacc_test
FESOM · Sep 25, 2024 · 25692cf · 25692cf
2 parents dd467cb + a6a0ba4
commit 25692cf
Show file tree

Hide file tree

Showing 12 changed files with 464 additions and 172 deletions.
diff --git a/config/namelist.io b/config/namelist.io
@@ -10,12 +10,15 @@ ldiag_DVD        =.false.
 ldiag_forc       =.false.
 ldiag_extflds    =.false.
 ldiag_trflx      =.false.
+ldiag_uvw_sqr    =.false.
+ldiag_trgrd_xyz  =.false.
+
 /
 
 &nml_general
 io_listsize    =120 !number of streams to allocate. shallbe large or equal to the number of streams in &nml_list
 vec_autorotate =.false.
-compression_level = 0
+compression_level = 1
 /
 
 ! for sea ice related variables use_ice should be true, otherewise there will be no output

diff --git a/env/levante.dkrz.de/shell.gnu b/env/levante.dkrz.de/shell.gnu
@@ -18,8 +18,7 @@ export FC=mpif90 CC=mpicc CXX=mpicxx
 # following is only needed for libblas which is needed by params lib and often provided by lapack
 #module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0
 # so use the LD_LIBRARY_PATH or other paths like prefix paths etc for cmake
-#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH
-spack load intel-oneapi-mkl@2022.0.1%gcc@11.2.0
+#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH spack load [email protected]%[email protected]
 
 #other alternative blas
 #spack load [email protected]%[email protected]

diff --git a/src/MOD_PARTIT.F90 b/src/MOD_PARTIT.F90
@@ -5,12 +5,12 @@ module MOD_PARTIT
 USE, intrinsic :: ISO_FORTRAN_ENV, only : int32
 USE MOD_WRITE_BINARY_ARRAYS
 USE MOD_READ_BINARY_ARRAYS
+USE mpi
 #if defined(_OPENMP)
     USE OMP_LIB
 #endif
 IMPLICIT NONE
 SAVE
-include 'mpif.h'
 integer, parameter   :: MAX_LAENDERECK=16
 integer, parameter   :: MAX_NEIGHBOR_PARTITIONS=32
 
@@ -217,4 +217,4 @@ subroutine READ_T_PARTIT(partit, unit, iostat, iomsg)
     read(unit, iostat=iostat, iomsg=iomsg) partit%pe_status
 end subroutine READ_T_PARTIT
 
-end module MOD_PARTIT
+end module MOD_PARTIT
diff --git a/src/fortran_utils.F90 b/src/fortran_utils.F90
@@ -1,5 +1,6 @@
  ! synopsis: basic Fortran utilities, no MPI, dependencies only to INTRINSIC modules
 module fortran_utils
+  use mpi
   implicit none
 
 contains
@@ -48,7 +49,6 @@ function mpirank_to_txt(mpicomm) result(txt)
     integer mype
     integer npes
     integer mpierr
-    include 'mpif.h'
 
     call MPI_Comm_Rank(mpicomm, mype, mpierr)
     call MPI_Comm_Size(mpicomm, npes, mpierr)

diff --git a/src/gen_modules_diag.F90 b/src/gen_modules_diag.F90
diff --git a/src/gen_modules_partitioning.F90 b/src/gen_modules_partitioning.F90
@@ -113,7 +113,7 @@ subroutine par_ex(COMM, mype, abort)       ! finalizes MPI
 #ifndef __oasis
   if (present(abort)) then
      if (mype==0) write(*,*) 'Run finished unexpectedly!'
-     call MPI_ABORT(MPI_COMM_WORLD, 1 )
+     call MPI_ABORT(MPI_COMM_WORLD, 1, error)
   else
           ! TODO: this is where fesom standalone, ifsinterface etc get to 
           !1. there no abort actually even when model calls abort, and barrier may hang
@@ -580,4 +580,3 @@ subroutine status_check(partit)
     call par_ex(partit%MPI_COMM_FESOM, partit%mype, 1)
 endif
 end subroutine status_check
-
diff --git a/src/gen_surface_forcing.F90 b/src/gen_surface_forcing.F90
@@ -683,9 +683,10 @@ SUBROUTINE nc_sbc_ini(partit, mesh)
       end do
          ! interpolate in time
 
-      if (partit%mype==0) then 
-        write(*,*) 'sbc_do --> mstep:',mstep, ' rdate=', rdate
-      end if 
+      !!PS if (partit%mype==0) then 
+      !!PS  write(*,*) 'sbc_do --> mstep:',mstep, ' rdate=', rdate
+      !!PS end if 
+
       call data_timeinterp(rdate, partit)
    END SUBROUTINE nc_sbc_ini
 
@@ -1577,9 +1578,9 @@ SUBROUTINE sbc_do(partit, mesh)
     end if
 #endif
 
-      if (partit%mype==0) then 
-        write(*,*) 'sbc_do --> mstep:',mstep, ' rdate=', rdate
-      end if 
+      !!PS if (partit%mype==0) then 
+      !!PS   write(*,*) 'sbc_do --> mstep:',mstep, ' rdate=', rdate
+      !!PS end if 
 
       ! interpolate in time
       call data_timeinterp(rdate, partit)

diff --git a/src/ifs_interface/mpp_io.F90 b/src/ifs_interface/mpp_io.F90
@@ -6,6 +6,7 @@
 !-----------------------------------------------------
 
 MODULE mpp_io
+    USE mpi
 #if defined(__MULTIO)        
     USE iom, only : iom_enable_multio, iom_initialize, iom_init_server, iom_finalize
 #endif
@@ -30,7 +31,6 @@ MODULE mpp_io
 
     SUBROUTINE mpp_io_init( iicomm,  lio, irequired, iprovided, lmpi1 ) 
 
-        INCLUDE "mpif.h"
         INTEGER, INTENT(INOUT) :: iicomm
         LOGICAL, INTENT(INOUT) :: lio
         INTEGER, INTENT(INOUT) :: irequired, iprovided
@@ -126,7 +126,6 @@ SUBROUTINE mpp_io_init_2( iicomm )
         INTEGER :: icode, ierr, icolor, iicommx, iicommm, iicommo
         INTEGER :: ji,inum
         LOGICAL :: lcompp
-        INCLUDE "mpif.h"
 
         ! Construct multio server communicator
 

diff --git a/src/io_meandata.F90 b/src/io_meandata.F90
@@ -1141,8 +1141,11 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh)
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_tot' ,  'tot. temperature DVD \n (Banerjee et al. 2023)' , 'K^2/s' , dvd_SD_tot(        :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_advh',  'temperature DVD horiz. adv.'                    , 'K^2/s' , dvd_SD_chi_adv_h(  :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_advv',  'temperature DVD vert. adv. '                    , 'K^2/s' , dvd_SD_chi_adv_v(  :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
-        call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difh',  'temperature DVD horiz. diff. expl.'             , 'K^2/s' , dvd_SD_chi_dif_he( :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        if (Redi .eqv. .False. .and. K_hor /= 0.0_WP) then
+            call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difh',  'temperature DVD horiz. diff. expl.'             , 'K^2/s' , dvd_SD_chi_dif_he( :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        end if
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difvi', 'temperature DVD vert. diff. impl.'              , 'K^2/s' , dvd_SD_chi_dif_vi( :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difsbc', 'temperature DVD sbc.'                          , 'K^2/s' , dvd_SD_chi_dif_sbc(:,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         if (.not. tracers%data(1)%i_vert_diff) then
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difve', 'temperature DVD vert. diff. expl.'          , 'K^2/s' , dvd_SD_chi_dif_ve( :,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         end if
@@ -1151,6 +1154,7 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh)
         end if 
         if (Redi) then 
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difheR', 'temperature DVD horiz. redi diff. expl.'   , 'K^2/s' , dvd_SD_chi_dif_heR(:,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difveR', 'temperature DVD vert. redi diff. expl.'    , 'K^2/s' , dvd_SD_chi_dif_veR(:,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_temp_SD_difviR', 'temperature DVD vert. redi diff. impl.'    , 'K^2/s' , dvd_SD_chi_dif_viR(:,:,1), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         end if     
@@ -1160,8 +1164,11 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh)
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_tot' ,  'tot. salinity DVD \n (Banerjee et al. 2023)'    , 'K^2/s' , dvd_SD_tot(        :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_advh',  'salinity DVD horiz. adv.'                       , 'K^2/s' , dvd_SD_chi_adv_h(  :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_advv',  'salinity DVD vert. adv. '                       , 'K^2/s' , dvd_SD_chi_adv_v(  :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
-        call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difh',  'salinity DVD horiz. diff. expl.'                , 'K^2/s' , dvd_SD_chi_dif_he( :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        if (Redi .eqv. .False. .and. K_hor /= 0.0_WP) then
+            call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difh',  'salinity DVD horiz. diff. expl.'                , 'K^2/s' , dvd_SD_chi_dif_he( :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        end if 
         call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difvi', 'salinity DVD vert. diff. impl.'                 , 'K^2/s' , dvd_SD_chi_dif_vi( :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difsbc','salinity DVD sbc.'                              , 'K^2/s' , dvd_SD_chi_dif_sbc(:,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         if (.not. tracers%data(1)%i_vert_diff) then
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difve', 'salinity DVD vert. diff. expl.'             , 'K^2/s' , dvd_SD_chi_dif_ve( :,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         end if
@@ -1170,6 +1177,7 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh)
         end if 
         if (Redi) then 
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difheR', 'salinity DVD horiz. redi diff. expl.'      , 'K^2/s' , dvd_SD_chi_dif_heR(:,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difveR', 'salinity DVD vert. redi diff. expl.'       , 'K^2/s' , dvd_SD_chi_dif_veR(:,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
             call def_stream((/nl-1, nod2D/), (/nl-1, myDim_nod2D/), 'dvd_salt_SD_difviR', 'salinity DVD vert. redi diff. impl.'       , 'K^2/s' , dvd_SD_chi_dif_viR(:,:,2), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
         end if     
@@ -1200,6 +1208,29 @@ subroutine ini_mean_io(ice, dynamics, tracers, partit, mesh)
         end if 
     end if
 
+!_______________________________________________________________________________
+! compute squared velocities of u, v, w
+CASE ('UVW_SQR   ')
+    if (ldiag_uvw_sqr) then
+        !___temperature DVD_____________________________________________________
+        call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'u2' , 'squared zonal velocity'     , 'm^2/s^2' , uv2(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/), 'v2' , 'squared meridional velocity', 'm^2/s^2' , uv2(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1, nod2D/) , (/nl-1, myDim_nod2D/) , 'w2' , 'squared vertical velocity'  , 'm^2/s^2' , wvel2(:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+    end if !--> if (ldiag_DVD) then    
+
+!_______________________________________________________________________________
+! compute horizontal and vertical tracer gradients
+CASE ('TRGRD_XYZ ')
+    if (ldiag_trgrd_xyz) then
+        call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/),  'temp_grdx',   'zonal temperature gradient',        'K/m', trgrd_x(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/),  'temp_grdy',   'meridional temperature gradient',   'K/m', trgrd_y(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1,  nod2D/), (/nl-1, myDim_nod2D /),  'temp_grdz',   'vertical temperature gradient',     'K/m', trgrd_z(1,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+
+        call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/),  'salt_grdx',   'zonal salinity gradient',         'psu/m', trgrd_x(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1, elem2D/), (/nl-1, myDim_elem2D/),  'salt_grdy',   'meridional salinity gradient',    'psu/m', trgrd_y(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+        call def_stream((/nl-1,  nod2D/), (/nl-1, myDim_nod2D /),  'salt_grdz',   'vertical salinity gradient',      'psu/m', trgrd_z(2,:,:), io_list(i)%freq, io_list(i)%unit, io_list(i)%precision, partit, mesh)
+    end if !--> if (ldiag_DVD) then    
+
 !_______________________________________________________________________________
 CASE DEFAULT
     if (mype==0) write(*,*) 'stream ', io_list(i)%id, ' is not defined !'

diff --git a/src/io_restart.F90 b/src/io_restart.F90
@@ -12,6 +12,7 @@ MODULE io_RESTART
   use MOD_PARTIT
   use MOD_PARSUP
   use fortran_utils
+  use mpi
 #if defined(__recom)
   use recom_glovar
   use recom_config
@@ -771,7 +772,6 @@ subroutine read_all_raw_restarts(mpicomm, mype)
   integer fileunit
   integer status
   integer mpierr
-  include 'mpif.h'
 
   if(mype == RAW_RESTART_METADATA_RANK) then
     ! read metadata info for the raw restart
@@ -860,7 +860,6 @@ subroutine finalize_restart()
 !
 !_______________________________________________________________________________
 subroutine read_restart(path, filegroup, mpicomm, mype)
-  include 'mpif.h'        
   character(len=*), intent(in) :: path
   type(restart_file_group), intent(inout) :: filegroup
   integer, intent(in) :: mpicomm
@@ -1004,12 +1003,11 @@ function is_due(unit, frequency, istep) result(d)
 !     integer mype
 !     integer npes
 !     integer mpierr
-!     include 'mpif.h'
 !   
 !     call MPI_Comm_Rank(mpicomm, mype, mpierr)
 !     call MPI_Comm_Size(mpicomm, npes, mpierr)
 !     txt = int_to_txt_pad(mype,int(log10(real(npes)))+1) ! pad to the width of the number of processes
 !   end function
 !!PS --> move this function also to fortran_utils.F90 
 
-end module
+end module
diff --git a/src/temp/MOD_PARTIT.F90 b/src/temp/MOD_PARTIT.F90
@@ -5,9 +5,9 @@ module MOD_PARTIT
 USE, intrinsic :: ISO_FORTRAN_ENV
 USE MOD_WRITE_BINARY_ARRAYS
 USE MOD_READ_BINARY_ARRAYS
+USE mpi
 IMPLICIT NONE
 SAVE
-include 'mpif.h'
 integer, parameter   :: MAX_LAENDERECK=16
 integer, parameter   :: MAX_NEIGHBOR_PARTITIONS=32
 
@@ -186,4 +186,4 @@ subroutine READ_T_PARTIT(partit, unit, iostat, iomsg)
     read(unit, iostat=iostat, iomsg=iomsg) partit%pe_status
 end subroutine READ_T_PARTIT
 
-end module MOD_PARTIT
+end module MOD_PARTIT
diff --git a/work/job_albedo_chain b/work/job_albedo_chain
@@ -97,25 +97,6 @@ cp -n ../config/namelist.ice     .
 cp -n ../config/namelist.io      .
 cp -n ../config/namelist.icepack .
 
-#___SET CHAIN_ID________________________________________________________________
-if [ -f "file_chain_id" ]; then
-    chain_id=$(<file_chain_id)
-else 
-    chain_id=${chain_s}
-    echo $chain_id > file_chain_id
-fi
-
-#___PRINT INPUT INFO____________________________________________________________
-echo -e "\033[1;7;33m_____JOB CHAIN INFO_____________________________________\033[0m"
-echo -e "\033[1;33m --> actual chain cycle: $chain_id \033[0m"
-echo -e "\033[1;33m --> max. number of chain cycles: $chain_n \033[0m"
-echo -e "\033[1;33m --> simulated time range: [ $year_s $year_e] \033[0m"
-echo -e "\033[1;33m --> slurm: wall-clock-time = $wcl \033[0m"
-echo -e "\033[1;33m --> slurm: ntasks = $ntasks \033[0m"
-echo -e "\033[1;33m --> slurm: nnodes = $nnodes \033[0m"
-if [ $prescribe_rlen -ne 0 ]; then
-    echo -e "\033[1;33m -->change run_length = $prescribe_rlen \033[0m"
-fi
 
 #___CREATE SAVE DIR INFRASTRUCTURE______________________________________________
 # extract resultpath from namelist.config
@@ -148,6 +129,25 @@ fi
 # identify real path in case a link is used
 dname_result="$(realpath "$dname_result")/"
 
+#___SET CHAIN_ID________________________________________________________________
+if [ -f "${dname_result}/file_chain_id" ]; then
+    chain_id=$(<${dname_result}/file_chain_id)
+else 
+    chain_id=${chain_s}
+    echo $chain_id > ${dname_result}/file_chain_id
+fi
+#___PRINT INPUT INFO____________________________________________________________
+echo -e "\033[1;7;33m_____JOB CHAIN INFO_____________________________________\033[0m"
+echo -e "\033[1;33m --> actual chain cycle: $chain_id \033[0m"
+echo -e "\033[1;33m --> max. number of chain cycles: $chain_n \033[0m"
+echo -e "\033[1;33m --> simulated time range: [ $year_s $year_e] \033[0m"
+echo -e "\033[1;33m --> slurm: wall-clock-time = $wcl \033[0m"
+echo -e "\033[1;33m --> slurm: ntasks = $ntasks \033[0m"
+echo -e "\033[1;33m --> slurm: nnodes = $nnodes \033[0m"
+if [ $prescribe_rlen -ne 0 ]; then
+    echo -e "\033[1;33m -->change run_length = $prescribe_rlen \033[0m"
+fi
+
 # if directory for chain_id doesn't exist --> create it 
 if [ ! -d "${dname_result}/${chain_id}" ]; then 
     echo -e "\033[33m --> chain_id directory does not exist --> will create it \033[0m"
@@ -234,8 +234,7 @@ if [ $is_newsimul -eq 1 ] ; then
 
         #___BACKUP NAMELIST.* FILES INTO RESULT DIRECTORY_______________________
 
-        cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \
-           namelist.dyn namelist.tra namelist.cvmix ${dname_result}/.
+        cp namelist.* ${dname_result}/.
         cp fesom.x ${dname_result}/.
 
     #___BACKUP SRC FILES INTO RESULT DIRECTORY__________________________________
@@ -335,6 +334,6 @@ fi
 # check if complete cycle is finished only than increase chain_id
 if [ $aux_yr_clock -gt $year_e ] && [ ${chain_id} -lt ${chain_n} ] ; then
     chain_id=$(( $chain_id + 1 ))
-    echo $chain_id > file_chain_id
+    echo $chain_id > ${dname_result}/file_chain_id
 fi