From 944f3ef55d4629634f4304e6cdcceed5f60325a4 Mon Sep 17 00:00:00 2001 From: Frank Kauker Date: Mon, 30 Sep 2024 17:32:30 +0200 Subject: [PATCH] first running version without correct restart --- CMakeLists.txt | 2 +- config/namelist.config | 25 +- config/namelist.cvmix | 2 + config/namelist.dyn | 4 +- config/namelist.icepack | 16 +- config/namelist.io | 17 +- config/namelist.tra | 5 +- env/mistral.dkrz.de/shell~ | 40 --- src/CMakeLists.txt | 1 + src/Makefile | 0 src/Makefile_hlrn | 0 src/Makefile_ollie | 0 src/cpl_driver.F90 | 0 src/fesom_main.F90 | 0 src/fesom_module.F90 | 0 src/fvom_init.F90 | 0 src/gen_bulk_formulae.F90 | 0 src/gen_comm.F90 | 0 src/gen_forcing_couple.F90 | 0 src/gen_forcing_init.F90 | 0 src/gen_halo_exchange.F90 | 0 src/gen_interpolation.F90 | 0 src/gen_model_setup.F90 | 0 src/gen_modules_clock.F90 | 0 src/gen_modules_config.F90 | 0 src/gen_modules_forcing.F90 | 0 src/gen_modules_read_NetCDF.F90 | 0 src/gen_modules_rotate_grid.F90 | 0 src/ice_EVP.F90 | 16 +- src/ice_fct.F90 | 0 src/ice_maEVP.F90 | 2 +- src/ice_modules.F90 | 0 src/ice_oce_coupling.F90 | 31 ++- src/ice_setup_step.F90 | 0 src/ice_thermo_oce.F90 | 0 src/icepack_drivers/associate_mesh.h | 15 +- src/icepack_drivers/icedrv_io.F90 | 145 +++++------ src/icepack_drivers/icedrv_set.F90 | 32 +-- src/icepack_drivers/icedrv_step.F90 | 35 ++- src/icepack_drivers/icedrv_transfer.F90 | 10 +- src/io_meandata.F90 | 17 +- src/io_restart.F90 | 9 +- src/oce_ale_mixing_kpp.F90 | 0 src/oce_dyn.F90 | 0 src/oce_local.F90 | 0 src/oce_mesh.F90 | 0 src/oce_modules.F90 | 0 src/oce_muscl_adv.F90 | 0 src/oce_setup_step.F90 | 0 src/oce_tracer_mod.F90 | 0 work/job_albedo | 32 ++- work/job_albedo_chain | 333 ------------------------ work/job_bsc | 36 --- work/job_ecmwf | 47 ---- work/job_ini_albedo | 4 +- work/job_ini_juwels | 32 --- work/job_ini_levante | 38 --- work/job_ini_mistral | 26 -- work/job_ini_ollie | 31 --- work/job_jureca | 32 --- work/job_juwels | 42 --- work/job_levante | 38 --- work/job_mistral | 79 ------ work/job_ollie | 34 --- work/job_ollie_chain | 317 ---------------------- work/job_ubuntu | 20 -- 66 files changed, 235 insertions(+), 1330 deletions(-) delete mode 100644 env/mistral.dkrz.de/shell~ mode change 100755 => 100644 src/Makefile mode change 100755 => 100644 src/Makefile_hlrn mode change 100755 => 100644 src/Makefile_ollie mode change 100755 => 100644 src/cpl_driver.F90 mode change 100755 => 100644 src/fesom_main.F90 mode change 100755 => 100644 src/fesom_module.F90 mode change 100755 => 100644 src/fvom_init.F90 mode change 100755 => 100644 src/gen_bulk_formulae.F90 mode change 100755 => 100644 src/gen_comm.F90 mode change 100755 => 100644 src/gen_forcing_couple.F90 mode change 100755 => 100644 src/gen_forcing_init.F90 mode change 100755 => 100644 src/gen_halo_exchange.F90 mode change 100755 => 100644 src/gen_interpolation.F90 mode change 100755 => 100644 src/gen_model_setup.F90 mode change 100755 => 100644 src/gen_modules_clock.F90 mode change 100755 => 100644 src/gen_modules_config.F90 mode change 100755 => 100644 src/gen_modules_forcing.F90 mode change 100755 => 100644 src/gen_modules_read_NetCDF.F90 mode change 100755 => 100644 src/gen_modules_rotate_grid.F90 mode change 100755 => 100644 src/ice_EVP.F90 mode change 100755 => 100644 src/ice_fct.F90 mode change 100755 => 100644 src/ice_modules.F90 mode change 100755 => 100644 src/ice_oce_coupling.F90 mode change 100755 => 100644 src/ice_setup_step.F90 mode change 100755 => 100644 src/ice_thermo_oce.F90 mode change 100755 => 100644 src/oce_ale_mixing_kpp.F90 mode change 100755 => 100644 src/oce_dyn.F90 mode change 100755 => 100644 src/oce_local.F90 mode change 100755 => 100644 src/oce_mesh.F90 mode change 100755 => 100644 src/oce_modules.F90 mode change 100755 => 100644 src/oce_muscl_adv.F90 mode change 100755 => 100644 src/oce_setup_step.F90 mode change 100755 => 100644 src/oce_tracer_mod.F90 delete mode 100755 work/job_albedo_chain delete mode 100755 work/job_bsc delete mode 100755 work/job_ecmwf delete mode 100755 work/job_ini_juwels delete mode 100755 work/job_ini_levante delete mode 100755 work/job_ini_mistral delete mode 100755 work/job_ini_ollie delete mode 100755 work/job_jureca delete mode 100755 work/job_juwels delete mode 100755 work/job_levante delete mode 100755 work/job_mistral delete mode 100755 work/job_ollie delete mode 100755 work/job_ollie_chain delete mode 100755 work/job_ubuntu diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ec4985d5..0fa690273 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ set(TOPLEVEL_DIR ${CMAKE_CURRENT_LIST_DIR}) set(FESOM_COUPLED OFF CACHE BOOL "compile fesom standalone or with oasis support (i.e. coupled)") set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs FESOM_COUPLED to work)") set(CRAY OFF CACHE BOOL "compile with cray ftn") -set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea ice column physics.") +set(USE_ICEPACK ON CACHE BOOL "compile fesom with the Icepack modules for sea ice column physics.") set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results") #set(VERBOSE OFF CACHE BOOL "toggle debug output") diff --git a/config/namelist.config b/config/namelist.config index ec665f53f..29ccd85fb 100644 --- a/config/namelist.config +++ b/config/namelist.config @@ -5,31 +5,31 @@ runid='fesom' / ×tep -step_per_day=32 !96 !96 !72 !72 !45 !72 !96 -run_length=62 !62 !62 !62 !28 +step_per_day=96 !96 !96 !72 !72 !45 !72 !96 +run_length=20 !62 !62 !62 !28 run_length_unit='y' ! y, m, d, s / &clockinit ! the model starts at timenew=0.0 daynew=1 -yearnew=1958 +yearnew=1960 / &paths -MeshPath='/work/ollie/projects/clidyn/FESOM2/meshes/core2/' -ClimateDataPath='/work/ollie/projects/clidyn/FESOM2/hydrography/phc3.0/' -ResultPath='../result_tmp/' +MeshPath='/albedo/work/user/pscholz/mesh_fesom2.0/core2_srt_dep@node/' +ClimateDataPath='/albedo/work/projects/p_fesom/FROM-OLLIE/FESOM2/hydrography/phc3.0/' +ResultPath='/albedo/scratch/user/fkauker/fesom2/core2_head-ip/' / &restart_log restart_length=1 ! --> do netcdf restart ( only required for d,h,s cases, y, m take 1) -restart_length_unit='y' !output period: y, d, h, s, off +restart_length_unit='m' !output period: y, d, h, s, off raw_restart_length=1 ! --> do core dump restart -raw_restart_length_unit='y' ! e.g. y, d, h, s, off +raw_restart_length_unit='m' ! e.g. y, d, h, s, off bin_restart_length=1 ! --> do derived type binary restart -bin_restart_length_unit='y' ! e.g. y, d, h, s, off -logfile_outfreq=960 !in logfile info. output frequency, # steps +bin_restart_length_unit='m' ! e.g. y, d, h, s, off +logfile_outfreq=96 !in logfile info. output frequency, # steps / &ale_def @@ -62,6 +62,7 @@ flag_debug=.false. / &machine -n_levels=2 -n_part= 12, 36 ! 432 number of partitions on each hierarchy level +n_levels=1 +n_part= 508 ! 508 number of partitions on each hierarchy level / + diff --git a/config/namelist.cvmix b/config/namelist.cvmix index 18a90c979..2bf06d887 100644 --- a/config/namelist.cvmix +++ b/config/namelist.cvmix @@ -26,7 +26,9 @@ idemix_mu0 = 0.33333333 ! dissipation parameter (dimensionless) idemix_sforcusage = 0.2 idemix_n_hor_iwe_prop_iter = 5 ! iterations for contribution from horiz. wave propagation idemix_surforc_file = '/work/ollie/clidyn/forcing/IDEMIX/fourier_smooth_2005_cfsr_inert_rgrid.nc' +idemix_surforc_vname= 'var706' idemix_botforc_file = '/work/ollie/clidyn/forcing/IDEMIX/tidal_energy_gx1v6_20090205_rgrid.nc' +idemix_botforc_vname= 'wave_dissipation' / ! namelist for PP diff --git a/config/namelist.dyn b/config/namelist.dyn index fb7919588..43ab9d0ba 100644 --- a/config/namelist.dyn +++ b/config/namelist.dyn @@ -4,7 +4,8 @@ visc_gamma1 = 0.1 ! [nodim], for computation of the flow aware viscosity visc_gamma2 = 0.285 ! [s/m], is only used in easy backscatter option visc_easybsreturn= 1.5 -opt_visc = 5 +opt_visc = 5 +check_opt_visc=.true. ! check if optvisc=5 is valid based on ratio resol/rossbyR ! 5=Kinematic (easy) Backscatter ! 6=Biharmonic flow aware (viscosity depends on velocity Laplacian) ! 7=Biharmonic flow aware (viscosity depends on velocity differences) @@ -20,5 +21,6 @@ use_wsplit = .false. ! Switch for implicite/explicte splitting of vert. veloci wsplit_maxcfl= 1.0 ! maximum allowed CFL criteria in vertical (0.5 < w_max_cfl < 1.) ! in older FESOM it used to be w_exp_max=1.e-3 ldiag_KE=.false. ! activates energy diagnostics +AB_order=2 / diff --git a/config/namelist.icepack b/config/namelist.icepack index ed0dd4d4c..e5c48426e 100644 --- a/config/namelist.icepack +++ b/config/namelist.icepack @@ -110,12 +110,16 @@ !!!!!!!!!! Icepack output namelist !!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +&nml_listsize + io_listsize = 100 +/ + &nml_list_icepack io_list_icepack = 'aicen ',1, 'm', 4, ! Sea ice concentration 'vicen ',1, 'm', 4, ! Volume per unit area of ice 'vsnon ',1, 'm', 4, ! Volume per unit area of snow - !'Tsfcn ',1, 'm', 4, ! Sea ice surf. temperature - !'iagen ',1, 'm', 4, ! Sea ice age + 'Tsfcn ',1, 'm', 4, ! Sea ice surf. temperature + 'iagen ',1, 'm', 4, ! Sea ice age !'FYn ',1, 'm', 4, ! First year ice !'lvln ',1, 'm', 4, ! Ridged sea ice area and volume !'pond_cesmn',1, 'm', 4, ! Melt ponds area, volume and refrozen lid thickness @@ -129,10 +133,10 @@ io_list_icepack = 'aicen ',1, 'm', 4, ! Sea ice concentration !'aice ',1, 'm', 4, ! Sea ice concentration !'vice ',1, 'm', 4, ! Volume per unit area of ice !'vsno ',1, 'm', 4, ! Volume per unit area of snow - !'Tsfc ',1, 'm', 4, ! Sea ice surf. temperature - !'iage ',1, 'm', 4, ! Sea ice age - !'FY ',1, 'm', 4, ! First year ice - !'lvl ',1, 'm', 4, ! Ridged sea ice area and volume + 'Tsfc ',1, 'm', 4, ! Sea ice surf. temperature + 'iage ',1, 'm', 4, ! Sea ice age + 'FY ',1, 'm', 4, ! First year ice + 'lvl ',1, 'm', 4, ! Ridged sea ice area and volume !'pond_cesm ',1, 'm', 4, ! Melt ponds area, volume and refrozen lid thickness !'pond_topo ',1, 'm', 4, ! Melt ponds area, volume and refrozen lid thickness !'pond_lvl ',1, 'm', 4, ! Melt ponds area, volume and refrozen lid thickness diff --git a/config/namelist.io b/config/namelist.io index f49eb58ef..8e6d4357b 100644 --- a/config/namelist.io +++ b/config/namelist.io @@ -12,7 +12,7 @@ ldiag_extflds =.false. / &nml_general -io_listsize =100 !number of streams to allocate. shallbe large or equal to the number of streams in &nml_list +io_listsize =120 !number of streams to allocate. shallbe large or equal to the number of streams in &nml_list vec_autorotate =.false. / @@ -21,22 +21,25 @@ vec_autorotate =.false. ! for 'fer_C', 'bolus_u', 'bolus_v', 'bolus_w', 'fer_K' to work Fer_GM must be .true. otherwise no output ! 'otracers' - all other tracers if applicable ! for 'dMOC' to work ldiag_dMOC must be .true. otherwise no output +! for 'utemp', 'vtemp', 'usalt', 'vsalt' output, set ldiag_trflx=.true. &nml_list io_list = 'sst ',1, 'm', 4, 'sss ',1, 'm', 4, 'ssh ',1, 'm', 4, - 'uice ',1, 'd', 4, - 'vice ',1, 'd', 4, + 'uice ',1, 'm', 4, + 'vice ',1, 'm', 4, 'a_ice ',1, 'm', 4, 'm_ice ',1, 'm', 4, 'm_snow ',1, 'm', 4, - 'MLD1 ',1, 'm', 4, + 'flice ',1, 'm', 4, + 'MLD1 ',1, 'm', 4, 'MLD2 ',1, 'm', 4, 'MLD3 ',1, 'm', 4, 'tx_sur ',1, 'm', 4, 'ty_sur ',1, 'm', 4, - 'temp ',1, 'y', 4, - 'salt ',1, 'y', 8, + 'temp ',1, 'm', 4, + 'salt ',1, 'm', 4, + 'otracers ',1, 'm', 4, 'N2 ',1, 'y', 4, 'Kv ',1, 'y', 4, 'u ',1, 'y', 4, @@ -48,4 +51,6 @@ io_list = 'sst ',1, 'm', 4, 'bolus_u ',1, 'y', 4, 'bolus_v ',1, 'y', 4, 'bolus_w ',1, 'y', 4, + 'fw ',1, 'm', 4, + 'fh ',1, 'm', 4, / diff --git a/config/namelist.tra b/config/namelist.tra index 9286412bb..ba85fc504 100644 --- a/config/namelist.tra +++ b/config/namelist.tra @@ -4,9 +4,8 @@ num_tracers=100 !number of tracers to allocate. shallbe large or equal to the nu &tracer_list nml_tracer_list = -1 , 'MFCT', 'QR4C', 'FCT ', 1., 1., -2 , 'MFCT', 'QR4C', 'FCT ', 1., 1., -!101, 'UPW1', 'UPW1', 'NON ', 0., 0. +1 , 'MFCT', 'QR4C', 'FCT ', 0., 1., +2 , 'MFCT', 'QR4C', 'FCT ', 0., 1., / &tracer_init3d ! initial conditions for tracers diff --git a/env/mistral.dkrz.de/shell~ b/env/mistral.dkrz.de/shell~ deleted file mode 100644 index 9b1c8acaa..000000000 --- a/env/mistral.dkrz.de/shell~ +++ /dev/null @@ -1,40 +0,0 @@ -# make the contents as shell agnostic as possible so we can include them with bash, zsh and others - -module load gcc/4.8.2 -export LD_LIBRARY_PATH=/sw/rhel6-x64/gcc/gcc-4.8.2/lib64:$LD_LIBRARY_PATH # avoid GLIBCXX_3.4.15 not found error -module unload intel && module load intel/18.0.0 - -export FC=mpiifort CC=mpiicc CXX=mpiicpc; module unload intelmpi && module load intelmpi/2017.0.098 -#export FC=mpif90 CC=mpicc CXX=mpicxx; module load mxm/3.3.3002 fca/2.5.2393 bullxmpi_mlx/bullxmpi_mlx-1.2.8.3 -#export FC=mpif90 CC=mpicc CXX=mpicxx; module load mxm/3.4.3082 fca/2.5.2393 bullxmpi_mlx/bullxmpi_mlx-1.2.9.2 - -# intelmpi settings from DKRZ -export I_MPI_FABRICS=shm:dapl -export I_MPI_FALLBACK=disable -export I_MPI_SLURM_EXT=1 -export I_MPI_LARGE_SCALE_THRESHOLD=8192 # Set to a value larger than the number of your MPI-tasks if you use 8192 or more tasks. -export I_MPI_DYNAMIC_CONNECTION=0 -export DAPL_NETWORK_NODES=$SLURM_NNODES -export DAPL_NETWORK_PPN=$SLURM_NTASKS_PER_NODE -export DAPL_WR_MAX=500 - -# bullxmpi settings from DKRZ -# Settings for Open MPI and MXM (MellanoX Messaging) library -export OMPI_MCA_pml=cm -export OMPI_MCA_mtl=mxm -export OMPI_MCA_mtl_mxm_np=0 -export MXM_RDMA_PORTS=mlx5_0:1 -export MXM_LOG_LEVEL=FATAL -# Disable GHC algorithm for collective communication -export OMPI_MCA_coll=^ghc - -module unload netcdf && module load netcdf_c/4.3.2-gcc48 -module unload cmake && module load cmake -# we will get a segfault at runtime if we use a gcc from any of the provided gcc modules -export PATH=/sw/rhel6-x64/gcc/binutils-2.24-gccsys/bin:${PATH} - -export NETCDF_Fortran_INCLUDE_DIRECTORIES=/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.2-intel14/include -export NETCDF_C_INCLUDE_DIRECTORIES=/sw/rhel6-x64/netcdf/netcdf_c-4.3.2-intel14/include -export NETCDF_CXX_INCLUDE_DIRECTORIES=/sw/rhel6-x64/netcdf/netcdf_cxx-4.2.1-gcc48/include - -export HDF5_C_INCLUDE_DIRECTORIES=/sw/rhel6-x64/hdf5/hdf5-1.8.14-threadsafe-intel14/include diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fb02eab70..db3b20a80 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -152,6 +152,7 @@ if(${CMAKE_Fortran_COMPILER_ID} STREQUAL Intel ) target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2) elseif(${FESOM_PLATFORM_STRATEGY} STREQUAL albedo) target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -O3 -ip -fPIC -qopt-malloc-options=2 -qopt-prefetch=5 -unroll-aggressive) #NEC mpi option + # target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -O0 -g -traceback -check bounds ) # Franks debug options else() target_compile_options(${PROJECT_NAME} PRIVATE -xHost) endif() diff --git a/src/Makefile b/src/Makefile old mode 100755 new mode 100644 diff --git a/src/Makefile_hlrn b/src/Makefile_hlrn old mode 100755 new mode 100644 diff --git a/src/Makefile_ollie b/src/Makefile_ollie old mode 100755 new mode 100644 diff --git a/src/cpl_driver.F90 b/src/cpl_driver.F90 old mode 100755 new mode 100644 diff --git a/src/fesom_main.F90 b/src/fesom_main.F90 old mode 100755 new mode 100644 diff --git a/src/fesom_module.F90 b/src/fesom_module.F90 old mode 100755 new mode 100644 diff --git a/src/fvom_init.F90 b/src/fvom_init.F90 old mode 100755 new mode 100644 diff --git a/src/gen_bulk_formulae.F90 b/src/gen_bulk_formulae.F90 old mode 100755 new mode 100644 diff --git a/src/gen_comm.F90 b/src/gen_comm.F90 old mode 100755 new mode 100644 diff --git a/src/gen_forcing_couple.F90 b/src/gen_forcing_couple.F90 old mode 100755 new mode 100644 diff --git a/src/gen_forcing_init.F90 b/src/gen_forcing_init.F90 old mode 100755 new mode 100644 diff --git a/src/gen_halo_exchange.F90 b/src/gen_halo_exchange.F90 old mode 100755 new mode 100644 diff --git a/src/gen_interpolation.F90 b/src/gen_interpolation.F90 old mode 100755 new mode 100644 diff --git a/src/gen_model_setup.F90 b/src/gen_model_setup.F90 old mode 100755 new mode 100644 diff --git a/src/gen_modules_clock.F90 b/src/gen_modules_clock.F90 old mode 100755 new mode 100644 diff --git a/src/gen_modules_config.F90 b/src/gen_modules_config.F90 old mode 100755 new mode 100644 diff --git a/src/gen_modules_forcing.F90 b/src/gen_modules_forcing.F90 old mode 100755 new mode 100644 diff --git a/src/gen_modules_read_NetCDF.F90 b/src/gen_modules_read_NetCDF.F90 old mode 100755 new mode 100644 diff --git a/src/gen_modules_rotate_grid.F90 b/src/gen_modules_rotate_grid.F90 old mode 100755 new mode 100644 diff --git a/src/ice_EVP.F90 b/src/ice_EVP.F90 old mode 100755 new mode 100644 index 7a1f0ccd7..c956f9dbf --- a/src/ice_EVP.F90 +++ b/src/ice_EVP.F90 @@ -428,11 +428,11 @@ subroutine EVPdynamics(ice, partit, mesh) !_______________________________________________________________ ! Hunke and Dukowicz c*h*p* -#if defined (__icepack) - ice_strength(el) = ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) -#else +! #if defined (__icepack) # frank.kauker@awi.de ice_strength(el) = ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) -#endif +! #else +! ice_strength(el) = ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) +! #endif # frank.kauker@awi.de ice_strength(el) = 0.5_WP*ice_strength(el) !_______________________________________________________________ @@ -481,11 +481,11 @@ subroutine EVPdynamics(ice, partit, mesh) asum = sum(a_ice(elnodes))/3.0_WP ! ===== Hunke and Dukowicz c*h*p* -#if defined (__icepack) - ice_strength(el) = ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) -#else +! #if defined (__icepack) # frank.kauker@awi.de ice_strength(el) = ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) -#endif +! #else +! ice_strength(el) = ice%pstar*msum*exp(-ice%c_pressure*(1.0_WP-asum)) +! #endif # frank.kauker@awi.de ice_strength(el) = 0.5_WP*ice_strength(el) ! use rhs_m and rhs_a for storing the contribution from elevation: diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 old mode 100755 new mode 100644 diff --git a/src/ice_maEVP.F90 b/src/ice_maEVP.F90 index c33ab6a4e..68e6f84b3 100644 --- a/src/ice_maEVP.F90 +++ b/src/ice_maEVP.F90 @@ -159,7 +159,7 @@ subroutine stress_tensor_m(ice, partit, mesh) eps2=eps11(elem)-eps22(elem) ! ====== moduli: - delta=eps1**2+vale*(eps2**2+4.0_WP*eps12(elem)**2) + delta=eps1**2 + vale * ( eps2**2 + 4.0_WP*eps12(elem)**2 ) delta=sqrt(delta) #if defined (__icepack) diff --git a/src/ice_modules.F90 b/src/ice_modules.F90 old mode 100755 new mode 100644 diff --git a/src/ice_oce_coupling.F90 b/src/ice_oce_coupling.F90 old mode 100755 new mode 100644 index ca20df96c..457a6884d --- a/src/ice_oce_coupling.F90 +++ b/src/ice_oce_coupling.F90 @@ -322,18 +322,35 @@ subroutine oce_fluxes(ice, dynamics, tracers, partit, mesh) vsno_out=m_snow, & fhocn_tot_out=net_heat_flux, & fresh_tot_out=fresh_wa_flux, & + dhs_dt_out=thdgrsn, & + dhi_dt_out=thdgr, & fsalt_out=real_salt_flux, & - dhi_dt_out=thdgrsn, & - dhs_dt_out=thdgr, & evap_ocn_out=evaporation ) - heat_flux(:) = - net_heat_flux(:) - water_flux(:) = - (fresh_wa_flux(:)/1000.0_WP) - runoff(:) + ! write(*,*) "************************" + ! write(*,*) "min/max a_ice = ", minval(a_ice), maxval(a_ice) + ! write(*,*) "min/max net_heat_flux = ", minval(net_heat_flux), maxval(net_heat_flux) + ! write(*,*) "min/max fresh_wa_flux = ", minval(fresh_wa_flux), maxval(fresh_wa_flux) + ! write(*,*) "min/max real_salt_flux = ", minval(real_salt_flux), maxval(real_salt_flux) + ! write(*,*) "min/max evaporation = ", minval(evaporation), maxval(evaporation) + ! write(*,*) "-------------------------" + ! frank.kauker@awi.de - ! Evaporation - evaporation(:) = - evaporation(:) / 1000.0_WP - ice_sublimation(:) = 0.0_WP +!$OMP PARALLEL DO + do n=1, myDim_nod2d+eDim_nod2d + heat_flux(n) = - net_heat_flux(n) + water_flux(n) = - (fresh_wa_flux(n)/1000.0_WP) - runoff(n) + ! Evaporation + evaporation(n) = - evaporation(n) / 1000.0_WP + ice_sublimation(n) = 0.0_WP + end do +!$OMP END PARALLEL DO + ! write(*,*) "min/max heat_flux = ", minval(heat_flux), maxval(heat_flux) + ! write(*,*) "min/max water_flux = ", minval(water_flux), maxval(water_flux) + ! write(*,*) "min/max evaporation = ", minval(evaporation), maxval(evaporation) + ! write(*,*) "************************" + call init_flux_atm_ocn() #else diff --git a/src/ice_setup_step.F90 b/src/ice_setup_step.F90 old mode 100755 new mode 100644 diff --git a/src/ice_thermo_oce.F90 b/src/ice_thermo_oce.F90 old mode 100755 new mode 100644 diff --git a/src/icepack_drivers/associate_mesh.h b/src/icepack_drivers/associate_mesh.h index c9c789c7a..e370a5617 100644 --- a/src/icepack_drivers/associate_mesh.h +++ b/src/icepack_drivers/associate_mesh.h @@ -7,8 +7,6 @@ real(kind=WP) , pointer :: ocean_areawithcav integer , pointer :: nl integer , pointer :: nn_size - - real(kind=WP), dimension(:,:), pointer :: coord_nod2D, geo_coord_nod2D integer, dimension(:,:) , pointer :: elem2D_nodes integer, dimension(:,:) , pointer :: edges @@ -62,7 +60,7 @@ ocean_areawithcav => mesh%ocean_areawithcav nl => mesh%nl nn_size => mesh%nn_size - +! needed to specify dimensions myDim_nod2D => p_partit%myDim_nod2D eDim_nod2D => p_partit%eDim_nod2D myDim_elem2D => p_partit%myDim_elem2D @@ -71,7 +69,6 @@ eXDim_elem2D => p_partit%eXDim_elem2D myDim_edge2D => p_partit%myDim_edge2D eDim_edge2D => p_partit%eDim_edge2D - coord_nod2D(1:2,1:myDim_nod2D+eDim_nod2D) => mesh%coord_nod2D geo_coord_nod2D(1:2,1:myDim_nod2D+eDim_nod2D) => mesh%geo_coord_nod2D elem2D_nodes(1:3, 1:myDim_elem2D+eDim_elem2D+eXDim_elem2D) => mesh%elem2D_nodes @@ -84,9 +81,9 @@ edge_cross_dxdy(1:4,1:myDim_edge2D+eDim_edge2D) => mesh%edge_cross_dx elem_cos(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D) => mesh%elem_cos metric_factor(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D) => mesh%metric_factor elem_neighbors(1:3,1:myDim_elem2D) => mesh%elem_neighbors -nod_in_elem2D => mesh%nod_in_elem2D ! (maxval(rmax),myDim_nod2D+eDim_nod2D) -x_corners => mesh%x_corners ! (myDim_nod2D, maxval(rmax)) -y_corners => mesh%y_corners ! (myDim_nod2D, maxval(rmax)) +nod_in_elem2D => mesh%nod_in_elem2D ! (maxval(rmax),myDim_nod2D+eDim_nod2D) +x_corners => mesh%x_corners ! (myDim_nod2D, maxval(rmax)) +y_corners => mesh%y_corners ! (myDim_nod2D, maxval(rmax)) nod_in_elem2D_num(1:myDim_nod2D+eDim_nod2D) => mesh%nod_in_elem2D_num depth(1:myDim_nod2D+eDim_nod2D) => mesh%depth gradient_vec(1:6,1:myDim_elem2D) => mesh%gradient_vec @@ -94,7 +91,7 @@ gradient_sca(1:6,1:myDim_elem2D) => mesh%gradient_sca bc_index_nod2D(1:myDim_nod2D+eDim_nod2D) => mesh%bc_index_nod2D zbar(1:mesh%nl) => mesh%zbar Z(1:mesh%nl-1) => mesh%Z -elem_depth => mesh%elem_depth ! never used, not even allocated +elem_depth => mesh%elem_depth ! never used, not even allocated nlevels(1:myDim_elem2D+eDim_elem2D+eXDim_elem2D) => mesh%nlevels nlevels_nod2D(1:myDim_nod2D+eDim_nod2D) => mesh%nlevels_nod2D nlevels_nod2D_min(1:myDim_nod2D+eDim_nod2D) => mesh%nlevels_nod2D_min @@ -130,5 +127,3 @@ zbar_n_bot(1:myDim_nod2D+eDim_nod2D) => mesh%zbar_n_bot zbar_e_bot(1:myDim_elem2D+eDim_elem2D) => mesh%zbar_e_bot zbar_n_srf(1:myDim_nod2D+eDim_nod2D) => mesh%zbar_n_srf zbar_e_srf(1:myDim_elem2D+eDim_elem2D) => mesh%zbar_e_srf - - diff --git a/src/icepack_drivers/icedrv_io.F90 b/src/icepack_drivers/icedrv_io.F90 index 7d8bb7c4d..8ef7b6591 100644 --- a/src/icepack_drivers/icedrv_io.F90 +++ b/src/icepack_drivers/icedrv_io.F90 @@ -63,7 +63,7 @@ module subroutine init_io_icepack(mesh) namelist /nml_listsize / io_listsize namelist /nml_list_icepack / io_list_icepack -#include "associate_mesh.h" +#include "associate_mesh.h" ! includes pointers to partit # frank.kauker@awi.de ! Get the tracers information from icepack call icepack_query_tracer_indices(nt_Tsfc_out=nt_Tsfc, nt_sice_out=nt_sice, & @@ -89,29 +89,19 @@ module subroutine init_io_icepack(mesh) tr_bgc_hum_out=tr_bgc_hum) ! OPEN and read namelist for icepack I/O - open( unit=nm_io_unit, file='namelist.io', form='formatted', access='sequential', status='old', iostat=iost ) - if (iost == 0) then - if (mype==0) write(*,*) ' file : ', 'namelist.io',' open ok' - else - if (mype==0) write(*,*) 'ERROR: --> bad opening file : ','namelist.io',' ; iostat=',iost - call par_ex(p_partit%MPI_COMM_FESOM, p_partit%mype) - stop - end if open( unit=nm_icepack_unit, file='namelist.icepack', form='formatted', access='sequential', status='old', iostat=iost ) if (iost == 0) then - if (mype==0) write(*,*) ' file : ', 'namelist.icepack',' open ok' - else - if (mype==0) write(*,*) 'ERROR: --> bad opening file : ','namelist.icepack',' ; iostat=',iost + if (mype==0) write(*,*) ' file : ', 'namelist.icepack',' open ok' + else + if (mype==0) write(*,*) 'ERROR: --> bad opening file : ','namelist.icepack',' ; iostat=',iost call par_ex(p_partit%MPI_COMM_FESOM, p_partit%mype) stop end if - - read(nm_io_unit, nml=nml_listsize, iostat=iost ) + read(nm_icepack_unit, nml=nml_listsize, iostat=iost ) allocate(io_list_icepack(io_listsize)) read(nm_icepack_unit, nml=nml_list_icepack, iostat=iost ) close(nm_icepack_unit) - do i=1, io_listsize if (trim(io_list_icepack(i)%id)=='unknown ') then if (mype==0) write(*,*) 'io_listsize will be changed from ', io_listsize, ' to ', i-1, '!' @@ -119,7 +109,6 @@ module subroutine init_io_icepack(mesh) exit end if end do - do i=1, io_listsize select case (trim(io_list_icepack(i)%id)) case ('aice0 ') @@ -145,7 +134,7 @@ module subroutine init_io_icepack(mesh) case ('Tsfc ') call def_stream2D(nod2D, nx_nh, 'Tsfc', 'sea ice surf. temperature', 'degC', trcr(:,nt_Tsfc), io_list_icepack(i)%freq, io_list_icepack(i)%unit, io_list_icepack(i)%precision, p_partit, mesh) case ('Tsfcn ') - call def_stream3D((/ncat, nod2D/), (/ncat, nx_nh/), 'Tsfcn', 'sea ice surf. temperature', 'degC', trcrn(:,nt_Tsfc,:), io_list_icepack(i)%freq, io_list_icepack(i)%unit, io_list_icepack(i)%precision, p_partit, mesh) + call def_stream3D((/ncat, nod2D/), (/ncat, nx_nh/), 'Tsfcn', 'sea ice surf. temperature', 'degC', trcrn(:,nt_Tsfc,:), io_list_icepack(i)%freq, io_list_icepack(i)%unit, io_list_icepack(i)%precision, p_partit, mesh, .true.) case ('strength ') call def_stream2D(nod2D, nx_nh, 'strength', 'sea ice strength', 'N', strength(:), io_list_icepack(i)%freq, io_list_icepack(i)%unit, io_list_icepack(i)%precision, p_partit, mesh) ! If the following tracers are not defined they will not be outputed @@ -279,7 +268,7 @@ module subroutine init_restart_icepack(year, mesh) use mod_mesh use g_config, only: runid, ResultPath - use io_restart, only: ip_id, def_variable_2d, def_dim + ! frank.kauker@awi.de use io_restart, only: ip_id, def_variable_2d, def_dim implicit none @@ -341,9 +330,9 @@ module subroutine init_restart_icepack(year, mesh) write(cyear,'(i4)') year ! Create an icepack restart file ! Only serial output implemented so far - ip_id%filename=trim(ResultPath)//trim(runid)//'.'//cyear//'.icepack.restart.nc' - if (ip_id%is_in_use) return - ip_id%is_in_use=.true. + ! frank.kauker@awi.de ip_id%filename=trim(ResultPath)//trim(runid)//'.'//cyear//'.icepack.restart.nc' + ! frank.kauker@awi.de if (ip_id%is_in_use) return + ! frank.kauker@awi.de ip_id%is_in_use=.true. ! Define the dimensions of the netCDF file ! @@ -353,8 +342,8 @@ module subroutine init_restart_icepack(year, mesh) ! reconstructed after the restart. Multidimensional variables would solve ! this. - call def_dim(ip_id, 'node', nod2D) ! Number of nodes - call def_dim(ip_id, 'ncat', ncat) ! Number of thickness classes + ! frank.kauker@awi.de call def_dim(ip_id, 'node', nod2D) ! Number of nodes + ! frank.kauker@awi.de call def_dim(ip_id, 'ncat', ncat) ! Number of thickness classes ! Define the netCDF variables for surface ! and vertically constant fields @@ -363,73 +352,79 @@ module subroutine init_restart_icepack(year, mesh) ! 3D restart fields (ncat) !----------------------------------------------------------------- - call def_variable_2d(ip_id, 'aicen', (/nod2D, ncat/), 'sea ice concentration', 'none', aicen(:,:)); - call def_variable_2d(ip_id, 'vicen', (/nod2D, ncat/), 'volum per unit area of ice', 'm', vicen(:,:)); - call def_variable_2d(ip_id, 'vsnon', (/nod2D, ncat/), 'volum per unit area of snow', 'm', vsnon(:,:)); - call def_variable_2d(ip_id, 'Tsfc', (/nod2D, ncat/), 'sea ice surf. temperature', 'degC', trcrn(:,nt_Tsfc,:)); + ! frank.kauker@awi.de + ! call def_variable_2d(ip_id, 'aicen', (/nod2D, ncat/), 'sea ice concentration', 'none', aicen(:,:)); + ! call def_variable_2d(ip_id, 'vicen', (/nod2D, ncat/), 'volum per unit area of ice', 'm', vicen(:,:)); + ! call def_variable_2d(ip_id, 'vsnon', (/nod2D, ncat/), 'volum per unit area of snow', 'm', vsnon(:,:)); + ! call def_variable_2d(ip_id, 'Tsfc', (/nod2D, ncat/), 'sea ice surf. temperature', 'degC', trcrn(:,nt_Tsfc,:)); - if (tr_iage) then - call def_variable_2d(ip_id, 'iage', (/nod2D, ncat/), 'sea ice age', 's', trcrn(:,nt_iage,:)); - end if + ! if (tr_iage) then + ! call def_variable_2d(ip_id, 'iage', (/nod2D, ncat/), 'sea ice age', 's', trcrn(:,nt_iage,:)); + ! end if - if (tr_FY) then - call def_variable_2d(ip_id, 'FY', (/nod2D, ncat/), 'first year ice', 'none', trcrn(:,nt_FY,:)); - end if + ! if (tr_FY) then + ! call def_variable_2d(ip_id, 'FY', (/nod2D, ncat/), 'first year ice', 'none', trcrn(:,nt_FY,:)); + ! end if - if (tr_lvl) then - call def_variable_2d(ip_id, 'alvl', (/nod2D, ncat/), 'ridged sea ice area', 'none', trcrn(:,nt_alvl,:)); - call def_variable_2d(ip_id, 'vlvl', (/nod2D, ncat/), 'ridged sea ice volume', 'm', trcrn(:,nt_vlvl,:)); - end if + ! if (tr_lvl) then + ! call def_variable_2d(ip_id, 'alvl', (/nod2D, ncat/), 'ridged sea ice area', 'none', trcrn(:,nt_alvl,:)); + ! call def_variable_2d(ip_id, 'vlvl', (/nod2D, ncat/), 'ridged sea ice volume', 'm', trcrn(:,nt_vlvl,:)); + ! end if - if (tr_pond_cesm) then - call def_variable_2d(ip_id, 'apnd', (/nod2D, ncat/), 'melt pond area fraction', 'none', trcrn(:,nt_apnd,:)); - call def_variable_2d(ip_id, 'hpnd', (/nod2D, ncat/), 'melt pond depth', 'm', trcrn(:,nt_hpnd,:)); - end if + ! if (tr_pond_cesm) then + ! call def_variable_2d(ip_id, 'apnd', (/nod2D, ncat/), 'melt pond area fraction', 'none', trcrn(:,nt_apnd,:)); + ! call def_variable_2d(ip_id, 'hpnd', (/nod2D, ncat/), 'melt pond depth', 'm', trcrn(:,nt_hpnd,:)); + ! end if - if (tr_pond_topo) then - call def_variable_2d(ip_id, 'apnd', (/nod2D, ncat/), 'melt pond area fraction', 'none', trcrn(:,nt_apnd,:)); - call def_variable_2d(ip_id, 'hpnd', (/nod2D, ncat/), 'melt pond depth', 'm', trcrn(:,nt_hpnd,:)); - call def_variable_2d(ip_id, 'ipnd', (/nod2D, ncat/), 'melt pond refrozen lid thickness', 'm', trcrn(:,nt_ipnd,:)); - end if + ! if (tr_pond_topo) then + ! call def_variable_2d(ip_id, 'apnd', (/nod2D, ncat/), 'melt pond area fraction', 'none', trcrn(:,nt_apnd,:)); + ! call def_variable_2d(ip_id, 'hpnd', (/nod2D, ncat/), 'melt pond depth', 'm', trcrn(:,nt_hpnd,:)); + ! call def_variable_2d(ip_id, 'ipnd', (/nod2D, ncat/), 'melt pond refrozen lid thickness', 'm', trcrn(:,nt_ipnd,:)); + ! end if - if (tr_pond_lvl) then - call def_variable_2d(ip_id, 'apnd', (/nod2D, ncat/), 'melt pond area fraction', 'none', trcrn(:,nt_apnd,:)); - call def_variable_2d(ip_id, 'hpnd', (/nod2D, ncat/), 'melt pond depth', 'm', trcrn(:,nt_hpnd,:)); - call def_variable_2d(ip_id, 'ipnd', (/nod2D, ncat/), 'melt pond refrozen lid thickness', 'm', trcrn(:,nt_ipnd,:)); - call def_variable_2d(ip_id, 'ffracn', (/nod2D, ncat/), 'fraction of fsurfn over pond used to melt ipond', 'none', ffracn); - call def_variable_2d(ip_id, 'dhsn', (/nod2D, ncat/), 'depth difference for snow on sea ice and pond ice', 'm', dhsn); - end if + ! if (tr_pond_lvl) then + ! call def_variable_2d(ip_id, 'apnd', (/nod2D, ncat/), 'melt pond area fraction', 'none', trcrn(:,nt_apnd,:)); + ! call def_variable_2d(ip_id, 'hpnd', (/nod2D, ncat/), 'melt pond depth', 'm', trcrn(:,nt_hpnd,:)); + ! call def_variable_2d(ip_id, 'ipnd', (/nod2D, ncat/), 'melt pond refrozen lid thickness', 'm', trcrn(:,nt_ipnd,:)); + ! call def_variable_2d(ip_id, 'ffracn', (/nod2D, ncat/), 'fraction of fsurfn over pond used to melt ipond', 'none', ffracn); + ! call def_variable_2d(ip_id, 'dhsn', (/nod2D, ncat/), 'depth difference for snow on sea ice and pond ice', 'm', dhsn); + ! end if - if (tr_brine) then - call def_variable_2d(ip_id, 'fbri', (/nod2D, ncat/), 'volume fraction of ice with dynamic salt', 'none', trcrn(:,nt_fbri,:)); - call def_variable_2d(ip_id, 'first_ice', (/nod2D, ncat/), 'distinguishes ice that disappears', 'logical', first_ice_real(:,:)); - end if + ! if (tr_brine) then + ! call def_variable_2d(ip_id, 'fbri', (/nod2D, ncat/), 'volume fraction of ice with dynamic salt', 'none', trcrn(:,nt_fbri,:)); + ! call def_variable_2d(ip_id, 'first_ice', (/nod2D, ncat/), 'distinguishes ice that disappears', 'logical', first_ice_real(:,:)); + ! end if + ! frank.kauker@awi.de !----------------------------------------------------------------- ! 4D restart fields, written as layers of 3D !----------------------------------------------------------------- ! Ice - - do k = 1,nilyr - write(trname,'(A6,i1)') 'sicen_', k - write(longname,'(A21,i1)') 'sea ice salinity lyr:', k - units='psu' - call def_variable_2d(ip_id, trim(trname), (/nod2D, ncat/), trim(longname), trim(units), trcrn(:,nt_sice+k-1,:)); - write(trname,'(A6,i1)') 'qicen_', k - write(longname,'(A21,i1)') 'sea ice enthalpy lyr:', k - units='J/m3' - call def_variable_2d(ip_id, trim(trname), (/nod2D, ncat/), trim(longname), trim(units), trcrn(:,nt_qice+k-1,:)); - end do + + ! frank.kauker@awi.de + ! do k = 1,nilyr + ! write(trname,'(A6,i1)') 'sicen_', k + ! write(longname,'(A21,i1)') 'sea ice salinity lyr:', k + ! units='psu' + ! call def_variable_2d(ip_id, trim(trname), (/nod2D, ncat/), trim(longname), trim(units), trcrn(:,nt_sice+k-1,:)); + ! write(trname,'(A6,i1)') 'qicen_', k + ! write(longname,'(A21,i1)') 'sea ice enthalpy lyr:', k + ! units='J/m3' + ! call def_variable_2d(ip_id, trim(trname), (/nod2D, ncat/), trim(longname), trim(units), trcrn(:,nt_qice+k-1,:)); + ! end do + ! frank.kauker@awi.de ! Snow - - do k = 1,nslyr - write(trname,'(A6,i1)') 'qsnon_', k - write(longname,'(A18,i1)') 'snow enthalpy lyr:', k - units='J/m3' - call def_variable_2d(ip_id, trim(trname), (/nod2D, ncat/), trim(longname), trim(units), trcrn(:,nt_qsno+k-1,:)); - end do + + ! frank.kauker@awi.de + ! do k = 1,nslyr + ! write(trname,'(A6,i1)') 'qsnon_', k + ! write(longname,'(A18,i1)') 'snow enthalpy lyr:', k + ! units='J/m3' + ! call def_variable_2d(ip_id, trim(trname), (/nod2D, ncat/), trim(longname), trim(units), trcrn(:,nt_qsno+k-1,:)); + ! end do + ! frank.kauker@awi.de ! ! All the other 4D tracers (linked to aerosols and biogeochemistry) are at the diff --git a/src/icepack_drivers/icedrv_set.F90 b/src/icepack_drivers/icedrv_set.F90 index dbfc55a1d..70642db77 100644 --- a/src/icepack_drivers/icedrv_set.F90 +++ b/src/icepack_drivers/icedrv_set.F90 @@ -8,7 +8,7 @@ submodule (icedrv_main) icedrv_set - use icepack_intfc, only: icepack_init_parameters + use icepack_intfc, only: icepack_init_parameters use icepack_intfc, only: icepack_init_tracer_flags use icepack_intfc, only: icepack_init_tracer_sizes use icepack_intfc, only: icepack_init_tracer_indices @@ -226,22 +226,32 @@ module subroutine set_icepack(ice, partit) trbgcs = 0 ! set to 1 for skeletal layer tracers (needs ndtd = 1 ! dynamic time steps per thermodynamic time step + !----------------------------------------------------------------- + ! Derived quantities used by the icepack model + !----------------------------------------------------------------- + call icedrv_system_init(partit) + p_partit => partit + nx = p_partit%myDim_nod2D + p_partit%eDim_nod2D + nx_elem = p_partit%myDim_elem2D + p_partit%eDim_elem2D + nx_nh = p_partit%myDim_nod2D + nx_elem_nh = p_partit%myDim_elem2D + mype = p_partit%mype + !----------------------------------------------------------------- ! Read namelist env_nml !----------------------------------------------------------------- - open (nu_nml, file=nml_filename, status='old',iostat=nml_error) + open (nu_nml, file=nml_filename, status='old', iostat=nml_error) if (nml_error /= 0) then nml_error = -1 else nml_error = 1 end if - do while (nml_error > 0) - if (mype == 0) print*,'Reading namelist file ',nml_filename - + do while (nml_error > 0) + if (mype == 0) print*,'Reading namelist file ',nml_filename if (mype == 0) print*,'Reading env_nml' - read(nu_nml, nml=env_nml,iostat=nml_error) + read(nu_nml, nml=env_nml ,iostat=nml_error) if (nml_error /= 0) exit end do @@ -252,16 +262,6 @@ module subroutine set_icepack(ice, partit) close(nu_nml) end if - !----------------------------------------------------------------- - ! Derived quantities used by the icepack model - !----------------------------------------------------------------- - call icedrv_system_init(partit) - p_partit => partit - nx = p_partit%myDim_nod2D + p_partit%eDim_nod2D - nx_elem = p_partit%myDim_elem2D + p_partit%eDim_elem2D - nx_nh = p_partit%myDim_nod2D - nx_elem_nh = p_partit%myDim_elem2D - mype = p_partit%mype ncat = nicecat ! number of categories nfsd = nfsdcat ! number of floe size categories nilyr = nicelyr ! number of ice layers per category diff --git a/src/icepack_drivers/icedrv_step.F90 b/src/icepack_drivers/icedrv_step.F90 index 5139454bf..f85e61789 100644 --- a/src/icepack_drivers/icedrv_step.F90 +++ b/src/icepack_drivers/icedrv_step.F90 @@ -772,12 +772,15 @@ end subroutine step_radiation ! Allows heat storage in ocean for uncoupled runs. ! - subroutine ocean_mixed_layer (dt) + subroutine ocean_mixed_layer (ice, dt) use icepack_intfc, only: icepack_ocn_mixed_layer, icepack_atm_boundary - + use mod_ice + implicit none + type(t_ice), target, intent(inout) :: ice + real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -852,7 +855,8 @@ subroutine ocean_mixed_layer (dt) !----------------------------------------------------------------- do i = 1, nx - call ocn_mixed_layer_icepack( alvdr_ocn=alvdr_ocn(i), swvdr=swvdr(i), & + call ocn_mixed_layer_icepack( ice, & + alvdr_ocn=alvdr_ocn(i), swvdr=swvdr(i), & alidr_ocn=alidr_ocn(i), swidr=swidr(i), & alvdf_ocn=alvdf_ocn(i), swvdf=swvdf(i), & alidf_ocn=alidf_ocn(i), swidf=swidf(i), & @@ -878,7 +882,7 @@ end subroutine ocean_mixed_layer !======================================================================= - subroutine ocn_mixed_layer_icepack( & + subroutine ocn_mixed_layer_icepack(ice, & alvdr_ocn, swvdr, & alidr_ocn, swidr, & alvdf_ocn, swvdf, & @@ -896,9 +900,9 @@ subroutine ocn_mixed_layer_icepack( & frzmlt, fsalt, & sss) - use i_therm_param, only: emiss_wat use g_forcing_param, only: use_virt_salt - + use mod_ice + implicit none real (kind=dbl_kind), intent(in) :: & @@ -952,6 +956,11 @@ subroutine ocn_mixed_layer_icepack( & character(len=*),parameter :: subname='(icepack_ocn_mixed_layer)' + type(t_ice), target, intent(inout) :: ice + real(kind=WP), pointer :: emiss_wat + + emiss_wat => ice%thermo%emiss_wat + call icepack_query_parameters( Tffresh_out=Tffresh, Lfresh_out=Lfresh, & stefan_boltzmann_out=stefan_boltzmann, & ice_ref_salinity_out=ice_ref_salinity, & @@ -992,15 +1001,19 @@ end subroutine ocn_mixed_layer_icepack !======================================================================= - subroutine coupling_prep(dt) + subroutine coupling_prep(ice, dt) - ! local variables + use mod_ice implicit none + + type(t_ice), target, intent(inout) :: ice real (kind=dbl_kind), intent(in) :: & dt ! time step + ! local variables + integer (kind=int_kind) :: & n , & ! thickness category index i , & ! horizontal index @@ -1029,7 +1042,7 @@ subroutine coupling_prep(dt) frzmlt_init (i) = frzmlt(i) enddo - call ocean_mixed_layer (dt) ! ocean surface fluxes and sst + call ocean_mixed_layer (ice, dt) ! ocean surface fluxes and sst !----------------------------------------------------------------- ! Aggregate albedos @@ -1207,7 +1220,7 @@ module subroutine step_icepack(ice, mesh, time_evp, time_advec, time_therm) !----------------------------------------------------------------- dhi_dt(:) = ( vice(:) - dhi_dt(:) ) / dt - dhs_dt(:) = ( vsno(:) - dhi_dt(:) ) / dt + dhs_dt(:) = ( vsno(:) - dhs_dt(:) ) / dt ! clean up, update tendency diagnostics @@ -1294,7 +1307,7 @@ module subroutine step_icepack(ice, mesh, time_evp, time_advec, time_therm) ! get ready for coupling and the next time step !----------------------------------------------------------------- - call coupling_prep (dt) + call coupling_prep (ice, dt) !----------------------------------------------------------------- ! icepack timing diff --git a/src/icepack_drivers/icedrv_transfer.F90 b/src/icepack_drivers/icedrv_transfer.F90 index 15d36eb1b..5ca9c1348 100644 --- a/src/icepack_drivers/icedrv_transfer.F90 +++ b/src/icepack_drivers/icedrv_transfer.F90 @@ -18,12 +18,6 @@ module subroutine fesom_to_icepack(ice, mesh) use g_forcing_param, only: ncar_bulk_z_wind, ncar_bulk_z_tair, & ncar_bulk_z_shum use g_sbf, only: l_mslp -! use i_arrays, only: S_oc_array, T_oc_array, & ! Ocean and sea ice fields -! u_w, v_w, & -! stress_atmice_x, stress_atmice_y -! ! u_ice, v_ice, & - -! use i_param, only: cd_oce_ice ! Sea ice parameters use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted use icepack_intfc, only: icepack_query_parameters use icepack_intfc, only: icepack_sea_freezing_temperature @@ -66,6 +60,7 @@ module subroutine fesom_to_icepack(ice, mesh) real(kind=WP), dimension(:), pointer :: u_ice, v_ice, S_oc_array, T_oc_array, & u_w, v_w, stress_atmice_x, stress_atmice_y #include "associate_mesh.h" + u_ice => ice%uice(:) v_ice => ice%vice(:) S_oc_array => ice%srfoce_salt(:) @@ -74,13 +69,14 @@ module subroutine fesom_to_icepack(ice, mesh) v_w => ice%srfoce_v(:) stress_atmice_x => ice%stress_atmice_x(:) stress_atmice_y => ice%stress_atmice_y(:) + ! Ice uvel(:) = u_ice(:) vvel(:) = v_ice(:) ! Atmosphere - + T_air(:) = Tair(:) + 273.15_dbl_kind Qa(:) = shum(:) uatm(:) = u_wind(:) diff --git a/src/io_meandata.F90 b/src/io_meandata.F90 index 2ce9ddb8e..1f6633c10 100644 --- a/src/io_meandata.F90 +++ b/src/io_meandata.F90 @@ -1045,17 +1045,25 @@ subroutine write_mean(entry, entry_index) ! ! !_______________________________________________________________________________ -subroutine update_means +subroutine update_means(mype) implicit none type(Meandata), pointer :: entry integer :: n integer :: I, J - + integer :: mype + DO n=1, io_NSTREAMS entry=>io_stream(n) + ! frank.kauker@awi.de + ! if (mype==0) write(*,*) "++++++++++++++++++++++++++++++" + ! if (mype==0) write(*,*) "io_stream: n = ", n, entry%name, i_real4, i_real8 + ! if (mype==0) write(*,*) "entry%accuracy = ", entry%accuracy + ! if (mype==0) write(*,*) "entry%flip = ", entry%flip + !_____________ compute in 8 byte accuracy ______________________________ IF (entry%accuracy == i_real8) then + ! write(*,*) "size(entry%local_values_r8) = ", size(entry%local_values_8) IF (entry%flip) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(I,J) DO J=1, size(entry%local_values_r8,dim=2) @@ -1076,7 +1084,8 @@ subroutine update_means !_____________ compute in 4 byte accuracy ______________________________ ELSE IF (entry%accuracy == i_real4) then - IF (entry%flip) then + ! write(*,*) "size(entry%local_values_r4) = ", size(entry%local_values_4) + IF (entry%flip) then !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(I,J) DO J=1, size(entry%local_values_r4,dim=2) DO I=1, size(entry%local_values_r4,dim=1) @@ -1148,7 +1157,7 @@ subroutine output(istep, ice, dynamics, tracers, partit, mesh) !___________________________________________________________________________ !PS if (partit%flag_debug .and. partit%mype==0) print *, achar(27)//'[33m'//' -I/O-> call update_means'//achar(27)//'[0m' - call update_means + call update_means(partit%mype) !___________________________________________________________________________ ! loop over defined streams diff --git a/src/io_restart.F90 b/src/io_restart.F90 index 7d8cbcd55..32cc6789d 100644 --- a/src/io_restart.F90 +++ b/src/io_restart.F90 @@ -155,7 +155,7 @@ end subroutine ini_ice_io subroutine restart(istep, l_read, which_readr, ice, dynamics, tracers, partit, mesh) #if defined(__icepack) - icepack restart not merged here ! produce a compiler error if USE_ICEPACK=ON; todo: merge icepack restart from 68d8b8b + use icedrv_main, only: init_restart_icepack #endif use fortran_utils @@ -225,9 +225,14 @@ subroutine restart(istep, l_read, which_readr, ice, dynamics, tracers, partit, m if (.not. l_read) then call ini_ocean_io(yearnew, dynamics, tracers, partit, mesh) if (use_ice) call ini_ice_io (yearnew, ice, partit, mesh) - else +#if defined(__icepack) + if (use_ice) call init_restart_icepack(yearnew, mesh) +#endif else call ini_ocean_io(yearold, dynamics, tracers, partit, mesh) if (use_ice) call ini_ice_io (yearold, ice, partit, mesh) +#if defined(__icepack) + if (use_ice) call init_restart_icepack(yearold, mesh) +#endif end if !___READING OF RESTART________________________________________________________ diff --git a/src/oce_ale_mixing_kpp.F90 b/src/oce_ale_mixing_kpp.F90 old mode 100755 new mode 100644 diff --git a/src/oce_dyn.F90 b/src/oce_dyn.F90 old mode 100755 new mode 100644 diff --git a/src/oce_local.F90 b/src/oce_local.F90 old mode 100755 new mode 100644 diff --git a/src/oce_mesh.F90 b/src/oce_mesh.F90 old mode 100755 new mode 100644 diff --git a/src/oce_modules.F90 b/src/oce_modules.F90 old mode 100755 new mode 100644 diff --git a/src/oce_muscl_adv.F90 b/src/oce_muscl_adv.F90 old mode 100755 new mode 100644 diff --git a/src/oce_setup_step.F90 b/src/oce_setup_step.F90 old mode 100755 new mode 100644 diff --git a/src/oce_tracer_mod.F90 b/src/oce_tracer_mod.F90 old mode 100755 new mode 100644 diff --git a/work/job_albedo b/work/job_albedo index 0c032ae7c..5364258f8 100755 --- a/work/job_albedo +++ b/work/job_albedo @@ -1,18 +1,21 @@ #!/bin/bash -#SBATCH --account=clidyn.p_fesom # edit your account -#SBATCH --job-name=run +#___SET SLURM OPTIONS___________________________________________________________ +#SBATCH --account=meereis.meereis # edit your account +#SBATCH --job-name=head #SBATCH --partition=mpp -#SBATCH --time=01:00:00 -#SBATCH --qos=12h -#SBATCH --nodes=48 # Number of tasks (MPI) tasks to be launched -#SBATCH --tasks-per-node 128 # using all 128CPus we exceed the maxload limit (>130) of many used nodes when us parallel I/O +# disable hyperthreading +#SBATCH --hint=nomultithread +#SBATCH --nodes=4 # Number of tasks (MPI) tasks to be launched +#SBATCH --tasks-per-node 127 # using all 128CPus we exceed the maxload limit (>130) of many used nodes #SBATCH --cpus-per-task 1 +#SBATCH --time=12:00:00 +#SBATCH --qos=12h #SBATCH -o fesom2_%x_%j.out #SBATCH -e fesom2_%x_%j.out -# disable hyperthreading -#SBATCH --hint=nomultithread +##SBATCH --mail-type=END +##SBATCH --mail-user=Frank.Kauker@awi.de module purge source ../env/albedo/shell @@ -39,10 +42,13 @@ fname="fesom2_${SLURM_JOB_NAME}_${jobid}.out" #___PUT JOB IN QUEUE____________________________________________________________ date srun --mpi=pmi2 ./fesom.x >> ${fname} +err_srun=$? +echo -e "\033[33m --> err_srun=${err_srun} \033[0m" date -#qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_ollie -#fi +#___SETUP JOBCHAIN______________________________________________________________ +if [ ${err_srun} -eq 0 ]; then + sbatch --dependency=aftercorr:$SLURM_JOB_ID job_albedo +fi + + diff --git a/work/job_albedo_chain b/work/job_albedo_chain deleted file mode 100755 index 1bcf0eefc..000000000 --- a/work/job_albedo_chain +++ /dev/null @@ -1,333 +0,0 @@ -#!/bin/bash -#___SET SLURM OPTIONS___________________________________________________________ -#SBATCH --account=clidyn.p_fesom # edit your account -#SBATCH --job-name=chain -#SBATCH --partition=mpp - -# disable hyperthreading -#SBATCH --hint=nomultithread -#SBATCH --nodes=48 # Number of tasks (MPI) tasks to be launched -#SBATCH --tasks-per-node 127 # using all 128CPus we exceed the maxload limit (>130) of many used nodes -#SBATCH --cpus-per-task 1 - -#SBATCH --time=12:00:00 -#SBATCH --qos=12h - -#SBATCH -o fesom2_%x_%j.out -#SBATCH -e fesom2_%x_%j.out - -##SBATCH --mail-type=END -##SBATCH --mail-user=Patrick.Scholz@awi.de - - -#___DEFAULT INPUT_______________________________________________________________ -# how many job chains should be applied -chain_n=3 # number chain cycles -chain_s=1 # starting chain id - -# time frame of model simulation -# ___COREv2___ -year_s=1948 -year_e=2009 -# ___JRA55____ -#year_s=1958 -#year_e=2018 - -prescribe_rlen=0 # run length in namelist.config --> if 0 value from namelist.config is taken -fedit=1 - -#___HELP OUTPUT_________________________________________________________________ -script_name=job_albedo_chain -function usage { - echo "usage: $script_name [-cn ...] [-cs ...] [-ys ...] [-ye ...] [-wcl ...]" - echo " -cn number of chain cylces (default: 3)" - echo " -cs starting chain id (default: 1)" - echo " -ys starting year of model simulation (default: 1948)" - echo " -ye ending year of model simulation (default: 2009)" - echo " -h display help" - echo " -rl prescribe used run_length" - echo " -noedit no automatic editing of namelist.config" - echo - echo " --> for changing the wall-clock-time interactively use " - echo " sbatch --time=00:10:00 job_ollie_chain ..." - echo " --> for changing the number of task interactively use " - echo " sbatch --ntask=288 job_ollie_chain ..." - echo " --> for changing the job name interactively use " - echo " sbatch --job-name=whatever job_ollie_chain ..." - exit 1 -} - -#___OVERRIDE DEFAULT INPUT BY COMMANDLINE INPUT_________________________________ -while [ "$1" != "" ]; do - case $1 in - -cn | -chain_n ) shift ; chain_n=$1 ;; - -cs | -chain_s ) shift ; chain_s=$1 ;; - -ys | -year_s ) shift ; year_s=$1 ;; - -ye | -year_e ) shift ; year_e=$1 ;; - -rl | --run_length ) shift ; prescribe_rlen=$1 ;; - -noedit | --noedit ) fedit=0 ;; - -h | --help ) usage ; exit ;; - esac - shift -done - -#___EXTRACT WALL-CLOCK-TIME FROM JOBINFO________________________________________ -# either setted via #SBATCH time=... or at command line sbatch --time=... job_ollie_chain -# need here to extract to give the next job chain cycle as input -jobinfo=$(scontrol show job $SLURM_JOB_ID) -wcl=$( echo ${jobinfo##*"TimeLimit="} | cut -d " " -f 1 ) -ntasks=$( echo ${jobinfo##*"NumTasks="} | cut -d " " -f 1 ) - -#___SET SLURM OPTIONS___________________________________________________________ -set -x -ulimit -s unlimited -export OMP_NUM_THREADS=1 -source ../env/albedo/shell - -#___SET NAMELIST'S & EXECUTABLE IF NOT ALREADY EXIST____________________________ -ln -s ../bin/fesom.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.dyn . -cp -n ../config/namelist.tra . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.io . -cp -n ../config/namelist.icepack . - -#___SET CHAIN_ID________________________________________________________________ -if [ -f "file_chain_id" ]; then - chain_id=$( file_chain_id -fi - -#___PRINT INPUT INFO____________________________________________________________ -echo -e "\033[1;7;33m_____JOB CHAIN INFO_____________________________________\033[0m" -echo -e "\033[1;33m --> actual chain cycle: $chain_id \033[0m" -echo -e "\033[1;33m --> max. number of chain cycles: $chain_n \033[0m" -echo -e "\033[1;33m --> simulated time range: [ $year_s $year_e] \033[0m" -echo -e "\033[1;33m --> slurm: wall-clock-time = $wcl \033[0m" -echo -e "\033[1;33m --> slurm: ntask = $ntasks \033[0m" -if [ $prescribe_rlen -ne 0 ]; then - echo -e "\033[1;33m -->change run_length = $prescribe_rlen \033[0m" -fi - -#___CREATE SAVE DIR INFRASTRUCTURE______________________________________________ -# extract resultpath from namelist.config -dname_result_link_orig=$(grep "ResultPath=" namelist.config | grep -v '^!' | \ - cut -d "=" -f 2 | \ - cut -d "'" -f 2) -dname_result_link=$(echo ${dname_result_link_orig::-1}) - -# identify higher directory --> ResultPath -dname_result="$(dirname "$dname_result_link")/" - -# if ResultPath directory doesn't exist --> create it -if [ ! -d "${dname_result}" ]; then - echo -e "\033[33m --> ResultPath directory does not exist --> will create it \033[0m" - mkdir "${dname_result}" -fi - -# check if in namelist.config a chain path is given (that mean -# ResultPath=.../ResultDir/chain/) if not change namelist.config accordingly -check=${dname_result_link##*${dname_result}} -if [ $fedit -eq 1 ] && [ $check != "chain" ]; then - echo -e "\033[33m --> replace in namelist.config ResultPath with chain path \033[0m" - dname_result_link="${dname_result}chain" - sed -i "s|${dname_result_link_orig}|${dname_result_link}/|g" namelist.config -fi - -# identify real path in case a link is used -dname_result="$(realpath "$dname_result")/" - -# if directory for chain_id doesn't exist --> create it -if [ ! -d "${dname_result}/${chain_id}" ]; then - echo -e "\033[33m --> chain_id directory does not exist --> will create it \033[0m" - mkdir "${dname_result}/${chain_id}" -fi - -# link directory of chain_id with original linkdirectory from namelist.config -ln -sfn ${dname_result}${chain_id} $dname_result_link - -#___CHECK IF SIMULATION NEEDS TO BE INITIALISED OR CONTINUED____________________ -is_newsimul=1 -if [ -f "$dname_result_link/fesom.clock" ] ; then - aux_yr_clock=$(<${dname_result_link}/fesom.clock) - aux_yr_clock=$(echo ${aux_yr_clock} | cut -d" " -f 6) - if [ $aux_yr_clock -le $year_e ]; then is_newsimul=0 ; fi - - #___________________________________________________________________________ - if [ $fedit -eq 1 ] ; then - if [ $is_newsimul -eq 0 ] ; then - year_d=$(( $year_e - $aux_yr_clock + 1 )) - rlen=$(grep "run_length=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1) - # do not prescribe run length in job fle - if [ $prescribe_rlen -eq 0 ] ; then - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $year_d \033[0m" - fi - # prescribe run length in job fle - else - aux_year_d=$prescribe_rlen - if [ $year_d -lt $aux_year_d ] ; then aux_year_d=$year_d ; fi - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$aux_year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $aux_year_d \033[0m" - fi - fi - fi - fi -else - #___________________________________________________________________________ - # set model run length in job_script and change namelist.config accordingly - # to match - if [ $fedit -eq 1 ] ; then - year_d=$(( $year_e - $year_s + 1 )) - rlen=$(grep "run_length=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1) - # do not prescribe run length in job fle - if [ $prescribe_rlen -eq 0 ] ; then - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $year_d \033[0m" - fi - # prescribe run length in job file - else - aux_year_d=$prescribe_rlen - if [ $year_d -lt $aux_year_d ] ; then aux_year_d=$year_d ; fi - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$aux_year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $aux_year_d \033[0m" - fi - fi - fi -fi - -#___CREATE CLOCK & RESTART INFRASTRUCTURE FOR COLD/WARM START___________________ -# only touch clock file when a new simulation is supposed to start, if an old one -# should be continued dont touch it -if [ $is_newsimul -eq 1 ] ; then - - # --> make cold start - if [ $chain_id -eq 1 ] ; then - #_______________________________________________________________________ - # create cold start clock file - echo -e "\033[33m --> create cold start clock file \033[0m" - printf "0 1 ${year_s}\n0 1 ${year_s}" > $dname_result_link/fesom.clock - - #_______________________________________________________________________ - # in case yearnew in namelist.config was changed from 1948 - yearnew=$(grep "yearnew=" namelist.config | cut -d "=" -f 2) - if [ $yearnew -ne $year_s ]; then - sed -i " s/yearnew=$yearnew/yearnew=$year_s/" namelist.config - fi - - #___BACKUP NAMELIST.* FILES INTO RESULT DIRECTORY_______________________ - cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.cvmix ${dname_result}/. - cp fesom.x ${dname_result}/. - - #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________ - if [ ! -d "${dname_result}/src" ]; then mkdir "${dname_result}/src" ; fi - cp ../src/*.F90 ${dname_result}/src/. - - - # --> make warm start - else - #_______________________________________________________________________ - # create warm start clock file - stepperday=$(grep "step_per_day=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1 ) - aux_sec=$(( 86400 - 86400 / $stepperday )) - aux_day=365 - aux_yr=$(( $year_s - 1 )) - echo -e "\033[33m --> create warm start clock file \033[0m" - printf "${aux_sec} ${aux_day} ${aux_yr}\n0 1 ${year_s}" > $dname_result_link/fesom.clock - - #_______________________________________________________________________ - # chain id from previous spinup cycle - prev_chain_id=$(( $chain_id - 1 )) - - #_______________________________________________________________________ - # copy restart ocean files/directories from previous spinup cycle - prev_rfile=${dname_result}${prev_chain_id}/fesom.${year_e}.oce.restart - if [ -d "${prev_rfile}" ]; then - cp -r ${prev_rfile} ${dname_result_link}/fesom.${aux_yr}.oce.restart - elif [ -f "${prev_rfile}.nc" ]; then - cp ${prev_rfile}.nc ${dname_result_link}/fesom.${aux_yr}.oce.restart.nc - else - echo -e "\033[1;31m --> ERROR: could not find ocean restart file \033[0m" - exit - fi - - # copy restart ice files/files/directories from previous spinup cycle - prev_rfile=${dname_result}${prev_chain_id}/fesom.${year_e}.ice.restart - if [ -d "${prev_rfile}" ]; then - cp -r ${prev_rfile} ${dname_result_link}/fesom.${aux_yr}.ice.restart - elif [ -f "${prev_rfile}.nc" ]; then - cp ${prev_rfile}.nc ${dname_result_link}/fesom.${aux_yr}.ice.restart.nc - else - echo -e "\033[1;31m --> ERROR: could not find ice restart file \033[0m" - exit - fi - - #_______________________________________________________________________ - # adapt year new in namelist.config otherwise fesom is not doing a - # restart - aux=$(grep "yearnew=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1 ) - sed -i " s/yearnew=$aux/yearnew=$aux_yr/" namelist.config - - #_______________________________________________________________________ - # backup namelist.* & fesom.x in case they dont exist - if [ ! -f "${dname_result}/namelist.config" ]; then - cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.tra namelist.dyn namelist.cvmix ${dname_result}/. - fi - if [ ! -f "${dname_result}/fesom.x" ]; then - cp fesom.x ${dname_result}/. - fi - #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________ - if [ ! -d "${dname_result}/src" ]; then mkdir "${dname_result}/src" ; fi - cp ../src/*.F90 ${dname_result}/src/. - fi -fi - -#___DETERMINE SLURM JOBID+OUTPUTFILE____________________________________________ -jobid=$(echo $SLURM_JOB_ID | cut -d"." -f1) -fname="fesom2_${SLURM_JOB_NAME}_${jobid}.out" - -#___PUT JOB IN QUEUE____________________________________________________________ -date -srun --mpi=pmi2 ./fesom.x >> ${fname} -err_srun=$? -echo -e "\033[33m --> err_srun=${err_srun} \033[0m" -date - -#___SHOW JOB INFORMATION________________________________________________________ -scontrol show job $SLURM_JOB_ID - -#___SETUP JOBCHAIN______________________________________________________________ -# check if complete cycle is finished only than increase chain_id -aux_yr_clock=$(<${dname_result_link}/fesom.clock) -aux_yr_clock=$(echo ${aux_yr_clock} | cut -d" " -f 6) - -# setup next chain job via dependence -if [ ${err_srun} -eq 0 ]; then - if [ $aux_yr_clock -lt $year_e ] || [ ${chain_id} -lt ${chain_n} ]; then - # aftercorr:job_id --> A task of this job array can begin execution after - # the corresponding task ID in the specified job has completed successfully - # (ran to completion with an exit code of zero). - echo -e "\033[33m --> setup next chain cycle \033[0m" - sbatch --time=$wcl --ntasks=$ntasks --job-name=${SLURM_JOB_NAME} --dependency=aftercorr:$SLURM_JOB_ID $script_name \ - -cn $chain_n -cs $chain_s -ys $year_s -ye $year_e -rl $prescribe_rlen - fi -fi - -#___CHECK FOR COMPLETNES________________________________________________________ -# check if complete cycle is finished only than increase chain_id -if [ $aux_yr_clock -gt $year_e ] && [ ${chain_id} -lt ${chain_n} ] ; then - chain_id=$(( $chain_id + 1 )) - echo $chain_id > file_chain_id -fi - diff --git a/work/job_bsc b/work/job_bsc deleted file mode 100755 index 3ff4b20a8..000000000 --- a/work/job_bsc +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH --ntasks=288 -#SBATCH --time=00:30:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH --qos=debug - -module purge -module load intel/2017.4 impi/2017.4 mkl/2017.4 bsc/1.0 netcdf/4.2 -ulimit -s unlimited - - -set -x -echo Submitted job: $jobid -squeue -u $USER - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -date -srun --mpi=pmi2 fesom.x > "fesom2.0.out" -date - -# qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_mistral -#fi - diff --git a/work/job_ecmwf b/work/job_ecmwf deleted file mode 100755 index 287f8e442..000000000 --- a/work/job_ecmwf +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -#PBS -S /usr/bin/ksh -#PBS -N fesom2-LandG -#PBS -q np -#PBS -l EC_total_tasks=288 - -# optionally, specifiy that no OpenMP is used -#PBS -l EC_threads_per_task=1 - -#PBS -l EC_hyperthreading=1 -#PBS -l EC_user_defined_priority=99 -#PBS -l walltime=00:57:00 - -##PBS -j oe #join out and err -#PBD -n -#PBS -o /scratch/rd/natr/run_core2_LandG/pbs.out -#PBS -e /scratch/rd/natr/run_core2_LandG/pbs.err - -#PBS -m abe -#PBS -M thomas.rackow@ecmwf.int - -#queue suitable for target processors min/max processors per node memory limit wall-clock -#np parallel MOM+CN 1/72 not shared 72 120 GB 48 hours - -path=`pwd` -echo Initial path: $path - -cd /scratch/rd/natr/run_core2_LandG/ - -# debug -set -x - -cp $HOME/fesom2/bin/fesom.x . # -# did manually -#cp -n $HOME/fesom2/config/namelist.config . # -#cp -n $HOME/fesom2/config/namelist.cvmix . # -#cp -n $HOME/fesom2/config/namelist.forcing . # -#cp -n $HOME/fesom2/config/namelist.oce . # -#cp -n $HOME/fesom2/config/namelist.io . # -#cp -n $HOME/fesom2/config/namelist.ice . # -#cp -n $HOME/fesom2/config/namelist.tra . # -#cp -n $HOME/fesom2/config/namelist.dyn . # - -date -echo tasks_per_node, total_tasks, HT: $EC_tasks_per_node $EC_total_tasks $EC_hyperthreads -aprun -N $EC_tasks_per_node -n $EC_total_tasks -j $EC_hyperthreads ./fesom.x > "fesom2.out" -date diff --git a/work/job_ini_albedo b/work/job_ini_albedo index d610663d7..a8ae43c62 100755 --- a/work/job_ini_albedo +++ b/work/job_ini_albedo @@ -1,5 +1,5 @@ #!/bin/bash -#SBATCH --account=clidyn.p_fesom # edit your account +#SBATCH --account=meereis.meereis # edit your account #SBATCH --job-name=ini #SBATCH --partition=mpp #SBATCH --time=00:30:00 @@ -15,7 +15,7 @@ ulimit -s unlimited source ../env/albedo/shell ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . +#cp -n ../config/namelist.config . #cp -n ../config/namelist.forcing . #cp -n ../config/namelist.oce . #cp -n ../config/namelist.ice . diff --git a/work/job_ini_juwels b/work/job_ini_juwels deleted file mode 100755 index b53e1c235..000000000 --- a/work/job_ini_juwels +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH --ntasks=1 -#SBATCH --time=00:30:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH --partition=batch - -module load CMake Intel IntelMPI imkl netCDF netCDF-Fortran - -set -x - -ulimit -s unlimited - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -date -srun --mpi=pmi2 ./fesom_ini.x > "fesom_ini.out" -date - -#qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_ollie -#fi diff --git a/work/job_ini_levante b/work/job_ini_levante deleted file mode 100755 index c84232fab..000000000 --- a/work/job_ini_levante +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ref -#SBATCH -p compute -#SBATCH --ntasks-per-node=1 -#SBATCH --ntasks=1 -#SBATCH --time=01:00:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH -A ab0995 - -source /sw/etc/profile.levante -source ../env/levante.dkrz.de/shell - -ulimit -s 102400 - -echo Submitted job: $jobid -squeue -u $USER - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.icepack . - -date -srun -l fesom_ini.x > "fesom_ini.out" -date - -# qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_mistral -#fi - diff --git a/work/job_ini_mistral b/work/job_ini_mistral deleted file mode 100755 index 16acdb48a..000000000 --- a/work/job_ini_mistral +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH -p compute2 -#SBATCH --ntasks=1 -#SBATCH --time=00:15:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH -Aab0246 -#SBATCH --mail-type=FAIL - -set -x - -ulimit -s unlimited - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -date -srun --mpi=pmi2 --ntasks=1 ./fesom_ini.x > "fesom_ini.out" -date diff --git a/work/job_ini_ollie b/work/job_ini_ollie deleted file mode 100755 index 693296bad..000000000 --- a/work/job_ini_ollie +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0_INI -#SBATCH -p mpp -#SBATCH --ntasks=32 -#SBATCH --time=12:00:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out - -set -x - -ulimit -s unlimited - -module purge -module load intel.compiler -module load intel.mpi -module load netcdf/4.4.0_intel - -ln -s ../bin/fesom_ini.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -date -srun --mpi=pmi2 --ntasks=1 ./fesom_ini.x > "fvom_ini.out" -date - diff --git a/work/job_jureca b/work/job_jureca deleted file mode 100755 index 057927b60..000000000 --- a/work/job_jureca +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH --ntasks=36 -#SBATCH --time=00:30:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH --partition=batch - -module load CMake Intel IntelMPI imkl netCDF netCDF-Fortran - -set -x - -ulimit -s unlimited - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -date -srun --mpi=pmi2 ./fesom.x > "fesom2.0.out" -date - -#qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_ollie -#fi diff --git a/work/job_juwels b/work/job_juwels deleted file mode 100755 index e05e70dfc..000000000 --- a/work/job_juwels +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fes_core -#SBATCH --ntasks-per-node=48 -#SBATCH --ntasks=288 -#SBATCH --time=00:30:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH --partition=batch -#SBATCH -A hhb19 - -module --force purge -module use /gpfs/software/juwels/otherstages -module load Stages/2019a -module load StdEnv -module load Intel/2019.3.199-GCC-8.3.0 IntelMPI/2018.5.288 imkl/2019.3.199 -# if you use ParaStation MPI -#module load Intel/2019.3.199-GCC-8.3.0 ParaStationMPI/5.4 imkl/2019.5.281 -module load netCDF/4.6.3 -module load netCDF-Fortran/4.4.5 - -set -x - -ulimit -s unlimited - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -date -srun --mpi=pmi2 ./fesom.x > "fesom2.0.out" -date - -#qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_ollie -#fi diff --git a/work/job_levante b/work/job_levante deleted file mode 100755 index 3e45afcb0..000000000 --- a/work/job_levante +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ref -#SBATCH -p compute -#SBATCH --ntasks-per-node=128 -#SBATCH --ntasks=512 -#SBATCH --time=08:00:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH -A ab0995 - -source /sw/etc/profile.levante -source ../env/levante.dkrz.de/shell - -ulimit -s 102400 - -echo Submitted job: $jobid -squeue -u $USER - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.icepack . - -date -srun -l fesom.x > "fesom2.0.out" -date - -# qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_mistral -#fi - diff --git a/work/job_mistral b/work/job_mistral deleted file mode 100755 index 9d2403b09..000000000 --- a/work/job_mistral +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH -p compute2,compute -#SBATCH --ntasks-per-node=36 -#SBATCH --ntasks=7560 -#SBATCH --time=00:30:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -#SBATCH -A ab0995 - -ulimit -s unlimited -whichMPI=INTELMPI - -if test "${whichMPI}" == "INTELMPI"; then -# intelmpi settings from DKRZ -export I_MPI_FABRICS=shm:dapl -export I_MPI_FALLBACK=disable -export I_MPI_SLURM_EXT=1 -export I_MPI_LARGE_SCALE_THRESHOLD=8192 # set to a value larger than the number of MPI-tasks used !!! -export I_MPI_DYNAMIC_CONNECTION=1 -export I_MPI_CHECK_DAPL_PROVIDER_COMPATIBILITY=0 -export I_MPI_HARD_FINALIZE=1 -fi - -if test "${whichMPI}" == "BULLMPI"; then -export OMPI_MCA_pml=cm # sets the point-to-point management layer -export OMPI_MCA_mtl=mxm # sets the matching transport layer (MPI-2 one-sided comm.) -export MXM_RDMA_PORTS=mlx5_0:1 -fi - -# openmpi settings from DKRZ (note, that some of above variables will be redefined) -if test "${whichMPI}" == "OPENMPI"; then -export OMPI_MCA_pml=cm # sets the point-to-point management layer -export OMPI_MCA_mtl=mxm # sets the matching transport layer (MPI-2 one-sided comm.) -export MXM_RDMA_PORTS=mlx5_0:1 -export MXM_LOG_LEVEL=ERROR -export MXM_HANDLE_ERRORS=bt -export UCX_HANDLE_ERRORS=bt - -# enable HCOLL based collectives -export OMPI_MCA_coll=^fca # disable FCA for collective MPI routines -export OMPI_MCA_coll_hcoll_enable=1 # enable HCOLL for collective MPI routines -export OMPI_MCA_coll_hcoll_priority=95 -export OMPI_MCA_coll_hcoll_np=8 # use HCOLL for all communications with more than 8 tasks -export HCOLL_MAIN_IB=mlx5_0:1 -export HCOLL_ENABLE_MCAST=1 -export HCOLL_ENABLE_MCAST_ALL=1 - -# disable specific HCOLL functions (strongly depends on the application) -export HCOLL_ML_DISABLE_BARRIER=1 -export HCOLL_ML_DISABLE_IBARRIER=1 -export HCOLL_ML_DISABLE_BCAST=1 -export HCOLL_ML_DISABLE_REDUCE=1 -fi - -set -x -echo Submitted job: $jobid -squeue -u $USER - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.icepack . - -date -srun --mpi=pmi2 fesom.x > "fesom2.0.out" -date - -# qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_mistral -#fi - diff --git a/work/job_ollie b/work/job_ollie deleted file mode 100755 index ac2516115..000000000 --- a/work/job_ollie +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=fesom2.0 -#SBATCH -p mpp -#SBATCH --ntasks=288 -#SBATCH --time=00:05:00 -#SBATCH -o slurm-out.out -#SBATCH -e slurm-err.out -module load intel.compiler intel.mpi netcdf/4.4.0_intel -module load centoslibs - -set -x - -ulimit -s unlimited - -# determine JOBID -JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1` - -ln -s ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.io . -cp -n ../config/namelist.icepack . - -date -srun --mpi=pmi2 ./fesom.x > "fesom2.0.out" -date - -#qstat -f $PBS_JOBID -#export EXITSTATUS=$? -#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then -#sbatch job_ollie -#fi diff --git a/work/job_ollie_chain b/work/job_ollie_chain deleted file mode 100755 index 8f98f1515..000000000 --- a/work/job_ollie_chain +++ /dev/null @@ -1,317 +0,0 @@ -#!/bin/bash -#___SET SLURM OPTIONS___________________________________________________________ -#SBATCH -J chain -#SBATCH -p mpp -#SBATCH --ntasks=432 -#SBATCH --time=10:00:00 -#SBATCH --mail-type=END -#SBATCH --mail-user=Patrick.Scholz@awi.de -#SBATCH -o fesom2.0_%x_%j.out -#SBATCH -e fesom2.0_%x_%j.out - -## module load intel.compiler intel.mpi netcdf centoslibs - -#___DEFAULT INPUT_______________________________________________________________ -# how many job chains should be applied -chain_n=3 # number chain cycles -chain_s=1 # starting chain id - -# time frame of model simulation -# ___COREv2___ -year_s=1948 -year_e=2009 -# ___JRA55____ -#year_s=1958 -#year_e=2018 - -prescribe_rlen=0 # run length in namelist.config --> if 0 value from namelist.config is taken -fedit=1 - -#___HELP OUTPUT_________________________________________________________________ -script_name=job_ollie_chain -function usage { - echo "usage: $script_name [-cn ...] [-cs ...] [-ys ...] [-ye ...] [-wcl ...]" - echo " -cn number of chain cylces (default: 3)" - echo " -cs starting chain id (default: 1)" - echo " -ys starting year of model simulation (default: 1948)" - echo " -ye ending year of model simulation (default: 2009)" - echo " -h display help" - echo " -rl prescribe used run_length" - echo " -noedit no automatic editing of namelist.config" - echo - echo " --> for changing the wall-clock-time interactively use " - echo " sbatch --time=00:10:00 job_ollie_chain ..." - echo " --> for changing the number of task interactively use " - echo " sbatch --ntask=288 job_ollie_chain ..." - echo " --> for changing the job name interactively use " - echo " sbatch --job-name=whatever job_ollie_chain ..." - exit 1 -} - -#___OVERRIDE DEFAULT INPUT BY COMMANDLINE INPUT_________________________________ -while [ "$1" != "" ]; do - case $1 in - -cn | -chain_n ) shift ; chain_n=$1 ;; - -cs | -chain_s ) shift ; chain_s=$1 ;; - -ys | -year_s ) shift ; year_s=$1 ;; - -ye | -year_e ) shift ; year_e=$1 ;; - -rl | --run_length ) shift ; prescribe_rlen=$1 ;; - -noedit | --noedit ) fedit=0 ;; - -h | --help ) usage ; exit ;; - esac - shift -done - -#___EXTRACT WALL-CLOCK-TIME FROM JOBINFO________________________________________ -# either setted via #SBATCH time=... or at command line sbatch --time=... job_ollie_chain -# need here to extract to give the next job chain cycle as input -jobinfo=$(scontrol show job $SLURM_JOB_ID) -wcl=$( echo ${jobinfo##*"TimeLimit="} | cut -d " " -f 1 ) -ntasks=$( echo ${jobinfo##*"NumTasks="} | cut -d " " -f 1 ) - - -#___SET NAMELIST'S & EXECUTABLE IF NOT ALREADY EXIST____________________________ -set -x -ulimit -s unlimited -ln -s ../bin/fesom.x . # cp -n ../bin/fvom_ini.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . - -#___SET CHAIN_ID________________________________________________________________ -if [ -f "file_chain_id" ]; then - chain_id=$( file_chain_id -fi - -#___PRINT INPUT INFO____________________________________________________________ -echo -e "\033[1;7;33m_____JOB CHAIN INFO_____________________________________\033[0m" -echo -e "\033[1;33m --> actual chain cycle: $chain_id \033[0m" -echo -e "\033[1;33m --> max. number of chain cycles: $chain_n \033[0m" -echo -e "\033[1;33m --> simulated time range: [ $year_s $year_e] \033[0m" -echo -e "\033[1;33m --> slurm: wall-clock-time = $wcl \033[0m" -echo -e "\033[1;33m --> slurm: ntask = $ntasks \033[0m" -if [ $prescribe_rlen -ne 0 ]; then - echo -e "\033[1;33m -->change run_length = $prescribe_rlen \033[0m" -fi - -#___CREATE SAVE DIR INFRASTRUCTURE______________________________________________ -# extract resultpath from namelist.config -dname_result_link_orig=$(grep "ResultPath=" namelist.config | grep -v '^!' | \ - cut -d "=" -f 2 | \ - cut -d "'" -f 2) -dname_result_link=$(echo ${dname_result_link_orig::-1}) - -# identify higher directory --> ResultPath -dname_result="$(dirname "$dname_result_link")/" - -# if ResultPath directory doesn't exist --> create it -if [ ! -d "${dname_result}" ]; then - echo -e "\033[33m --> ResultPath directory does not exist --> will create it \033[0m" - mkdir "${dname_result}" -fi - -# check if in namelist.config a chain path is given (that mean -# ResultPath=.../ResultDir/chain/) if not change namelist.config accordingly -check=${dname_result_link##*${dname_result}} -if [ $fedit -eq 1 ] && [ $check != "chain" ]; then - echo -e "\033[33m --> replace in namelist.config ResultPath with chain path \033[0m" - dname_result_link="${dname_result}chain" - sed -i "s|${dname_result_link_orig}|${dname_result_link}/|g" namelist.config -fi - -# identify real path in case a link is used -dname_result="$(realpath "$dname_result")/" - -# if directory for chain_id doesn't exist --> create it -if [ ! -d "${dname_result}/${chain_id}" ]; then - echo -e "\033[33m --> chain_id directory does not exist --> will create it \033[0m" - mkdir "${dname_result}/${chain_id}" -fi - -# link directory of chain_id with original linkdirectory from namelist.config -ln -sfn ${dname_result}${chain_id} $dname_result_link - -#___CHECK IF SIMULATION NEEDS TO BE INITIALISED OR CONTINUED____________________ -is_newsimul=1 -if [ -f "$dname_result_link/fesom.clock" ] ; then - aux_yr_clock=$(<${dname_result_link}/fesom.clock) - aux_yr_clock=$(echo ${aux_yr_clock} | cut -d" " -f 6) - if [ $aux_yr_clock -le $year_e ]; then is_newsimul=0 ; fi - - #___________________________________________________________________________ - if [ $fedit -eq 1 ] ; then - if [ $is_newsimul -eq 0 ] ; then - year_d=$(( $year_e - $aux_yr_clock + 1 )) - rlen=$(grep "run_length=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1) - # do not prescribe run length in job fle - if [ $prescribe_rlen -eq 0 ] ; then - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $year_d \033[0m" - fi - # prescribe run length in job fle - else - aux_year_d=$prescribe_rlen - if [ $year_d -lt $aux_year_d ] ; then aux_year_d=$year_d ; fi - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$aux_year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $aux_year_d \033[0m" - fi - fi - fi - fi -else - #___________________________________________________________________________ - # set model run length in job_script and change namelist.config accordingly - # to match - if [ $fedit -eq 1 ] ; then - year_d=$(( $year_e - $year_s + 1 )) - rlen=$(grep "run_length=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1) - # do not prescribe run length in job fle - if [ $prescribe_rlen -eq 0 ] ; then - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $year_d \033[0m" - fi - # prescribe run length in job file - else - aux_year_d=$prescribe_rlen - if [ $year_d -lt $aux_year_d ] ; then aux_year_d=$year_d ; fi - if [ $rlen -ne $year_d ] ; then - sed -i " s/run_length=$rlen/run_length=$aux_year_d/" namelist.config - echo -e "\033[1;33m --> change run_length to: $aux_year_d \033[0m" - fi - fi - fi -fi - -#___CREATE CLOCK & RESTART INFRASTRUCTURE FOR COLD/WARM START___________________ -# only touch clock file when a new simulation is supposed to start, if an old one -# should be continued dont touch it -if [ $is_newsimul -eq 1 ] ; then - - # --> make cold start - if [ $chain_id -eq 1 ] ; then - #_______________________________________________________________________ - # create cold start clock file - echo -e "\033[33m --> create cold start clock file \033[0m" - printf "0 1 ${year_s}\n0 1 ${year_s}" > $dname_result_link/fesom.clock - - #_______________________________________________________________________ - # in case yearnew in namelist.config was changed from 1948 - yearnew=$(grep "yearnew=" namelist.config | cut -d "=" -f 2) - if [ $yearnew -ne $year_s ]; then - sed -i " s/yearnew=$yearnew/yearnew=$year_s/" namelist.config - fi - - #___BACKUP NAMELIST.* FILES INTO RESULT DIRECTORY_______________________ - cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.cvmix namelist.tra namelist.dyn ${dname_result}/. - cp fesom.x ${dname_result}/. - - #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________ - if [ ! -d "${dname_result}/src" ]; then mkdir "${dname_result}/src" ; fi - cp ../src/*.F90 ${dname_result}/src/. - - - # --> make warm start - else - #_______________________________________________________________________ - # create warm start clock file - stepperday=$(grep "step_per_day=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1 ) - aux_sec=$(( 86400 - 86400 / $stepperday )) - aux_day=365 - aux_yr=$(( $year_s - 1 )) - echo -e "\033[33m --> create warm start clock file \033[0m" - printf "${aux_sec} ${aux_day} ${aux_yr}\n0 1 ${year_s}" > $dname_result_link/fesom.clock - - #_______________________________________________________________________ - # chain id from previous spinup cycle - prev_chain_id=$(( $chain_id - 1 )) - - #_______________________________________________________________________ - # copy restart ocean files/directories from previous spinup cycle - prev_rfile=${dname_result}${prev_chain_id}/fesom.${year_e}.oce.restart - if [ -d "${prev_rfile}" ]; then - cp -r ${prev_rfile} ${dname_result_link}/fesom.${aux_yr}.oce.restart - elif [ -f "${prev_rfile}.nc" ]; then - cp ${prev_rfile}.nc ${dname_result_link}/fesom.${aux_yr}.oce.restart.nc - else - echo -e "\033[1;31m --> ERROR: could not find ocean restart file \033[0m" - exit - fi - - # copy restart ice files/files/directories from previous spinup cycle - prev_rfile=${dname_result}${prev_chain_id}/fesom.${year_e}.ice.restart - if [ -d "${prev_rfile}" ]; then - cp -r ${prev_rfile} ${dname_result_link}/fesom.${aux_yr}.ice.restart - elif [ -f "${prev_rfile}.nc" ]; then - cp ${prev_rfile}.nc ${dname_result_link}/fesom.${aux_yr}.ice.restart.nc - else - echo -e "\033[1;31m --> ERROR: could not find ice restart file \033[0m" - exit - fi - - #_______________________________________________________________________ - # adapt year new in namelist.config otherwise fesom is not doing a - # restart - aux=$(grep "yearnew=" namelist.config | cut -d "=" -f 2 | cut -d " " -f 1 ) - sed -i " s/yearnew=$aux/yearnew=$aux_yr/" namelist.config - - #_______________________________________________________________________ - # backup namelist.* & fesom.x in case they dont exist - if [ ! -f "${dname_result}/namelist.config" ]; then - cp namelist.config namelist.oce namelist.ice namelist.forcing namelist.io \ - namelist.cvmix namelist.tra namelist.dyn ${dname_result}/. - fi - if [ ! -f "${dname_result}/fesom.x" ]; then - cp fesom.x ${dname_result}/. - fi - #___BACKUP SRC FILES INTO RESULT DIRECTORY______________________________ - if [ ! -d "${dname_result}/src" ]; then mkdir "${dname_result}/src" ; fi - cp ../src/*.F90 ${dname_result}/src/. - fi -fi - -#___DETERMINE SLURM JOBID+OUTPUTFILE____________________________________________ -jobid=$(echo $SLURM_JOB_ID | cut -d"." -f1) -fname="fesom2.0_${SLURM_JOB_NAME}_${jobid}.out" - -#___PUT JOB IN QUEUE____________________________________________________________ -date -srun --mpi=pmi2 ./fesom.x >> ${fname} -err_srun=$? -echo -e "\033[33m --> err_srun=${err_srun} \033[0m" -date - -#___SHOW JOB INFORMATION________________________________________________________ -scontrol show job $SLURM_JOB_ID - -#___SETUP JOBCHAIN______________________________________________________________ -# check if complete cycle is finished only than increase chain_id -aux_yr_clock=$(<${dname_result_link}/fesom.clock) -aux_yr_clock=$(echo ${aux_yr_clock} | cut -d" " -f 6) - -# setup next chain job via dependence -if [ ${err_srun} -eq 0 ]; then - if [ $aux_yr_clock -lt $year_e ] || [ ${chain_id} -lt ${chain_n} ]; then - # aftercorr:job_id --> A task of this job array can begin execution after - # the corresponding task ID in the specified job has completed successfully - # (ran to completion with an exit code of zero). - echo -e "\033[33m --> setup next chain cycle \033[0m" - sbatch --time=$wcl --ntasks=$ntasks --job-name=${SLURM_JOB_NAME} --dependency=aftercorr:$SLURM_JOB_ID $script_name \ - -cn $chain_n -cs $chain_s -ys $year_s -ye $year_e -rl $prescribe_rlen - fi -fi - -#___CHECK FOR COMPLETNES________________________________________________________ -# check if complete cycle is finished only than increase chain_id -if [ $aux_yr_clock -gt $year_e ] && [ ${chain_id} -lt ${chain_n} ] ; then - chain_id=$(( $chain_id + 1 )) - echo $chain_id > file_chain_id -fi - diff --git a/work/job_ubuntu b/work/job_ubuntu deleted file mode 100755 index a5a3466e4..000000000 --- a/work/job_ubuntu +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -e - -ulimit -s unlimited - -export OMP_NUM_THREADS=1 - -ln -sf ../bin/fesom.x . # cp -n ../bin/fesom.x -cp -n ../config/namelist.config . -cp -n ../config/namelist.forcing . -cp -n ../config/namelist.oce . -cp -n ../config/namelist.ice . -cp -n ../config/namelist.io . - -date -time mpirun --allow-run-as-root --mca btl_vader_single_copy_mechanism none -n 2 fesom.x -date - -