update paths in soca scripts and tests (NOAA-EMC#811)

scripts/exgdas_global_marine_analysis_prep.py

@@ -85,9 +85,10 @@ def cice_hist2fms(input_filename, output_filename):
     """
     Simple reformatting utility to allow soca/fms to read CICE's history
     """
+    input_filename_real = os.path.realpath(input_filename)
 
     # open the CICE history file
-    ds = xr.open_dataset(input_filename)
+    ds = xr.open_dataset(input_filename_real)
 
     if 'aicen' in ds.variables and 'hicen' in ds.variables and 'hsnon' in ds.variables:
         logging.info(f"*** Already reformatted, skipping.")
@@ -100,7 +101,8 @@ def cice_hist2fms(input_filename, output_filename):
     ds = ds.rename({'aice_h': 'aicen', 'hi_h': 'hicen', 'hs_h': 'hsnon'})
 
     # Save the new netCDF file
-    ds.to_netcdf(output_filename, mode='w')
+    output_filename_real = os.path.realpath(output_filename)
+    ds.to_netcdf(output_filename_real, mode='w')
 
 
 def test_hist_date(histfile, ref_date):
@@ -280,7 +282,8 @@ def find_clim_ens(input_date):
 
 # concatenate altimeters into one obs space
 # TODO (SAMG)temporary, move this into the obs procecing eventually
-adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt"
+com_obs = os.path.realpath(f"{os.getenv('COM_OBS')}")
+adt_obs = f"{com_obs}/{RUN}.t{cyc}z.adt"
 ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True)
 
 # get the list of observations
@@ -293,7 +296,7 @@ def find_clim_ens(input_date):
 for obs_file in obs_files:
     logging.info(f"******* {obs_file}")
     obs_src = os.path.join(os.getenv('COM_OBS'), obs_file)
-    obs_dst = os.path.join(os.path.abspath(obs_in), obs_file)
+    obs_dst = os.path.join(os.path.realpath(obs_in), obs_file)
     if os.path.exists(obs_src):
         logging.info(f"******* fetching {obs_file}")
         obs_list.append([obs_src, obs_dst])
@@ -319,20 +322,21 @@ def find_clim_ens(input_date):
     for mem in range(1, nmem_ens+1):
         for domain in ['ocn', 'ice']:
             # TODO(Guillaume): make use and define ensemble COM in the j-job
+            ensroot = os.getenv('COM_OCEAN_HISTORY_PREV')
             ensdir = os.path.join(os.getenv('COM_OCEAN_HISTORY_PREV'), '..', '..', '..', '..', '..',
                                   f'enkf{RUN}.{PDY}', f'{gcyc}', f'mem{str(mem).zfill(3)}',
                                   'model_data', longname[domain], 'history')
-            ensdir = os.path.normpath(ensdir)
+            ensdir_real = os.path.realpath(ensdir)
             f009 = f'enkfgdas.t{gcyc}z.{domain}f009.nc'
 
-            fname_in = os.path.abspath(os.path.join(ensdir, f009))
-            fname_out = os.path.abspath(os.path.join(static_ens, domain+"."+str(mem)+".nc"))
+            fname_in = os.path.abspath(os.path.join(ensdir_real, f009))
+            fname_out = os.path.realpath(os.path.join(static_ens, domain+"."+str(mem)+".nc"))
             ens_member_list.append([fname_in, fname_out])
     FileHandler({'copy': ens_member_list}).sync()
 
     # reformat the cice history output
     for mem in range(1, nmem_ens+1):
-        cice_fname = os.path.abspath(os.path.join(static_ens, "ice."+str(mem)+".nc"))
+        cice_fname = os.path.realpath(os.path.join(static_ens, "ice."+str(mem)+".nc"))
         cice_hist2fms(cice_fname, cice_fname)
 else:
     logging.info("---------------- Stage offline ensemble members")
@@ -357,8 +361,8 @@ def find_clim_ens(input_date):
 # copy yaml for grid generation
 
 logging.info(f"---------------- generate gridgen.yaml")
-gridgen_yaml_src = os.path.abspath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml'))
-gridgen_yaml_dst = os.path.abspath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml'))
+gridgen_yaml_src = os.path.realpath(os.path.join(gdas_home, 'parm', 'soca', 'gridgen', 'gridgen.yaml'))
+gridgen_yaml_dst = os.path.realpath(os.path.join(stage_cfg['stage_dir'], 'gridgen.yaml'))
 FileHandler({'copy': [[gridgen_yaml_src, gridgen_yaml_dst]]}).sync()
 
 ################################################################################

scripts/exgdas_global_marine_analysis_vrfy.py

@@ -26,9 +26,9 @@
 import subprocess
 from datetime import datetime, timedelta
 
-comout = os.getenv('COM_OCEAN_ANALYSIS')
-com_ice_history = os.getenv('COM_ICE_HISTORY_PREV')
-com_ocean_history = os.getenv('COM_OCEAN_HISTORY_PREV')
+comout = os.path.realpath(os.getenv('COM_OCEAN_ANALYSIS'))
+com_ice_history = os.path.realpath(os.getenv('COM_ICE_HISTORY_PREV'))
+com_ocean_history = os.path.realpath(os.getenv('COM_OCEAN_HISTORY_PREV'))
 cyc = os.getenv('cyc')
 RUN = os.getenv('RUN')
 gcyc = str((int(cyc) - 6) % 24).zfill(2)

scripts/exglobal_prep_ocean_obs.py

@@ -4,7 +4,7 @@
 from datetime import datetime, timedelta
 import logging
 import os
-import prep_marine_obs
+from soca import prep_marine_obs
 import subprocess
 from wxflow import YAMLFile, save_as_yaml, FileHandler
 

test/soca/gw/CMakeLists.txt

@@ -27,7 +27,7 @@ add_test(NAME test_gdasapp_soca_concatioda
          WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/concatioda)
        set_tests_properties(test_gdasapp_soca_concatioda
       PROPERTIES
-      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
 
 # Clean-up
 add_test(NAME test_gdasapp_soca_run_clean
@@ -78,7 +78,7 @@ foreach(jjob ${jjob_list})
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/testrun)
   set_tests_properties(${test_name}
     PROPERTIES
-    ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+    ENVIRONMENT "PYTHONPATH=${PROJECT_SOURCE_DIR}/ush:${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
 
 
   set(setup "--skip") # Only run the setup of the first test, if not, it will hang
@@ -89,7 +89,7 @@ endforeach()
 set(ctest_list "socahybridweights" "incr_handler" "ens_handler")
 foreach(ctest ${ctest_list})
   set(TEST ${ctest})
-  set(EXEC ${PROJECT_BINARY_DIR}/bin/gdas_${ctest}.x)
+  set(EXEC ${PROJECT_BINARY_DIR}/../bin/gdas_${ctest}.x)
   set(YAML ${PROJECT_SOURCE_DIR}/test/soca/testinput/${ctest}.yaml)
   configure_file(${PROJECT_SOURCE_DIR}/test/soca/gw/run_gdas_apps.yaml.test
                  ${PROJECT_BINARY_DIR}/test/soca/gw/testrun/run_gdas_apps_${ctest}.yaml)
@@ -102,5 +102,5 @@ foreach(ctest ${ctest_list})
 
   set_tests_properties(${test_name}
       PROPERTIES
-      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
+      ENVIRONMENT "PYTHONPATH=${PROJECT_BINARY_DIR}/../lib/python${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}:$ENV{PYTHONPATH}")
 endforeach()

test/soca/gw/prep.sh

@@ -35,7 +35,7 @@ done
 for day in $(seq 1 2 9); do
     cp ${COM}/06/model_data/ocean/history/gdas.t06z.ocnf003.nc \
        ${project_binary_dir}/soca_static/bkgerr/stddev/ocn.ensstddev.fc.2019-04-0${day}T00:00:00Z.PT0S.nc
-    cp ${project_source_dir}/soca/test/Data/72x35x25/ice.bkgerror.nc \
+    cp ${project_source_dir}/sorc/soca/test/Data/72x35x25/ice.bkgerror.nc \
        ${project_binary_dir}/soca_static/bkgerr/stddev/ice.ensstddev.fc.2019-04-0${day}T00:00:00Z.PT0S.nc
 done
 

test/soca/gw/run_jjobs.yaml.test

@@ -21,12 +21,12 @@ gw environement:
     assym_freq: 6
 
   backgrounds:
-    COM_SRC: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/COM
+    COM_SRC: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/COM
 
   working directories:
-    ROTDIRS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs/ROTDIRS
-    EXPDIRS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs/experiments
-    STMP: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/testrun/testjjobs
+    ROTDIRS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs/ROTDIRS
+    EXPDIRS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs/experiments
+    STMP: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/testrun/testjjobs
 
   jedi:
     OOPS_TRACE: 1
@@ -46,10 +46,10 @@ setup_expt config:
     NMEM_ENS: "4"
     DOHYBVAR: "YES"
   ocnanal:
-    SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static
+    SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/soca_static
     CASE_ANL: C48
     SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml
-    COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/test/soca/obs/r2d2-shared
+    COMIN_OBS: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/obs/r2d2-shared
     SOCA_NINNER: 1
     R2D2_OBS_SRC: gdasapp
     R2D2_OBS_DUMP: soca

test/soca/gw/static.sh

@@ -10,12 +10,12 @@ mkdir -p ${soca_static}/INPUT
 mkdir -p ${soca_static}/bump
 mkdir -p ${soca_static}/bkgerr/stddev
 
-lowres=${project_source_dir}/soca/test/Data
+lowres=${project_source_dir}/sorc/soca/test/Data
 
 cp -L ${lowres}/workdir/{diag_table,field_table} ${soca_static}
 cp -L ${project_source_dir}/test/soca/fix/MOM_input ${soca_static}
 cp -L ${lowres}/{fields_metadata.yml,godas_sst_bgerr.nc,rossrad.dat} ${soca_static}
 mv ${soca_static}/fields_metadata.yml ${soca_static}/fields_metadata.yaml
-cp -L ${project_source_dir}/soca/test/testinput/obsop_name_map.yml ${soca_static}/obsop_name_map.yaml
+cp -L ${project_source_dir}/sorc/soca/test/testinput/obsop_name_map.yml ${soca_static}/obsop_name_map.yaml
 cp -L ${lowres}/72x35x25/input.nml ${soca_static}/inputnml
 cp -L ${lowres}/72x35x25/INPUT/{hycom1_25.nc,ocean_mosaic.nc,grid_spec.nc,layer_coord25.nc,ocean_hgrid.nc,ocean_topog.nc} ${soca_static}/INPUT