Merge branch 'fix/darshan' into fix/darshan-trace

esmf-org · Aug 1, 2023 · cf804a2 · cf804a2
2 parents d06b7d6 + 858b8d3
commit cf804a2
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 5 deletions.
diff --git a/src/Infrastructure/Trace/preload/makefile b/src/Infrastructure/Trace/preload/makefile
@@ -31,7 +31,7 @@ ifeq  ($(ESMF_OS),MinGW)
   ESMF_TRACE_ESMFLIB = -lesmf
 endif
 
-tracelib_preload: preload.o preload_io.o preload_mpi.o wrappers.o wrappers_io.o wrappers_mpi.o
+tracelib_preload: preload.o preload_mpi.o wrappers.o wrappers_mpi.o
 	$(ESMF_SL_LIBLINKER) $(ESMF_SL_LIBOPTS) -o $(ESMF_LDIR)/libesmftrace_preload.$(ESMF_SL_SUFFIX) $^ $(ESMF_CXXLINKOPTS) $(ESMF_CXXLINKPATHS) $(ESMF_CXXLINKRPATHS) $(ESMF_CXXLINKLIBS) $(ESMF_TRACE_ESMFLIB)
 	$(MAKE) ESMF_PRELOADDIR=$(ESMF_LIBDIR) build_preload_script
 

diff --git a/src/Infrastructure/VM/src/ESMCI_VMKernel.C b/src/Infrastructure/VM/src/ESMCI_VMKernel.C
@@ -119,6 +119,10 @@ typedef DWORD pid_t;
 #define VM_MPI_THREAD_LEVEL MPI_THREAD_MULTIPLE
 #endif
 
+// Utility macros to get a macro value into a string
+#define XSTR(X) STR(X)
+#define STR(X) #X
+
 namespace ESMCI {
 
 // Definition of class static data members
@@ -682,8 +686,14 @@ void VMK::finalize(int finalizeMpi){
     delete [] cid[i];
   delete [] cid;
   delete [] ssiLocalPetList;
-  // finalize the MPI tool interface
-  MPI_T_finalize();
+  // Special MPI Tool Interface handling for OpenMPI
+  std::string esmf_comm(XSTR(ESMF_COMM));
+  bool mpi_t_done = false;
+  if (esmf_comm == "openmpi"){
+    // OpenMPI must call MPI_T_finalize() before MPI_Finalize()
+    mpi_t_done = true;
+    MPI_T_finalize();
+  }
   // conditionally finalize MPI
   int finalized;
   MPI_Finalized(&finalized);
@@ -695,6 +705,12 @@ void VMK::finalize(int finalizeMpi){
     if (finalizeMpi)
       MPI_Finalize();
   }
+  MPI_Finalized(&finalized);
+  if (finalized && !mpi_t_done){
+    // finalize the MPI tool interface
+    // do this _after_ MPI_Finalize() or else Darshan dies with SEGV (not clear why)
+    MPI_T_finalize();
+  }
 }
 
 

diff --git a/src/Superstructure/ESMFMod/interface/ESMCI_Init_F.C b/src/Superstructure/ESMFMod/interface/ESMCI_Init_F.C
@@ -90,8 +90,6 @@ void FTN_X(c_esmc_initget_esmf_dir) (
 #undef  ESMC_METHOD
 #define ESMC_METHOD "c_esmc_initget_esmf_dir()"
 
-#define XSTR(X) STR(X)
-#define STR(X) #X
   const char* esmf_dir = XSTR(ESMF_DIR);
   int localrc = ESMC_CtoF90string (esmf_dir, esmf_dir_str, esmf_dir_str_l);
   if (ESMC_LogDefault.MsgFoundError(localrc, ESMCI_ERR_PASSTHRU, ESMC_CONTEXT,