EQSIM benchmark code and fixes

.gitlab-ci.yml

@@ -23,7 +23,7 @@ perlmutter-no-cache-build:
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
     script:
-        - module load libfabric/1.15.2.0
+        - module load libfabric
         - module list
         - mkdir -p ${PDC_BUILD_PATH}/perlmutter/no-cache
         - cd ${PDC_BUILD_PATH}/perlmutter/no-cache
@@ -51,7 +51,7 @@ perlmutter-cache-build:
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
     script:
-        - module load libfabric/1.15.2.0
+        - module load libfabric
         - module list
         - mkdir -p ${PDC_BUILD_PATH}/perlmutter/cache
         - cd ${PDC_BUILD_PATH}/perlmutter/cache
@@ -374,7 +374,7 @@ perlmutter-metrics-build:
         SUPERCOMPUTER: "perlmutter"
         MERCURY_DIR: "/global/cfs/cdirs/${PDC_PROJECT}/pdc-perlmutter/mercury/install"
     script:
-        - module load libfabric/1.15.2.0
+        - module load libfabric
         - module list
         - mkdir -p ${PDC_BUILD_PATH}/perlmutter/metrics
         - cd ${PDC_BUILD_PATH}/perlmutter/metrics

CMakeLists.txt

@@ -388,10 +388,10 @@ endif()
 option(PDC_SERVER_CACHE "Enable Server Caching." ON)
 if(PDC_SERVER_CACHE)
   set(PDC_SERVER_CACHE 1)
-  set(PDC_SERVER_CACHE_MAX_GB "3" CACHE STRING "Max GB for server cache")
-  set(PDC_SERVER_CACHE_FLUSH_TIME "30" CACHE STRING "Flush time for server cache")
+  set(PDC_SERVER_CACHE_MAX_GB "32" CACHE STRING "Max GB for server cache")
+  set(PDC_SERVER_IDLE_CACHE_FLUSH_TIME "3" CACHE STRING "Idle time to initiate flush from server cache")
 
-  add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_CACHE_FLUSH_TIME=${PDC_SERVER_CACHE_FLUSH_TIME})
+  add_compile_definitions(PDC_SERVER_CACHE_MAX_GB=${PDC_SERVER_CACHE_MAX_GB} PDC_SERVER_IDLE_CACHE_FLUSH_TIME=${PDC_SERVER_IDLE_CACHE_FLUSH_TIME})
 endif()
 
 
@@ -487,6 +487,14 @@ if(PDC_ENABLE_SQLITE3)
     set(ENABLE_SQLITE3 1)
 endif()
 
+#-----------------------------------------------------------------------------
+# ZFP option
+#-----------------------------------------------------------------------------
+option(PDC_ENABLE_ZFP "Enable ZFP." OFF)
+if(PDC_ENABLE_ZFP)
+    set(ENABLE_ZFP 1)
+endif()
+
 # Check availability of symbols
 #-----------------------------------------------------------------------------
 check_symbol_exists(malloc_usable_size "malloc.h" HAVE_MALLOC_USABLE_SIZE)

src/api/pdc_region/pdc_region_transfer.c

@@ -1787,25 +1787,29 @@ release_region_buffer(char *buf, uint64_t *obj_dims, int local_ndim, uint64_t *l
     if (local_ndim == 2) {
         if (access_type == PDC_READ) {
             ptr = new_buf;
-            for (i = 0; i < local_size[0]; ++i) {
-                memcpy(buf + ((local_offset[0] + i) * obj_dims[1] + local_offset[1]) * unit, ptr,
-                       local_size[1] * unit);
-                ptr += local_size[1] * unit;
-            }
+            // Tang
+            memcpy(buf, ptr, local_size[0] * local_size[1] * unit);
+            /* for (i = 0; i < local_size[0]; ++i) { */
+            /*     memcpy(buf + ((local_offset[0] + i) * obj_dims[1] + local_offset[1]) * unit, ptr, */
+            /*            local_size[1] * unit); */
+            /*     ptr += local_size[1] * unit; */
+            /* } */
         }
     }
     else if (local_ndim == 3) {
         if (access_type == PDC_READ) {
             ptr = new_buf;
-            for (i = 0; i < local_size[0]; ++i) {
-                for (j = 0; j < local_size[1]; ++j) {
-                    memcpy(buf + ((local_offset[0] + i) * obj_dims[1] * obj_dims[2] +
-                                  (local_offset[1] + j) * obj_dims[2] + local_offset[2]) *
-                                     unit,
-                           ptr, local_size[2] * unit);
-                    ptr += local_size[2] * unit;
-                }
-            }
+            // Tang
+            memcpy(buf, ptr, local_size[0] * local_size[1] * local_size[2] * unit);
+            /* for (i = 0; i < local_size[0]; ++i) { */
+            /*     for (j = 0; j < local_size[1]; ++j) { */
+            /*         memcpy(buf + ((local_offset[0] + i) * obj_dims[1] * obj_dims[2] + */
+            /*                       (local_offset[1] + j) * obj_dims[2] + local_offset[2]) * */
+            /*                          unit, */
+            /*                ptr, local_size[2] * unit); */
+            /*         ptr += local_size[2] * unit; */
+            /*     } */
+            /* } */
         }
     }
     if (bulk_buf_ref) {

src/commons/utils/pdc_timing.c

@@ -1,5 +1,6 @@
 #include "pdc_timing.h"
 #include "assert.h"
+#include "mpi.h"
 
 #ifdef PDC_TIMING
 static double pdc_base_time;

src/server/CMakeLists.txt

@@ -17,6 +17,13 @@ if(PDC_ENABLE_SQLITE3)
     find_package(SQLite3 3.31.0 REQUIRED)
 endif()
 
+if(PDC_ENABLE_ZFP)
+    add_definitions(-DENABLE_ZFP=1)
+    find_package(ZFP REQUIRED)
+    # find_path(ZFP_INCLUDE_DIR include/zfp.h)
+endif()
+
+
 include_directories(
   ${PDC_COMMON_INCLUDE_DIRS}
   ${PDC_INCLUDES_BUILD_TIME}
@@ -40,6 +47,7 @@ include_directories(
   ${MERCURY_INCLUDE_DIR}
   ${FASTBIT_INCLUDE_DIR}
   ${ROCKSDB_INCLUDE_DIR}
+  ${ZFP_INCLUDE_DIRS}
 )
 
 add_definitions( -DIS_PDC_SERVER=1 )
@@ -70,6 +78,8 @@ add_library(pdc_server_lib
 if(PDC_ENABLE_FASTBIT)
     message(STATUS "Enabled fastbit")
     target_link_libraries(pdc_server_lib ${MERCURY_LIBRARY} ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${FASTBIT_LIBRARY}/libfastbit.so)
+elseif(PDC_ENABLE_ZFP)
+    target_link_libraries(pdc_server_lib ${MERCURY_LIBRARY} ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} zfp::zfp)
 elseif(PDC_ENABLE_ROCKSDB)
     if(PDC_ENABLE_SQLITE3)
         target_link_libraries(pdc_server_lib ${MERCURY_LIBRARY} ${PDC_COMMONS_LIBRARIES} -lm -ldl ${PDC_EXT_LIB_DEPENDENCIES} ${ROCKSDB_LIBRARY} SQLite::SQLite3)

src/server/pdc_server.c

@@ -96,6 +96,7 @@ char **                   all_addr_strings_g       = NULL;
 int                       is_hash_table_init_g     = 0;
 int                       lustre_stripe_size_mb_g  = 16;
 int                       lustre_total_ost_g       = 0;
+int                       pdc_disable_checkpoint_g = 0;
 
 hg_id_t get_remote_metadata_register_id_g;
 hg_id_t buf_map_server_register_id_g;
@@ -719,15 +720,9 @@ PDC_Server_set_close(void)
 #ifdef PDC_TIMING
         start = MPI_Wtime();
 #endif
-        char *tmp_env_char = getenv("PDC_DISABLE_CHECKPOINT");
-        if (tmp_env_char != NULL && strcmp(tmp_env_char, "TRUE") == 0) {
-            if (pdc_server_rank_g == 0) {
-                printf("==PDC_SERVER[0]: checkpoint disabled!\n");
-            }
-        }
-        else {
+        if (pdc_disable_checkpoint_g == 0)
             PDC_Server_checkpoint();
-        }
+
 #ifdef PDC_TIMING
         pdc_server_timings->PDCserver_checkpoint += MPI_Wtime() - start;
 #endif
@@ -1204,7 +1199,8 @@ PDC_Server_recv_shm_cb(const struct hg_cb_info *callback_info)
 hg_return_t
 PDC_Server_checkpoint_cb()
 {
-    PDC_Server_checkpoint();
+    if (pdc_disable_checkpoint_g == 0)
+        PDC_Server_checkpoint();
 
     return HG_SUCCESS;
 }
@@ -1862,7 +1858,7 @@ PDC_Server_loop(hg_context_t *hg_context)
 #ifdef PDC_ENABLE_CHECKPOINT
         checkpoint_interval++;
         // Avoid calling clock() every operation
-        if (checkpoint_interval % PDC_CHECKPOINT_CHK_OP_INTERVAL == 0) {
+        if (pdc_disable_checkpoint_g == 0 && checkpoint_interval % PDC_CHECKPOINT_CHK_OP_INTERVAL == 0) {
             cur_time            = clock();
             double elapsed_time = ((double)(cur_time - last_checkpoint_time)) / CLOCKS_PER_SEC;
             /* fprintf(stderr, "PDC_SERVER: loop elapsed time %.2f\n", elapsed_time); */
@@ -2117,7 +2113,7 @@ PDC_Server_get_env()
         data_sieving_g = atoi(tmp_env_char);
     }
     else {
-        data_sieving_g = 0;
+        data_sieving_g = 1;
     }
 
     // Get number of OST per file
@@ -2158,7 +2154,7 @@ PDC_Server_get_env()
 
     tmp_env_char = getenv("PDC_GEN_HIST");
     if (tmp_env_char != NULL)
-        gen_hist_g = 1;
+        gen_hist_g = atoi(tmp_env_char);
 
     tmp_env_char = getenv("PDC_GEN_FASTBIT_IDX");
     if (tmp_env_char != NULL)
@@ -2184,6 +2180,13 @@ PDC_Server_get_env()
             printf("==PDC_SERVER[%d]: using SQLite3 for kvtag\n", pdc_server_rank_g);
     }
 
+    tmp_env_char = getenv("PDC_DISABLE_CHECKPOINT");
+    if (tmp_env_char != NULL && strcmp(tmp_env_char, "TRUE") == 0) {
+        pdc_disable_checkpoint_g = 1;
+        if (pdc_server_rank_g == 0)
+            printf("==PDC_SERVER[0]: checkpoint disabled!\n");
+    }
+
     if (pdc_server_rank_g == 0) {
         printf("==PDC_SERVER[%d]: using [%s] as tmp dir, %d OSTs, %d OSTs per data file, %d%% to BB\n",
                pdc_server_rank_g, pdc_server_tmp_dir_g, lustre_total_ost_g, pdc_nost_per_file_g,