initial hack

Signed-off-by: Wenduo Wang <[email protected]>

ompi/mca/pml/base/pml_base_frame.c

@@ -192,6 +192,9 @@ static int mca_pml_base_close(void)
     OBJ_DESTRUCT(&mca_pml_base_send_requests);
     OBJ_DESTRUCT(&mca_pml_base_recv_requests);
 
+    OBJ_DESTRUCT(&mca_pml_ob1_send_requests);
+    OBJ_DESTRUCT(&mca_pml_ob1_recv_requests);
+
     mca_pml.pml_progress = mca_pml_base_progress;
 
     /* Free all the strings in the array */
@@ -223,6 +226,9 @@ static int mca_pml_base_open(mca_base_open_flag_t flags)
     OBJ_CONSTRUCT(&mca_pml_base_send_requests, opal_free_list_t);
     OBJ_CONSTRUCT(&mca_pml_base_recv_requests, opal_free_list_t);
 
+    OBJ_CONSTRUCT(&mca_pml_ob1_send_requests, opal_free_list_t);
+    OBJ_CONSTRUCT(&mca_pml_ob1_recv_requests, opal_free_list_t);
+
     OBJ_CONSTRUCT(&mca_pml_base_pml, opal_pointer_array_t);
 
     /* Open up all available components */
@@ -261,6 +267,7 @@ static int mca_pml_base_open(mca_base_open_flag_t flags)
             opal_pointer_array_add(&mca_pml_base_pml, strdup("ob1"));
             opal_pointer_array_add(&mca_pml_base_pml, strdup("ucx"));
             opal_pointer_array_add(&mca_pml_base_pml, strdup("cm"));
+            opal_pointer_array_add(&mca_pml_base_pml, strdup("topo"));
         } else {
 #if OPAL_ENABLE_DEBUG
             char **req_pml = opal_argv_split(default_pml[0], ',');

ompi/mca/pml/base/pml_base_request.c

@@ -30,6 +30,9 @@
 opal_free_list_t mca_pml_base_send_requests = {{{0}}};
 opal_free_list_t mca_pml_base_recv_requests = {{{0}}};
 
+opal_free_list_t mca_pml_ob1_send_requests = {{{0}}};
+opal_free_list_t mca_pml_ob1_recv_requests = {{{0}}};
+
 static void mca_pml_base_request_construct(mca_pml_base_request_t* req)
 {
     req->req_ompi.req_type = OMPI_REQUEST_PML;

ompi/mca/pml/base/pml_base_request.h

@@ -44,6 +44,9 @@ BEGIN_C_DECLS
 OMPI_DECLSPEC extern opal_free_list_t mca_pml_base_send_requests;
 OMPI_DECLSPEC extern opal_free_list_t mca_pml_base_recv_requests;
 
+OMPI_DECLSPEC extern opal_free_list_t mca_pml_ob1_send_requests;
+OMPI_DECLSPEC extern opal_free_list_t mca_pml_ob1_recv_requests;
+
 /**
  * Type of request.
  */

ompi/mca/pml/base/pml_base_select.c

@@ -189,7 +189,11 @@ int mca_pml_base_select(bool enable_progress_threads,
          item = opal_list_remove_first(&opened)) {
         om = (opened_component_t *) item;
 
-        if (om->om_component != best_component ) {
+        if (om->om_component != best_component
+            && strncmp(om->om_component->pmlm_version.mca_component_name, "ob1",
+                       strlen(om->om_component->pmlm_version.mca_component_name))
+            && strncmp(om->om_component->pmlm_version.mca_component_name, "cm",
+                       strlen(om->om_component->pmlm_version.mca_component_name))) {
             /* Finalize */
 
             if (NULL != om->om_component->pmlm_finalize) {
@@ -213,9 +217,20 @@ int mca_pml_base_select(bool enable_progress_threads,
        available list all unselected components.  The available list will
        contain only the selected component. */
 
-    mca_base_components_close(ompi_pml_base_framework.framework_output,
-                              &ompi_pml_base_framework.framework_components,
-                              (mca_base_component_t *) best_component);
+    // mca_base_components_close(ompi_pml_base_framework.framework_output,
+    //                           &ompi_pml_base_framework.framework_components,
+    //                           (mca_base_component_t *) best_component);
+
+    OPAL_LIST_FOREACH (cli, &ompi_pml_base_framework.framework_components, mca_base_component_list_item_t) {
+        component = (mca_pml_base_component_t *) cli->cli_component;
+        if (component != best_component
+            && strncmp(om->om_component->pmlm_version.mca_component_name, "ob1",
+                       strlen(om->om_component->pmlm_version.mca_component_name))
+            && strncmp(om->om_component->pmlm_version.mca_component_name, "cm",
+                       strlen(om->om_component->pmlm_version.mca_component_name))) {
+            mca_base_component_close(component, ompi_pml_base_framework.framework_output);
+        }
+    }
 
     /* register the winner's callback */
     if( NULL != mca_pml.pml_progress ) {

ompi/mca/pml/cm/pml_cm.c

@@ -127,11 +127,11 @@ mca_pml_cm_add_procs(struct ompi_proc_t** procs, size_t nprocs)
 #endif
 
     /* make sure remote procs are using the same PML as us */
-    if (OMPI_SUCCESS != (ret = mca_pml_base_pml_check_selected("cm",
-                                                              procs,
-                                                              nprocs))) {
-        return ret;
-    }
+    // if (OMPI_SUCCESS != (ret = mca_pml_base_pml_check_selected("cm",
+    //                                                           procs,
+    //                                                           nprocs))) {
+    //     return ret;
+    // }
 
     ret = OMPI_MTL_CALL(add_procs(ompi_mtl, nprocs, procs));
     return ret;

ompi/mca/pml/ob1/pml_ob1.c

@@ -175,7 +175,7 @@ int mca_pml_ob1_enable(bool enable)
      * should get ownership for the send and receive requests list, and
      * initialize them with the size of our own requests.
      */
-    opal_free_list_init ( &mca_pml_base_send_requests,
+    opal_free_list_init ( &mca_pml_ob1_send_requests,
                           sizeof(mca_pml_ob1_send_request_t) +
                           sizeof(mca_pml_ob1_com_btl_t[mca_pml_ob1.max_rdma_per_request]),
                           opal_cache_line_size,
@@ -186,7 +186,7 @@ int mca_pml_ob1_enable(bool enable)
                           mca_pml_ob1.free_list_inc,
                           NULL, 0, NULL, NULL, NULL);
 
-    opal_free_list_init ( &mca_pml_base_recv_requests,
+    opal_free_list_init ( &mca_pml_ob1_recv_requests,
                           sizeof(mca_pml_ob1_recv_request_t) +
                           sizeof(mca_pml_ob1_com_btl_t[mca_pml_ob1.max_rdma_per_request]),
                           opal_cache_line_size,
@@ -388,11 +388,11 @@ int mca_pml_ob1_add_procs(ompi_proc_t** procs, size_t nprocs)
         return rc;
 
     /* make sure remote procs are using the same PML as us */
-    if (OMPI_SUCCESS != (rc = mca_pml_base_pml_check_selected("ob1",
-                                                              procs,
-                                                              nprocs))) {
-        return rc;
-    }
+    // if (OMPI_SUCCESS != (rc = mca_pml_base_pml_check_selected("ob1",
+    //                                                           procs,
+    //                                                           nprocs))) {
+    //     return rc;
+    // }
 
     OBJ_CONSTRUCT(&reachable, opal_bitmap_t);
     rc = opal_bitmap_init(&reachable, (int)nprocs);

ompi/mca/pml/ob1/pml_ob1.h

@@ -94,6 +94,8 @@ typedef struct mca_pml_ob1_t mca_pml_ob1_t;
 extern mca_pml_ob1_t mca_pml_ob1;
 extern int mca_pml_ob1_output;
 extern bool mca_pml_ob1_matching_protection;
+OMPI_DECLSPEC extern opal_free_list_t mca_pml_ob1_send_requests;
+OMPI_DECLSPEC extern opal_free_list_t mca_pml_ob1_recv_requests;
 /*
  * PML interface functions.
  */

ompi/mca/pml/ob1/pml_ob1_component.c

@@ -344,12 +344,12 @@ int mca_pml_ob1_component_fini(void)
     /* return the static receive/send requests to the respective free list and
      * let the free list handle destruction. */
     if( NULL != mca_pml_ob1_recvreq ) {
-        opal_free_list_return (&mca_pml_base_recv_requests, (opal_free_list_item_t *) mca_pml_ob1_recvreq);
+        opal_free_list_return (&mca_pml_ob1_recv_requests, (opal_free_list_item_t *) mca_pml_ob1_recvreq);
         mca_pml_ob1_recvreq = NULL;
     }
 
     if( NULL != mca_pml_ob1_sendreq ) {
-        opal_free_list_return (&mca_pml_base_send_requests, (opal_free_list_item_t *) mca_pml_ob1_sendreq);
+        opal_free_list_return (&mca_pml_ob1_send_requests, (opal_free_list_item_t *) mca_pml_ob1_sendreq);
         mca_pml_ob1_sendreq = NULL;
     }
 
@@ -375,17 +375,17 @@ int mca_pml_ob1_component_fini(void)
     }
 
 #if 0
-    if (mca_pml_base_send_requests.fl_num_allocated !=
-        mca_pml_base_send_requests.super.opal_list_length) {
+    if (mca_pml_ob1_send_requests.fl_num_allocated !=
+        mca_pml_ob1_send_requests.super.opal_list_length) {
         opal_output(0, "ob1 send requests: %d allocated %d returned\n",
-                    mca_pml_base_send_requests.fl_num_allocated,
-                    mca_pml_base_send_requests.super.opal_list_length);
+                    mca_pml_ob1_send_requests.fl_num_allocated,
+                    mca_pml_ob1_send_requests.super.opal_list_length);
     }
-    if (mca_pml_base_recv_requests.fl_num_allocated !=
-        mca_pml_base_recv_requests.super.opal_list_length) {
+    if (mca_pml_ob1_recv_requests.fl_num_allocated !=
+        mca_pml_ob1_recv_requests.super.opal_list_length) {
         opal_output(0, "ob1 recv requests: %d allocated %d returned\n",
-                    mca_pml_base_recv_requests.fl_num_allocated,
-                    mca_pml_base_recv_requests.super.opal_list_length);
+                    mca_pml_ob1_recv_requests.fl_num_allocated,
+                    mca_pml_ob1_recv_requests.super.opal_list_length);
     }
 #endif
 

ompi/mca/pml/ob1/pml_ob1_recvreq.h

@@ -82,7 +82,7 @@ static inline bool unlock_recv_request(mca_pml_ob1_recv_request_t *recvreq)
 #define MCA_PML_OB1_RECV_REQUEST_ALLOC(recvreq)                    \
 do {                                                               \
     recvreq = (mca_pml_ob1_recv_request_t *)                          \
-        opal_free_list_get (&mca_pml_base_recv_requests);             \
+        opal_free_list_get (&mca_pml_ob1_recv_requests);             \
 } while(0)
 
 
@@ -115,6 +115,9 @@ do {                                                                \
                                     tag,                            \
                                     comm,                           \
                                     persistent);                    \
+    (request)->req_rdma_cnt = 0;                                    \
+    (request)->rdma_bml = 0;                                        \
+    (request)->local_handle = NULL;                                 \
 } while(0)
 
 /**
@@ -144,7 +147,7 @@ static inline void mca_pml_ob1_recv_request_fini (mca_pml_ob1_recv_request_t *re
 #define MCA_PML_OB1_RECV_REQUEST_RETURN(recvreq)                        \
     {                                                                   \
         mca_pml_ob1_recv_request_fini (recvreq);                        \
-        opal_free_list_return (&mca_pml_base_recv_requests,             \
+        opal_free_list_return (&mca_pml_ob1_recv_requests,             \
                                (opal_free_list_item_t*)(recvreq));      \
     }
 

ompi/mca/pml/ob1/pml_ob1_sendreq.h

@@ -130,7 +130,7 @@ get_request_from_send_pending(mca_pml_ob1_send_pending_t *type)
                                                                         \
         if( OPAL_LIKELY(NULL != proc) ) {                               \
             sendreq = (mca_pml_ob1_send_request_t*)                     \
-                opal_free_list_wait (&mca_pml_base_send_requests);      \
+                opal_free_list_wait (&mca_pml_ob1_send_requests);      \
             sendreq->req_send.req_base.req_proc = proc;                 \
         }                                                               \
     }
@@ -159,6 +159,7 @@ get_request_from_send_pending(mca_pml_ob1_send_pending_t *type)
                                        0); /* convertor_flags */        \
         (sendreq)->req_recv.pval = NULL;                                \
         (sendreq)->ob1_proc = ob1_proc;                                 \
+        (sendreq)->req_rdma_cnt = 0;                                    \
     }
 
 #define MCA_PML_OB1_SEND_REQUEST_RESET(sendreq)                         \
@@ -241,7 +242,7 @@ static inline void mca_pml_ob1_send_request_fini (mca_pml_ob1_send_request_t *se
 #define MCA_PML_OB1_SEND_REQUEST_RETURN(sendreq)                        \
     do {                                                                \
         mca_pml_ob1_send_request_fini (sendreq);                        \
-        opal_free_list_return ( &mca_pml_base_send_requests,            \
+        opal_free_list_return ( &mca_pml_ob1_send_requests,            \
                                 (opal_free_list_item_t*)sendreq);       \
         sendreq = NULL;  /* for safety */                               \
     } while(0)

ompi/mca/pml/topo/Makefile.am

@@ -0,0 +1,40 @@
+#
+# Copyright (c) Amazon.com, Inc. or its affiliates. All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+EXTRA_DIST = post_configure.sh
+
+if MCA_BUILD_ompi_pml_topo_DSO
+component_noinst =
+component_install = mca_pml_topo.la
+else
+component_noinst = libmca_pml_topo.la
+component_install =
+endif
+
+local_sources = \
+	pml_topo.c \
+	pml_topo.h \
+	pml_topo_component.c \
+	pml_topo_component.h
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_pml_topo_la_SOURCES = $(local_sources)
+mca_pml_topo_la_LIBADD = $(top_builddir)/ompi/lib@[email protected] \
+	$(pml_topo_LIBS)
+mca_pml_topo_la_LDFLAGS = -module -avoid-version $(pml_topo_LDFLAGS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_pml_topo_la_SOURCES = $(local_sources)
+libmca_pml_topo_la_LIBADD = $(pml_topo_LIBS)
+libmca_pml_topo_la_LDFLAGS = -module -avoid-version $(pml_topo_LDFLAGS)