coll/han: set OMPI_COMM_DISJOINT flag for up/low comms

Utilize the communicator flag to pass on hierarchy information to the underlying collective modules. Signed-off-by: Wenduo Wang <[email protected]>
wenduwan · Aug 18, 2023 · 849c6ac · 849c6ac
1 parent 4bbd17d
commit 849c6ac
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/ompi/mca/coll/han/coll_han_subcomms.c b/ompi/mca/coll/han/coll_han_subcomms.c
@@ -128,6 +128,8 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
         goto return_with_error;
     }
 
+    (*low_comm)->c_flags &= ~OMPI_COMM_DISJOINT;
+
     /*
      * Get my local rank and the local size
      */
@@ -145,6 +147,8 @@ int mca_coll_han_comm_create_new(struct ompi_communicator_t *comm,
         goto return_with_error;
     }
 
+    (*up_comm)->c_flags |= OMPI_COMM_DISJOINT;
+
     up_rank = ompi_comm_rank(*up_comm);
 
     /*
@@ -283,6 +287,8 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
                          &comm_info, &(low_comms[0]));
 
+    low_comms[0]->c_flags &= ~OMPI_COMM_DISJOINT;
+
     /*
      * Get my local rank and the local size
      */
@@ -297,6 +303,8 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     ompi_comm_split_type(comm, MPI_COMM_TYPE_SHARED, 0,
                          &comm_info, &(low_comms[1]));
 
+    low_comms[1]->c_flags &= ~OMPI_COMM_DISJOINT;
+
     /*
      * Upgrade libnbc module priority to set up up_comms[0] with libnbc module
      * This sub-communicator contains one process per node: processes with the
@@ -305,6 +313,8 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     opal_info_set(&comm_info, "ompi_comm_coll_preference", "libnbc,^han");
     ompi_comm_split_with_info(comm, low_rank, w_rank, &comm_info, &(up_comms[0]), false);
 
+    up_comms[0]->c_flags |= OMPI_COMM_DISJOINT;
+
     up_rank = ompi_comm_rank(up_comms[0]);
 
     /*
@@ -314,6 +324,8 @@ int mca_coll_han_comm_create(struct ompi_communicator_t *comm,
     opal_info_set(&comm_info, "ompi_comm_coll_preference", "adapt,^han");
     ompi_comm_split_with_info(comm, low_rank, w_rank, &comm_info, &(up_comms[1]), false);
 
+    up_comms[1]->c_flags |= OMPI_COMM_DISJOINT;
+
     /*
      * Set my virtual rank number.
      * my rank # = <intra-node comm size> * <inter-node rank number>