1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
4 * University Research and Technology
5 * Corporation. All rights reserved.
6 * Copyright (c) 2004-2013 The University of Tennessee and The University
7 * of Tennessee Research Foundation. All rights
8 * reserved.
9 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
10 * University of Stuttgart. All rights reserved.
11 * Copyright (c) 2004-2005 The Regents of the University of California.
12 * All rights reserved.
13 * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
14 * Copyright (c) 2012-2013 Inria. All rights reserved.
15 * Copyright (c) 2014 Los Alamos National Security, LLC. All right
16 * reserved.
17 * Copyright (c) 2014-2015 Research Organization for Information Science
18 * and Technology (RIST). All rights reserved.
19 * $COPYRIGHT$
20 *
21 * Additional copyrights may follow
22 *
23 * $HEADER$
24 */
25
26 #include "ompi_config.h"
27 #include "ompi/constants.h"
28
29 #include "ompi/mca/topo/base/base.h"
30 #include "ompi/mca/topo/topo.h"
31
32 /*
33 * function - makes a new communicator to which topology information
34 * has been attached
35 *
36 * @param comm input communicator (handle)
37 * @param ndims number of dimensions of cartesian grid (integer)
38 * @param dims integer array of size ndims specifying the number of processes in
39 * each dimension
40 * @param periods logical array of size ndims specifying whether the grid is
41 * periodic (true) or not (false) in each dimension
42 * @param reorder ranking may be reordered (true) or not (false) (logical)
43 * @param comm_cart communicator with new cartesian topology (handle)
44 *
45 * Open MPI currently ignores the 'reorder' flag.
46 *
47 * @retval OMPI_SUCCESS
48 */
49
mca_topo_base_cart_create(mca_topo_base_module_t * topo,ompi_communicator_t * old_comm,int ndims,const int * dims,const int * periods,bool reorder,ompi_communicator_t ** comm_topo)50 int mca_topo_base_cart_create(mca_topo_base_module_t *topo,
51 ompi_communicator_t* old_comm,
52 int ndims,
53 const int *dims,
54 const int *periods,
55 bool reorder,
56 ompi_communicator_t** comm_topo)
57 {
58 int nprocs = 1, i, new_rank, num_procs, ret;
59 ompi_communicator_t *new_comm;
60 ompi_proc_t **topo_procs = NULL;
61 mca_topo_base_comm_cart_2_2_0_t* cart;
62
63 num_procs = old_comm->c_local_group->grp_proc_count;
64 new_rank = old_comm->c_local_group->grp_my_rank;
65 assert(topo->type == OMPI_COMM_CART);
66
67 /* Calculate the number of processes in this grid */
68 for (i = 0; i < ndims; ++i) {
69 if(dims[i] <= 0) {
70 return OMPI_ERROR;
71 }
72 nprocs *= dims[i];
73 }
74
75 /* check for the error condition */
76 if (num_procs < nprocs) {
77 return MPI_ERR_DIMS;
78 }
79
80 /* check if we have to trim the list of processes */
81 if (nprocs < num_procs) {
82 num_procs = nprocs;
83 }
84
85 if (new_rank > (nprocs-1)) {
86 ndims = 0;
87 new_rank = MPI_UNDEFINED;
88 num_procs = 0;
89 }
90
91 cart = OBJ_NEW(mca_topo_base_comm_cart_2_2_0_t);
92 if( NULL == cart ) {
93 return OMPI_ERR_OUT_OF_RESOURCE;
94 }
95 cart->ndims = ndims;
96
97 /* MPI-2.1 allows 0-dimension cartesian communicators, so prevent
98 a 0-byte malloc -- leave dims as NULL */
99 if( ndims > 0 ) {
100 cart->dims = (int*)malloc(sizeof(int) * ndims);
101 if (NULL == cart->dims) {
102 OBJ_RELEASE(cart);
103 return OMPI_ERROR;
104 }
105 memcpy(cart->dims, dims, ndims * sizeof(int));
106
107 /* Cartesian communicator; copy the right data to the common information */
108 cart->periods = (int*)malloc(sizeof(int) * ndims);
109 if (NULL == cart->periods) {
110 OBJ_RELEASE(cart);
111 return OMPI_ERR_OUT_OF_RESOURCE;
112 }
113 memcpy(cart->periods, periods, ndims * sizeof(int));
114
115 cart->coords = (int*)malloc(sizeof(int) * ndims);
116 if (NULL == cart->coords) {
117 OBJ_RELEASE(cart);
118 return OMPI_ERR_OUT_OF_RESOURCE;
119 }
120 { /* setup the cartesian topology */
121 int nprocs = num_procs, rank = new_rank;
122
123 for (i = 0; i < ndims; ++i) {
124 nprocs /= cart->dims[i];
125 cart->coords[i] = rank / nprocs;
126 rank %= nprocs;
127 }
128 }
129 }
130
131 /* JMS: This should really be refactored to use
132 comm_create_group(), because ompi_comm_allocate() still
133 complains about 0-byte mallocs in debug builds for 0-member
134 groups. */
135 if (num_procs > 0) {
136 /* Copy the proc structure from the previous communicator over to
137 the new one. The topology module is then able to work on this
138 copy and rearrange it as it deems fit. */
139 topo_procs = (ompi_proc_t**)malloc(num_procs * sizeof(ompi_proc_t *));
140 if (NULL == topo_procs) {
141 OBJ_RELEASE(cart);
142 return OMPI_ERR_OUT_OF_RESOURCE;
143 }
144 if(OMPI_GROUP_IS_DENSE(old_comm->c_local_group)) {
145 memcpy(topo_procs,
146 old_comm->c_local_group->grp_proc_pointers,
147 num_procs * sizeof(ompi_proc_t *));
148 } else {
149 for(i = 0 ; i < num_procs; i++) {
150 topo_procs[i] = ompi_group_peer_lookup(old_comm->c_local_group,i);
151 }
152 }
153 }
154
155 /* allocate a new communicator */
156 new_comm = ompi_comm_allocate(num_procs, 0);
157 if (NULL == new_comm) {
158 free(topo_procs);
159 OBJ_RELEASE(cart);
160 return MPI_ERR_INTERN;
161 }
162
163 ret = ompi_comm_enable(old_comm, new_comm,
164 new_rank, num_procs, topo_procs);
165 if (OMPI_SUCCESS != ret) {
166 /* something wrong happened during setting the communicator */
167 free(topo_procs);
168 OBJ_RELEASE(cart);
169 if (MPI_COMM_NULL != new_comm) {
170 new_comm->c_topo = NULL;
171 new_comm->c_flags &= ~OMPI_COMM_CART;
172 ompi_comm_free (&new_comm);
173 }
174 return ret;
175 }
176
177 new_comm->c_topo = topo;
178 new_comm->c_topo->mtc.cart = cart;
179 new_comm->c_topo->reorder = reorder;
180 new_comm->c_flags |= OMPI_COMM_CART;
181 *comm_topo = new_comm;
182
183 if( MPI_UNDEFINED == new_rank ) {
184 ompi_comm_free(&new_comm);
185 *comm_topo = MPI_COMM_NULL;
186 }
187
188 /* end here */
189 return OMPI_SUCCESS;
190 }
191
mca_topo_base_comm_cart_2_2_0_construct(mca_topo_base_comm_cart_2_2_0_t * cart)192 static void mca_topo_base_comm_cart_2_2_0_construct(mca_topo_base_comm_cart_2_2_0_t * cart) {
193 cart->ndims = 0;
194 cart->dims = NULL;
195 cart->periods = NULL;
196 cart->coords = NULL;
197 }
198
mca_topo_base_comm_cart_2_2_0_destruct(mca_topo_base_comm_cart_2_2_0_t * cart)199 static void mca_topo_base_comm_cart_2_2_0_destruct(mca_topo_base_comm_cart_2_2_0_t * cart) {
200 if (NULL != cart->dims) {
201 free(cart->dims);
202 }
203 if (NULL != cart->periods) {
204 free(cart->periods);
205 }
206 if (NULL != cart->coords) {
207 free(cart->coords);
208 }
209 }
210
211 OBJ_CLASS_INSTANCE(mca_topo_base_comm_cart_2_2_0_t, opal_object_t,
212 mca_topo_base_comm_cart_2_2_0_construct,
213 mca_topo_base_comm_cart_2_2_0_destruct);
214