1 #if HAVE_CONFIG_H
2 # include "config.h"
3 #endif
4
5 #include <stdlib.h>
6 #include <assert.h>
7
8 #include "comex.h"
9 #include "comex_impl.h"
10 #include "groups.h"
11
12
13 /* the HEAD of the group linked list */
14 comex_igroup_t *group_list = NULL;
15
16
17 /* static functions implemented in this file */
18 static void comex_create_group_and_igroup(
19 comex_group_t *id, comex_igroup_t **igroup);
20 static void comex_igroup_finalize(comex_igroup_t *igroup);
21
22
23 /**
24 * Return the comex igroup instance given the group id.
25 *
26 * The group linked list is searched sequentially until the given group
27 * is found. It is an error if this function is called before
28 * comex_group_init(). An error occurs if the given group is not found.
29 */
comex_get_igroup_from_group(comex_group_t id)30 comex_igroup_t* comex_get_igroup_from_group(comex_group_t id)
31 {
32 comex_igroup_t *current_group_list_item = group_list;
33
34 assert(group_list != NULL);
35 while (current_group_list_item != NULL) {
36 if (current_group_list_item->id == id) {
37 return current_group_list_item;
38 }
39 current_group_list_item = current_group_list_item->next;
40 }
41 comex_error("comex group lookup failed", -1);
42
43 return NULL;
44 }
45
46
47 /**
48 * Creates and associates an comex group with an comex igroup.
49 *
50 * This does *not* initialize the members of the comex igroup.
51 */
comex_create_group_and_igroup(comex_group_t * id,comex_igroup_t ** igroup)52 static void comex_create_group_and_igroup(
53 comex_group_t *id, comex_igroup_t **igroup)
54 {
55 comex_igroup_t *new_group_list_item = NULL;
56 comex_igroup_t *last_group_list_item = NULL;
57
58 /* find the last group in the group linked list */
59 last_group_list_item = group_list;
60 while (last_group_list_item->next != NULL) {
61 last_group_list_item = last_group_list_item->next;
62 }
63
64 /* create, init, and insert the new node for the linked list */
65 new_group_list_item = malloc(sizeof(comex_igroup_t));
66 new_group_list_item->id = last_group_list_item->id + 1;
67 new_group_list_item->comm = MPI_COMM_NULL;
68 new_group_list_item->group = MPI_GROUP_NULL;
69 new_group_list_item->next = NULL;
70 last_group_list_item->next = new_group_list_item;
71
72 /* return the group id and comex igroup */
73 *igroup = new_group_list_item;
74 *id = new_group_list_item->id;
75 }
76
77
comex_group_rank(comex_group_t group,int * rank)78 int comex_group_rank(comex_group_t group, int *rank)
79 {
80 int status;
81
82 comex_igroup_t *igroup = comex_get_igroup_from_group(group);
83 status = MPI_Group_rank(igroup->group, rank);
84 if (status != MPI_SUCCESS) {
85 comex_error("MPI_Group_rank: Failed ", status);
86 }
87
88 return COMEX_SUCCESS;
89 }
90
91
comex_group_size(comex_group_t group,int * size)92 int comex_group_size(comex_group_t group, int *size)
93 {
94 int status;
95
96 comex_igroup_t *igroup = comex_get_igroup_from_group(group);
97 status = MPI_Group_size(igroup->group, size);
98 if (status != MPI_SUCCESS) {
99 comex_error("MPI_Group_size: Failed ", status);
100 }
101
102 return COMEX_SUCCESS;
103 }
104
105
comex_group_comm(comex_group_t group,MPI_Comm * comm)106 int comex_group_comm(comex_group_t group, MPI_Comm *comm)
107 {
108 int status;
109
110 comex_igroup_t *igroup = comex_get_igroup_from_group(group);
111 *comm = igroup->comm;
112
113 return COMEX_SUCCESS;
114 }
115
116
comex_group_translate_world(comex_group_t group,int group_rank,int * world_rank)117 int comex_group_translate_world(comex_group_t group, int group_rank, int *world_rank)
118 {
119 if (COMEX_GROUP_WORLD == group) {
120 *world_rank = group_rank;
121 }
122 else {
123 comex_igroup_t *igroup = comex_get_igroup_from_group(group);
124 comex_igroup_t *world_igroup = comex_get_igroup_from_group(COMEX_GROUP_WORLD);
125 int status = MPI_Group_translate_ranks(
126 igroup->group, 1, &group_rank, world_igroup->group, world_rank);
127 if (status != MPI_SUCCESS) {
128 comex_error("MPI_Group_translate_ranks: Failed ", status);
129 }
130 }
131
132 return COMEX_SUCCESS;
133 }
134
135
136 /**
137 * Destroys the given comex igroup.
138 */
comex_igroup_finalize(comex_igroup_t * igroup)139 static void comex_igroup_finalize(comex_igroup_t *igroup)
140 {
141 int status;
142
143 assert(igroup);
144
145 if (igroup->group != MPI_GROUP_NULL) {
146 status = MPI_Group_free(&igroup->group);
147 if (status != MPI_SUCCESS) {
148 comex_error("MPI_Group_free: Failed ", status);
149 }
150 }
151
152 if (igroup->comm != MPI_COMM_NULL) {
153 status = MPI_Comm_free(&igroup->comm);
154 if (status != MPI_SUCCESS) {
155 comex_error("MPI_Comm_free: Failed ", status);
156 }
157 }
158 }
159
160
comex_group_free(comex_group_t id)161 int comex_group_free(comex_group_t id)
162 {
163 comex_igroup_t *current_group_list_item = group_list;
164 comex_igroup_t *previous_group_list_item = NULL;
165
166 /* find the group to free */
167 while (current_group_list_item != NULL) {
168 if (current_group_list_item->id == id) {
169 break;
170 }
171 previous_group_list_item = current_group_list_item;
172 current_group_list_item = current_group_list_item->next;
173 }
174 /* make sure we found a group */
175 assert(current_group_list_item != NULL);
176 /* remove the group from the linked list */
177 if (previous_group_list_item != NULL) {
178 previous_group_list_item->next = current_group_list_item->next;
179 }
180 /* free the group */
181 comex_igroup_finalize(current_group_list_item);
182 free(current_group_list_item);
183
184 return COMEX_SUCCESS;
185 }
186
187
comex_group_create(int n,int * pid_list,comex_group_t id_parent,comex_group_t * id_child)188 int comex_group_create(
189 int n, int *pid_list, comex_group_t id_parent, comex_group_t *id_child)
190 {
191 int status;
192 int grp_me;
193 comex_igroup_t *igroup_child = NULL;
194 MPI_Group *group_child = NULL;
195 MPI_Comm *comm_child = NULL;
196 comex_igroup_t *igroup_parent = NULL;
197 MPI_Group *group_parent = NULL;
198 MPI_Comm *comm_parent = NULL;
199
200 /* create the node in the linked list of groups and */
201 /* get the child's MPI_Group and MPI_Comm, to be populated shortly */
202 comex_create_group_and_igroup(id_child, &igroup_child);
203 group_child = &(igroup_child->group);
204 comm_child = &(igroup_child->comm);
205
206 /* get the parent's MPI_Group and MPI_Comm */
207 igroup_parent = comex_get_igroup_from_group(id_parent);
208 group_parent = &(igroup_parent->group);
209 comm_parent = &(igroup_parent->comm);
210
211 status = MPI_Group_incl(*group_parent, n, pid_list, group_child);
212 if (status != MPI_SUCCESS) {
213 comex_error("MPI_Group_incl: Failed ", status);
214 }
215
216 {
217 MPI_Comm comm, comm1, comm2;
218 int lvl=1, local_ldr_pos;
219 MPI_Group_rank(*group_child, &grp_me);
220 if (grp_me == MPI_UNDEFINED) {
221 /* FIXME: keeping the group around for now */
222 return COMEX_SUCCESS;
223 }
224 /* SK: sanity check for the following bitwise operations */
225 assert(grp_me>=0);
226 MPI_Comm_dup(MPI_COMM_SELF, &comm); /* FIXME: can be optimized away */
227 local_ldr_pos = grp_me;
228 while(n>lvl) {
229 int tag=0;
230 int remote_ldr_pos = local_ldr_pos^lvl;
231 if (remote_ldr_pos < n) {
232 int remote_leader = pid_list[remote_ldr_pos];
233 MPI_Comm peer_comm = *comm_parent;
234 int high = (local_ldr_pos<remote_ldr_pos)?0:1;
235 MPI_Intercomm_create(
236 comm, 0, peer_comm, remote_leader, tag, &comm1);
237 MPI_Comm_free(&comm);
238 MPI_Intercomm_merge(comm1, high, &comm2);
239 MPI_Comm_free(&comm1);
240 comm = comm2;
241 }
242 local_ldr_pos &= ((~0)^lvl);
243 lvl<<=1;
244 }
245 *comm_child = comm;
246 /* cleanup temporary group (from MPI_Group_incl above) */
247 MPI_Group_free(group_child);
248 /* get the actual group associated with comm */
249 MPI_Comm_group(*comm_child, group_child);
250 }
251
252 return COMEX_SUCCESS;
253 }
254
255
256 /**
257 * Initialize group linked list. Prepopulate with world group.
258 */
comex_group_init()259 void comex_group_init()
260 {
261 /* create the head of the group linked list */
262 assert(group_list == NULL);
263 group_list = malloc(sizeof(comex_igroup_t));
264 group_list->id = COMEX_GROUP_WORLD;
265 group_list->next = NULL;
266
267 /* save MPI world group and communicatior in COMEX_GROUP_WORLD */
268 group_list->comm = l_state.world_comm;
269 MPI_Comm_group(group_list->comm, &(group_list->group));
270 }
271
272
comex_group_finalize()273 void comex_group_finalize()
274 {
275 comex_igroup_t *current_group_list_item = group_list;
276 comex_igroup_t *previous_group_list_item = NULL;
277
278 /* don't free the world group (the list head) */
279 current_group_list_item = current_group_list_item->next;
280
281 while (current_group_list_item != NULL) {
282 previous_group_list_item = current_group_list_item;
283 current_group_list_item = current_group_list_item->next;
284 comex_igroup_finalize(previous_group_list_item);
285 free(previous_group_list_item);
286 }
287
288 /* ok, now free the world group, but not the world comm */
289 MPI_Group_free(&(group_list->group));
290 free(group_list);
291 group_list = NULL;
292 }
293