1 /*
2 * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved.
3 * Copyright (c) 2014 The University of Tennessee and The University
4 * of Tennessee Research Foundation. All rights
5 * reserved.
6 * $COPYRIGHT$
7 *
8 * Additional copyrights may follow
9 *
10 * $HEADER$
11 */
12
13 #include "ompi_config.h"
14
15 #include "ompi/mca/osc/osc.h"
16 #include "ompi/mca/osc/base/base.h"
17 #include "ompi/mca/osc/base/osc_base_obj_convert.h"
18
19 #include "osc_portals4.h"
20
21 enum locktype_t {
22 lock_nocheck,
23 lock_exclusive,
24 lock_shared
25 };
26
27 struct ompi_osc_portals4_outstanding_lock_t {
28 opal_list_item_t super;
29 int target;
30 enum locktype_t lock_type;
31 };
32 typedef struct ompi_osc_portals4_outstanding_lock_t ompi_osc_portals4_outstanding_lock_t;
33 OBJ_CLASS_INSTANCE(ompi_osc_portals4_outstanding_lock_t, opal_list_item_t,
34 NULL, NULL);
35
36 static inline int
lk_cas64(ompi_osc_portals4_module_t * module,int target,int64_t write_val,int64_t comp_val,int64_t * result_val)37 lk_cas64(ompi_osc_portals4_module_t *module,
38 int target,
39 int64_t write_val,
40 int64_t comp_val,
41 int64_t *result_val)
42 {
43 int ret;
44 size_t offset = offsetof(ompi_osc_portals4_node_state_t, lock);
45
46 (void)opal_atomic_add_fetch_64(&module->opcount, 1);
47
48 ret = PtlSwap(module->md_h,
49 (ptl_size_t) result_val,
50 module->md_h,
51 (ptl_size_t) &write_val,
52 sizeof(int64_t),
53 ompi_osc_portals4_get_peer(module, target),
54 module->pt_idx,
55 module->match_bits | OSC_PORTALS4_MB_CONTROL,
56 offset,
57 NULL,
58 0,
59 &comp_val,
60 PTL_CSWAP,
61 PTL_INT64_T);
62 if (OMPI_SUCCESS != ret) {
63 return ret;
64 }
65
66 ret = ompi_osc_portals4_complete_all(module);
67 return ret;
68 }
69
70
71 static inline int
lk_write64(ompi_osc_portals4_module_t * module,int target,int64_t write_val)72 lk_write64(ompi_osc_portals4_module_t *module,
73 int target,
74 int64_t write_val)
75 {
76 int ret;
77 size_t offset = offsetof(ompi_osc_portals4_node_state_t, lock);
78
79 (void)opal_atomic_add_fetch_64(&module->opcount, 1);
80
81 ret = PtlPut(module->md_h,
82 (ptl_size_t) &write_val,
83 sizeof(int64_t),
84 PTL_ACK_REQ,
85 ompi_osc_portals4_get_peer(module, target),
86 module->pt_idx,
87 module->match_bits | OSC_PORTALS4_MB_CONTROL,
88 offset,
89 NULL,
90 0);
91 if (OMPI_SUCCESS != ret) {
92 return ret;
93 }
94
95 ret = ompi_osc_portals4_complete_all(module);
96 return ret;
97 }
98
99
100 static inline int
lk_add64(ompi_osc_portals4_module_t * module,int target,int64_t write_val,int64_t * result_val)101 lk_add64(ompi_osc_portals4_module_t *module,
102 int target,
103 int64_t write_val,
104 int64_t *result_val)
105 {
106 int ret;
107 size_t offset = offsetof(ompi_osc_portals4_node_state_t, lock);
108
109 (void)opal_atomic_add_fetch_64(&module->opcount, 1);
110
111 ret = PtlFetchAtomic(module->md_h,
112 (ptl_size_t) result_val,
113 module->md_h,
114 (ptl_size_t) &write_val,
115 sizeof(int64_t),
116 ompi_osc_portals4_get_peer(module, target),
117 module->pt_idx,
118 module->match_bits | OSC_PORTALS4_MB_CONTROL,
119 offset,
120 NULL,
121 0,
122 PTL_SUM,
123 PTL_INT64_T);
124 if (OMPI_SUCCESS != ret) {
125 return ret;
126 }
127
128 ret = ompi_osc_portals4_complete_all(module);
129 return ret;
130 }
131
132
133 static inline int
start_exclusive(ompi_osc_portals4_module_t * module,int target)134 start_exclusive(ompi_osc_portals4_module_t *module,
135 int target)
136 {
137 int64_t result;
138 int ret;
139
140 while (true) {
141 ret = lk_cas64(module, target, LOCK_EXCLUSIVE, 0, &result);
142 if (OMPI_SUCCESS != ret) return ret;
143 if (LOCK_ILLEGAL == (LOCK_ILLEGAL & result)) return OMPI_ERR_RMA_SYNC;
144 if (0 == result) break;
145 }
146
147 return OMPI_SUCCESS;
148 }
149
150
151 static inline int
end_exclusive(ompi_osc_portals4_module_t * module,int target)152 end_exclusive(ompi_osc_portals4_module_t *module,
153 int target)
154 {
155 int ret;
156
157 ret = lk_write64(module, target, LOCK_UNLOCKED);
158 return ret;
159 }
160
161
162 static inline int
start_shared(ompi_osc_portals4_module_t * module,int target)163 start_shared(ompi_osc_portals4_module_t *module,
164 int target)
165 {
166 int64_t result;
167 int ret;
168
169 while (true) {
170 ret = lk_add64(module, target, 1, &result);
171 if (OMPI_SUCCESS != ret) return ret;
172
173 if (result > (int64_t)LOCK_EXCLUSIVE) {
174 if (LOCK_ILLEGAL == (LOCK_ILLEGAL & result)) return OMPI_ERR_RMA_SYNC;
175 ret = lk_add64(module, target, -1, &result);
176 if (OMPI_SUCCESS != ret) return ret;
177 } else {
178 break;
179 }
180 }
181
182 return OMPI_SUCCESS;
183 }
184
185
186 static inline int
end_shared(ompi_osc_portals4_module_t * module,int target)187 end_shared(ompi_osc_portals4_module_t *module,
188 int target)
189 {
190 int64_t result;
191 int ret;
192
193 ret = lk_add64(module, target, -1, &result);
194 return ret;
195 }
196
197
198 int
ompi_osc_portals4_lock(int lock_type,int target,int assert,struct ompi_win_t * win)199 ompi_osc_portals4_lock(int lock_type,
200 int target,
201 int assert,
202 struct ompi_win_t *win)
203 {
204 ompi_osc_portals4_module_t *module =
205 (ompi_osc_portals4_module_t*) win->w_osc_module;
206 ompi_osc_portals4_outstanding_lock_t* lock;
207 int ret;
208
209 module->passive_target_access_epoch = true;
210
211 lock = OBJ_NEW(ompi_osc_portals4_outstanding_lock_t);
212 lock->target = target;
213
214 if (0 == (assert & MPI_MODE_NOCHECK)) {
215 if (MPI_LOCK_EXCLUSIVE == lock_type) {
216 lock->lock_type = lock_exclusive;
217 ret = start_exclusive(module, target);
218 } else {
219 lock->lock_type = lock_shared;
220 ret = start_shared(module, target);
221 }
222 } else {
223 lock->lock_type = lock_nocheck;
224 ret = OMPI_SUCCESS;
225 }
226
227 if (OMPI_SUCCESS == ret) {
228 opal_list_append(&module->outstanding_locks, &lock->super);
229 } else {
230 OBJ_RELEASE(lock);
231 }
232
233 return ret;
234 }
235
236
237 int
ompi_osc_portals4_unlock(int target,struct ompi_win_t * win)238 ompi_osc_portals4_unlock(int target,
239 struct ompi_win_t *win)
240 {
241 ompi_osc_portals4_module_t *module =
242 (ompi_osc_portals4_module_t*) win->w_osc_module;
243 ompi_osc_portals4_outstanding_lock_t *lock = NULL, *item;
244 int ret;
245
246 OPAL_LIST_FOREACH(item, &module->outstanding_locks,
247 ompi_osc_portals4_outstanding_lock_t) {
248 if (item->target == target) {
249 lock = item;
250 break;
251 }
252 }
253 if (NULL != item) {
254 opal_list_remove_item(&module->outstanding_locks, &lock->super);
255 } else {
256 return OMPI_ERR_RMA_SYNC;
257 }
258
259 ret = ompi_osc_portals4_complete_all(module);
260 if (ret != OMPI_SUCCESS) return ret;
261
262 if (lock->lock_type == lock_exclusive) {
263 ret = end_exclusive(module, target);
264 } else if (lock->lock_type == lock_shared) {
265 ret = end_shared(module, target);
266 } else {
267 ret = OMPI_SUCCESS;
268 }
269
270 module->passive_target_access_epoch = false;
271
272 OBJ_RELEASE(lock);
273
274 return ret;
275 }
276
277
278 int
ompi_osc_portals4_lock_all(int assert,struct ompi_win_t * win)279 ompi_osc_portals4_lock_all(int assert,
280 struct ompi_win_t *win)
281 {
282 ompi_osc_portals4_module_t *module =
283 (ompi_osc_portals4_module_t*) win->w_osc_module;
284 ompi_osc_portals4_outstanding_lock_t* lock;
285 int ret = OMPI_SUCCESS;
286
287 module->passive_target_access_epoch = true;
288
289 lock = OBJ_NEW(ompi_osc_portals4_outstanding_lock_t);
290 lock->target = -1;
291
292 if (0 == (assert & MPI_MODE_NOCHECK)) {
293 int i, comm_size;
294
295 lock->lock_type = lock_shared;
296 comm_size = ompi_comm_size(module->comm);
297
298 for (i = 0 ; i < comm_size ; ++i) {
299 ret |= start_shared(module, i);
300 }
301 } else {
302 lock->lock_type = lock_nocheck;
303 ret = OMPI_SUCCESS;
304 }
305
306 if (OMPI_SUCCESS == ret) {
307 opal_list_append(&module->outstanding_locks, &lock->super);
308 } else {
309 OBJ_RELEASE(lock);
310 }
311
312 return ret;
313 }
314
315
316 int
ompi_osc_portals4_unlock_all(struct ompi_win_t * win)317 ompi_osc_portals4_unlock_all(struct ompi_win_t *win)
318 {
319 ompi_osc_portals4_module_t *module =
320 (ompi_osc_portals4_module_t*) win->w_osc_module;
321 ompi_osc_portals4_outstanding_lock_t *lock = NULL, *item;
322 int ret;
323
324 OPAL_LIST_FOREACH(item, &module->outstanding_locks,
325 ompi_osc_portals4_outstanding_lock_t) {
326 if (item->target == -1) {
327 lock = item;
328 break;
329 }
330 }
331 if (NULL != item) {
332 opal_list_remove_item(&module->outstanding_locks, &lock->super);
333 } else {
334 return OMPI_ERR_RMA_SYNC;
335 }
336
337 ret = ompi_osc_portals4_complete_all(module);
338 if (ret != OMPI_SUCCESS) return ret;
339
340 if (lock->lock_type == lock_shared) {
341 int i, comm_size;
342
343 comm_size = ompi_comm_size(module->comm);
344
345 for (i = 0 ; i < comm_size ; ++i) {
346 ret |= end_shared(module, i);
347 }
348 }
349
350 module->passive_target_access_epoch = false;
351
352 OBJ_RELEASE(lock);
353
354 return OMPI_SUCCESS;
355 }
356
357
358 int
ompi_osc_portals4_sync(struct ompi_win_t * win)359 ompi_osc_portals4_sync(struct ompi_win_t *win)
360 {
361 /* Not sure this is strictly necessary, but why not? */
362 opal_atomic_mb();
363 PtlAtomicSync();
364
365 return OMPI_SUCCESS;
366 }
367
368
369 int
ompi_osc_portals4_flush(int target,struct ompi_win_t * win)370 ompi_osc_portals4_flush(int target,
371 struct ompi_win_t *win)
372 {
373 ompi_osc_portals4_module_t *module =
374 (ompi_osc_portals4_module_t*) win->w_osc_module;
375
376 /* flush is only allowed from within a passive target epoch */
377 if (!module->passive_target_access_epoch) {
378 return OMPI_ERR_RMA_SYNC;
379 }
380
381 return ompi_osc_portals4_complete_all(module);
382 }
383
384
385 int
ompi_osc_portals4_flush_all(struct ompi_win_t * win)386 ompi_osc_portals4_flush_all(struct ompi_win_t *win)
387 {
388 ompi_osc_portals4_module_t *module =
389 (ompi_osc_portals4_module_t*) win->w_osc_module;
390
391 /* flush is only allowed from within a passive target epoch */
392 if (!module->passive_target_access_epoch) {
393 return OMPI_ERR_RMA_SYNC;
394 }
395
396 return ompi_osc_portals4_complete_all(module);
397 }
398
399
400 int
ompi_osc_portals4_flush_local(int target,struct ompi_win_t * win)401 ompi_osc_portals4_flush_local(int target,
402 struct ompi_win_t *win)
403 {
404 ompi_osc_portals4_module_t *module =
405 (ompi_osc_portals4_module_t*) win->w_osc_module;
406
407 /* flush is only allowed from within a passive target epoch */
408 if (!module->passive_target_access_epoch) {
409 return OMPI_ERR_RMA_SYNC;
410 }
411
412 return ompi_osc_portals4_complete_all(module);
413 }
414
415
416 int
ompi_osc_portals4_flush_local_all(struct ompi_win_t * win)417 ompi_osc_portals4_flush_local_all(struct ompi_win_t *win)
418 {
419 ompi_osc_portals4_module_t *module =
420 (ompi_osc_portals4_module_t*) win->w_osc_module;
421
422 /* flush is only allowed from within a passive target epoch */
423 if (!module->passive_target_access_epoch) {
424 return OMPI_ERR_RMA_SYNC;
425 }
426
427 return ompi_osc_portals4_complete_all(module);
428 }
429