1 /*
2 * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights reserved.
3 * Copyright (c) 2015-2017 Cray Inc. All rights reserved.
4 * Copyright (c) 2019 Triad National Security, LLC. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <errno.h>
38 #include <getopt.h>
39 #include <poll.h>
40 #include <time.h>
41 #include <string.h>
42 #include <pthread.h>
43
44
45 #include <stdio.h>
46 #include <stdlib.h>
47 #include <inttypes.h>
48
49 #include "gnix_vc.h"
50 #include "gnix_cm_nic.h"
51 #include "gnix_hashtable.h"
52 #include "gnix_atomic.h"
53 #include "gnix_util.h"
54
55 #include <criterion/criterion.h>
56 #include "gnix_rdma_headers.h"
57 #include "common.h"
58
59 #if 1
60 #define dbg_printf(...)
61 #else
62 #define dbg_printf(...) \
63 do { \
64 printf(__VA_ARGS__); \
65 fflush(stdout); \
66 } while (0)
67 #endif
68
69 #define NUMEPS 2
70
71 /* Note: Set to ~FI_NOTIFY_FLAGS_ONLY since this was written before api 1.5 */
72 static uint64_t mode_bits = ~FI_NOTIFY_FLAGS_ONLY;
73 static struct fid_fabric *fab;
74 static struct fid_domain *dom[NUMEPS];
75 struct fi_gni_ops_domain *gni_domain_ops[NUMEPS];
76 static struct fid_ep *ep[NUMEPS];
77 static struct fid_av *av[NUMEPS];
78 static struct fi_info *hints;
79 static struct fi_info *fi;
80 void *ep_name[NUMEPS];
81 size_t gni_addr[NUMEPS];
82 static struct fid_cq *send_cq[NUMEPS];
83 static struct fid_cq *recv_cq[NUMEPS];
84 static struct fi_cq_attr cq_attr;
85
86 #define BUF_SZ (64*1024)
87 char *target, *target_base;
88 char *source, *source_base;
89 char *uc_source, *uc_source_base;
90 struct fid_mr *rem_mr[NUMEPS], *loc_mr[NUMEPS];
91 uint64_t mr_key[NUMEPS];
92
93 static struct fid_cntr *write_cntr[NUMEPS], *read_cntr[NUMEPS];
94 static struct fid_cntr *rwrite_cntr;
95 static struct fid_cntr *rread_cntr;
96 static struct fi_cntr_attr cntr_attr = {
97 .events = FI_CNTR_EVENTS_COMP,
98 .flags = 0
99 };
100 static uint64_t writes[NUMEPS] = {0}, reads[NUMEPS] = {0},
101 write_errs[NUMEPS] = {0}, read_errs[NUMEPS] = {0};
102
common_atomic_setup(uint32_t version,int mr_mode)103 static void common_atomic_setup(uint32_t version, int mr_mode)
104 {
105 int ret = 0, i = 0, j = 0;
106 struct fi_av_attr attr;
107 size_t addrlen = 0;
108
109 memset(&attr, 0, sizeof(attr));
110 attr.type = FI_AV_MAP;
111 attr.count = NUMEPS;
112
113 cq_attr.format = FI_CQ_FORMAT_TAGGED;
114 cq_attr.size = 1024;
115 cq_attr.wait_obj = 0;
116
117 hints->ep_attr->type = FI_EP_RDM;
118 hints->domain_attr->mr_mode = mr_mode;
119 hints->domain_attr->cq_data_size = 4;
120 hints->mode = mode_bits;
121 hints->fabric_attr->prov_name = strdup("gni");
122 hints->caps |= FI_ATOMIC | FI_READ | FI_REMOTE_READ |
123 FI_WRITE | FI_REMOTE_WRITE;
124
125 target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
126 assert(target_base);
127
128 source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
129 assert(source_base);
130
131 uc_source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
132 assert(uc_source_base);
133
134 target = GNIT_ALIGN_BUFFER(char *, target_base);
135 source = GNIT_ALIGN_BUFFER(char *, source_base);
136 uc_source = GNIT_ALIGN_BUFFER(char *, uc_source_base);
137
138 ret = fi_getinfo(version, NULL, 0, 0, hints, &fi);
139 cr_assert(!ret, "fi_getinfo");
140
141 ret = fi_fabric(fi->fabric_attr, &fab, NULL);
142 cr_assert(!ret, "fi_fabric");
143
144 for (; i < NUMEPS; i++) {
145 ret = fi_domain(fab, fi, dom + i, NULL);
146 cr_assert(!ret, "fi_domain");
147
148 ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
149 0, (void **) (gni_domain_ops + i), NULL);
150 cr_assert(!ret, "fi_open_ops");
151
152 ret = fi_av_open(dom[i], &attr, av + i, NULL);
153 cr_assert(!ret, "fi_av_open");
154
155 ret = fi_endpoint(dom[i], fi, &ep[i], NULL);
156 cr_assert(!ret, "fi_endpoint");
157
158 ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
159 cr_assert(addrlen > 0);
160
161 ep_name[i] = malloc(addrlen);
162 cr_assert(ep_name[i] != NULL);
163
164 ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
165 cr_assert(addrlen > 0);
166
167 ret = fi_cq_open(dom[i], &cq_attr, send_cq + i, 0);
168 cr_assert(!ret, "fi_cq_open");
169
170 ret = fi_ep_bind(ep[i], &send_cq[i]->fid, FI_TRANSMIT);
171 cr_assert(!ret, "fi_ep_bind");
172
173 ret = fi_cq_open(dom[i], &cq_attr, recv_cq + i, 0);
174 cr_assert(!ret, "fi_cq_open");
175
176 ret = fi_ep_bind(ep[i], &recv_cq[i]->fid, FI_RECV);
177 cr_assert(!ret, "fi_ep_bind");
178 }
179
180 for (i = 0; i < NUMEPS; i++) {
181 int target_requested_key = USING_SCALABLE(fi) ? (i * 2) : 0;
182 int source_requested_key = USING_SCALABLE(fi) ? (i * 2) + 1 : 0;
183
184 for (j = 0; j < NUMEPS; j++) {
185 ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
186 0, NULL);
187 cr_assert(ret == 1);
188 }
189
190 ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
191 cr_assert(!ret, "fi_ep_bind");
192
193 ret = fi_mr_reg(dom[i],
194 target,
195 BUF_SZ,
196 FI_REMOTE_WRITE,
197 0,
198 target_requested_key,
199 0,
200 rem_mr + i,
201 &target);
202 cr_assert_eq(ret, 0);
203
204 ret = fi_mr_reg(dom[i],
205 source,
206 BUF_SZ,
207 FI_REMOTE_WRITE,
208 0,
209 source_requested_key,
210 0,
211 loc_mr + i,
212 &source);
213 cr_assert_eq(ret, 0);
214
215 if (USING_SCALABLE(fi)) {
216 MR_ENABLE(rem_mr[i], target, BUF_SZ);
217 MR_ENABLE(loc_mr[i], source, BUF_SZ);
218 }
219
220 mr_key[i] = fi_mr_key(rem_mr[i]);
221
222 ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
223 cr_assert(!ret, "fi_cntr_open");
224
225 ret = fi_ep_bind(ep[i], &write_cntr[i]->fid, FI_WRITE);
226 cr_assert(!ret, "fi_ep_bind");
227
228 ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
229 cr_assert(!ret, "fi_cntr_open");
230
231 ret = fi_ep_bind(ep[i], &read_cntr[i]->fid, FI_READ);
232 cr_assert(!ret, "fi_ep_bind");
233
234 if (i != 1) {
235 ret = fi_enable(ep[i]);
236 cr_assert(!ret, "fi_ep_enable");
237 }
238 }
239
240 if (hints->caps & FI_RMA_EVENT) {
241 ret = fi_cntr_open(dom[1], &cntr_attr, &rwrite_cntr, 0);
242 cr_assert(!ret, "fi_cntr_open");
243
244 ret = fi_ep_bind(ep[1], &rwrite_cntr->fid, FI_REMOTE_WRITE);
245 cr_assert(!ret, "fi_ep_bind");
246
247 ret = fi_cntr_open(dom[1], &cntr_attr, &rread_cntr, 0);
248 cr_assert(!ret, "fi_cntr_open");
249
250 ret = fi_ep_bind(ep[1], &rread_cntr->fid, FI_REMOTE_READ);
251 cr_assert(!ret, "fi_ep_bind");
252
253 }
254
255 ret = fi_enable(ep[1]);
256 cr_assert(!ret, "fi_ep_enable");
257
258 }
259
__rdm_atomic_setup(uint32_t version,int mr_mode)260 static inline void __rdm_atomic_setup(uint32_t version, int mr_mode)
261 {
262 hints = fi_allocinfo();
263 cr_assert(hints, "fi_allocinfo");
264 common_atomic_setup(version, mr_mode);
265 }
266
rdm_atomic_default_setup(void)267 static void rdm_atomic_default_setup(void)
268 {
269 __rdm_atomic_setup(fi_version(), GNIX_DEFAULT_MR_MODE);
270 }
271
rdm_atomic_basic_setup(void)272 static void rdm_atomic_basic_setup(void)
273 {
274 __rdm_atomic_setup(fi_version(), GNIX_MR_BASIC);
275 }
276
rdm_atomic_scalable_setup(void)277 static void rdm_atomic_scalable_setup(void)
278 {
279 __rdm_atomic_setup(fi_version(), GNIX_MR_SCALABLE);
280 }
281
__rdm_atomic_rcntr_setup(uint32_t version,int mr_mode)282 static inline void __rdm_atomic_rcntr_setup(uint32_t version, int mr_mode)
283 {
284 hints = fi_allocinfo();
285 cr_assert(hints, "fi_allocinfo");
286 hints->caps = FI_RMA_EVENT;
287 common_atomic_setup(version, mr_mode);
288 }
289
rdm_atomic_rcntr_basic_setup(void)290 static void rdm_atomic_rcntr_basic_setup(void)
291 {
292 __rdm_atomic_rcntr_setup(fi_version(), GNIX_MR_BASIC);
293 }
294
rdm_atomic_rcntr_scalable_setup(void)295 static void rdm_atomic_rcntr_scalable_setup(void)
296 {
297 __rdm_atomic_rcntr_setup(fi_version(), GNIX_MR_SCALABLE);
298 }
299
rdm_atomic_teardown(void)300 void rdm_atomic_teardown(void)
301 {
302 int ret = 0, i = 0;
303
304 if (hints->caps & FI_RMA_EVENT) {
305 ret = fi_close(&rwrite_cntr->fid);
306 cr_assert(!ret, "failure in closing dom[1] rwrite counter.");
307
308 ret = fi_close(&rread_cntr->fid);
309 cr_assert(!ret, "failure in closing dom[1] rread counter.");
310 }
311
312 for (; i < NUMEPS; i++) {
313 ret = fi_close(&read_cntr[i]->fid);
314 cr_assert(!ret, "failure in closing read counter.");
315
316 ret = fi_close(&write_cntr[i]->fid);
317 cr_assert(!ret, "failure in closing write counter.");
318
319 ret = fi_close(&loc_mr[i]->fid);
320 cr_assert(!ret, "failure in closing av.");
321
322 ret = fi_close(&rem_mr[i]->fid);
323 cr_assert(!ret, "failure in closing av.");
324
325 ret = fi_close(&ep[i]->fid);
326 cr_assert(!ret, "failure in closing ep.");
327
328 ret = fi_close(&recv_cq[i]->fid);
329 cr_assert(!ret, "failure in recv cq.");
330
331 ret = fi_close(&send_cq[i]->fid);
332 cr_assert(!ret, "failure in send cq.");
333
334 ret = fi_close(&av[i]->fid);
335 cr_assert(!ret, "failure in closing av.");
336
337 ret = fi_close(&dom[i]->fid);
338 cr_assert(!ret, "failure in closing domain.");
339
340 free(ep_name[i]);
341 }
342
343 free(target_base);
344 free(source_base);
345 free(uc_source_base);
346
347 ret = fi_close(&fab->fid);
348 cr_assert(!ret, "failure in closing fabric.");
349
350 fi_freeinfo(fi);
351 fi_freeinfo(hints);
352 }
353
rdm_atomic_check_tcqe(struct fi_cq_tagged_entry * tcqe,void * ctx,uint64_t flags,uint64_t data)354 void rdm_atomic_check_tcqe(struct fi_cq_tagged_entry *tcqe, void *ctx,
355 uint64_t flags, uint64_t data)
356 {
357 cr_assert(tcqe->op_context == ctx, "CQE Context mismatch");
358 cr_assert(tcqe->flags == flags, "CQE flags mismatch");
359
360 if (flags & FI_REMOTE_CQ_DATA) {
361 cr_assert(tcqe->data == data, "CQE data invalid");
362 } else {
363 cr_assert(tcqe->data == 0, "CQE data invalid");
364 }
365
366 cr_assert(tcqe->len == 0, "CQE length mismatch");
367 cr_assert(tcqe->buf == 0, "CQE address mismatch");
368 cr_assert(tcqe->tag == 0, "CQE tag invalid");
369 }
370
rdm_atomic_check_cntrs(uint64_t w[],uint64_t r[],uint64_t w_e[],uint64_t r_e[])371 void rdm_atomic_check_cntrs(uint64_t w[], uint64_t r[], uint64_t w_e[],
372 uint64_t r_e[])
373 {
374 int i = 0;
375
376 for (; i < NUMEPS; i++) {
377 writes[i] += w[i];
378 reads[i] += r[i];
379 write_errs[i] += w_e[i];
380 read_errs[i] += r_e[i];
381
382 cr_assert(fi_cntr_read(write_cntr[i]) == writes[i],
383 "Bad write count");
384 cr_assert(fi_cntr_read(read_cntr[i]) == reads[i],
385 "Bad read count");
386 cr_assert(fi_cntr_readerr(write_cntr[i]) == write_errs[i],
387 "Bad write err count");
388 cr_assert(fi_cntr_readerr(read_cntr[i]) == read_errs[i],
389 "Bad read err count");
390 }
391
392 if (hints->caps & FI_RMA_EVENT) {
393 cr_assert(fi_cntr_read(rwrite_cntr) == writes[0],
394 "Bad rwrite count");
395 cr_assert(fi_cntr_read(rread_cntr) == reads[0],
396 "Bad rread count");
397 cr_assert(fi_cntr_readerr(rwrite_cntr) == 0,
398 "Bad rwrite err count");
399 cr_assert(fi_cntr_readerr(rread_cntr) == 0,
400 "Bad rread err count");
401 }
402 }
403
rdm_atomic_xfer_for_each_size(void (* xfer)(int len),int slen,int elen)404 void rdm_atomic_xfer_for_each_size(void (*xfer)(int len), int slen, int elen)
405 {
406 int i;
407
408 for (i = slen; i <= elen; i *= 2) {
409 xfer(i);
410 }
411 }
412
rdm_atomic_err_inject_enable(void)413 void rdm_atomic_err_inject_enable(void)
414 {
415 int ret, err_count_val = 1, i = 0;
416
417 for (; i < NUMEPS; i++) {
418 ret = gni_domain_ops[i]->set_val(&dom[i]->fid,
419 GNI_ERR_INJECT_COUNT,
420 &err_count_val);
421 cr_assert(!ret, "setval(GNI_ERR_INJECT_COUNT)");
422 }
423 }
424
425 /*******************************************************************************
426 * Test RMA functions
427 ******************************************************************************/
428
429 TestSuite(rdm_atomic_default,
430 .init = rdm_atomic_default_setup,
431 .fini = rdm_atomic_teardown,
432 .disabled = false);
433
434 TestSuite(rdm_atomic_basic,
435 .init = rdm_atomic_basic_setup,
436 .fini = rdm_atomic_teardown,
437 .disabled = false);
438
439 TestSuite(rdm_atomic_scalable,
440 .init = rdm_atomic_scalable_setup,
441 .fini = rdm_atomic_teardown,
442 .disabled = false);
443
444 #if 1
445 #define SOURCE_DATA 0xBBBB0000CCCCULL
446 #define TARGET_DATA 0xAAAA0000DDDDULL
447 #define SOURCE_DATA_FP 0.83203125
448 #define TARGET_DATA_FP 0.83984375
449 #else
450 /* swapped */
451 #define TARGET_DATA 0xB0000000CULL
452 #define SOURCE_DATA 0xA0000000DULL
453 #define TARGET_DATA_FP 0.83203125
454 #define SOURCE_DATA_FP 0.83984375
455 #endif
456 #define FETCH_SOURCE_DATA 0xACEDACEDULL
457 #define DATA_MASK 0xa5a5a5a5a5a5a5a5
458 #define U32_MASK 0xFFFFFFFFULL
459
460 #define ALL_GNI_DATATYPES_SUPPORTED { 0,0,0,0,1,1,1,1,1,1 }
461 #define GNI_DATATYPES_NO_FP_SUPPORTED { 0,0,0,0,1,1,1,1,0,0 }
462 #define NO_DATATYPES_SUPPORTED { }
463
464 /******************************************************************************
465 *
466 * Basic atomics
467 *
468 *****************************************************************************/
469
470 int supported_atomic_ops[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
471 [FI_MIN] = { 0,0,0,0,1,0,1,0,1,1 },
472 [FI_MAX] = { 0,0,0,0,1,0,1,0,1,1 },
473 [FI_SUM] = { 0,0,0,0,1,1,1,1,1,0 }, /* GNI DP sum is broken */
474 [FI_PROD] = NO_DATATYPES_SUPPORTED,
475 [FI_LOR] = NO_DATATYPES_SUPPORTED,
476 [FI_LAND] = NO_DATATYPES_SUPPORTED,
477 [FI_BOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
478 [FI_BAND] = GNI_DATATYPES_NO_FP_SUPPORTED,
479 [FI_LXOR] = NO_DATATYPES_SUPPORTED,
480 [FI_BXOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
481 [FI_ATOMIC_READ] = NO_DATATYPES_SUPPORTED,
482 [FI_ATOMIC_WRITE] = ALL_GNI_DATATYPES_SUPPORTED,
483 [FI_CSWAP] = NO_DATATYPES_SUPPORTED,
484 [FI_CSWAP_NE] = NO_DATATYPES_SUPPORTED,
485 [FI_CSWAP_LE] = NO_DATATYPES_SUPPORTED,
486 [FI_CSWAP_LT] = NO_DATATYPES_SUPPORTED,
487 [FI_CSWAP_GE] = NO_DATATYPES_SUPPORTED,
488 [FI_CSWAP_GT] = NO_DATATYPES_SUPPORTED,
489 [FI_MSWAP] = NO_DATATYPES_SUPPORTED,
490 };
491
do_invalid_atomic(enum fi_datatype dt,enum fi_op op)492 void do_invalid_atomic(enum fi_datatype dt, enum fi_op op)
493 {
494 ssize_t sz;
495 size_t count;
496
497 if (!supported_atomic_ops[op][dt]) {
498 sz = fi_atomic(ep[0], source, 1,
499 loc_mr[0], gni_addr[1],
500 _REM_ADDR(fi, target, target),
501 mr_key[1], dt, op, target);
502
503 cr_assert(sz == -FI_EOPNOTSUPP);
504
505 sz = fi_atomicvalid(ep[0], dt, op, &count);
506 cr_assert(sz == -FI_EOPNOTSUPP, "fi_atomicvalid() succeeded\n");
507 } else {
508 sz = fi_atomicvalid(ep[0], dt, op, &count);
509 cr_assert(!sz, "fi_atomicvalid() failed\n");
510 cr_assert(count == 1, "fi_atomicvalid(): bad count \n");
511 }
512 }
513
Test(rdm_atomic_default,invalid_atomic)514 Test(rdm_atomic_default, invalid_atomic)
515 {
516 int i, j;
517
518 for(i = 0; i < FI_ATOMIC_OP_LAST; i++) {
519 for(j = 0; j < FI_DATATYPE_LAST; j++) {
520 do_invalid_atomic(j, i);
521 }
522 }
523 }
524
do_min(int len)525 void do_min(int len)
526 {
527 int ret;
528 ssize_t sz;
529 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
530 (void *) -1, UINT_MAX, UINT_MAX };
531 uint64_t min;
532 float min_fp;
533 double min_dp;
534 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
535 uint64_t r_e[NUMEPS] = {0};
536
537 /* i64 */
538 *((int64_t *)source) = SOURCE_DATA;
539 *((int64_t *)target) = TARGET_DATA;
540 sz = fi_atomic(ep[0], source, 1,
541 loc_mr[0], gni_addr[1],
542 _REM_ADDR(fi, target, target), mr_key[1],
543 FI_INT64, FI_MIN, target);
544 cr_assert_eq(sz, 0);
545
546 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
547 pthread_yield();
548 }
549
550 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
551 dump_cq_error(send_cq[0], target, 0));
552 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
553
554 w[0] = 1;
555 rdm_atomic_check_cntrs(w, r, w_e, r_e);
556
557 dbg_printf("got write context event!\n");
558
559 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
560 SOURCE_DATA : TARGET_DATA;
561 ret = *((int64_t *)target) == min;
562 cr_assert(ret, "Data mismatch");
563
564 /* i32 */
565 *((int64_t *)source) = SOURCE_DATA;
566 *((int64_t *)target) = TARGET_DATA;
567 sz = fi_atomic(ep[0], source, 1,
568 loc_mr[0], gni_addr[1],
569 _REM_ADDR(fi, target, target), mr_key[1],
570 FI_INT32, FI_MIN, target);
571 cr_assert_eq(sz, 0);
572
573 /* reset cqe */
574 cqe.op_context = cqe.buf = (void *) -1;
575 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
576 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
577 pthread_yield();
578 }
579
580 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
581 dump_cq_error(send_cq[0], target, 0));
582 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
583 rdm_atomic_check_cntrs(w, r, w_e, r_e);
584
585 dbg_printf("got write context event!\n");
586
587 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
588 SOURCE_DATA : TARGET_DATA;
589 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
590 ret = *((int64_t *)target) == min;
591 cr_assert(ret, "Data mismatch");
592
593 /* float */
594 *((float *)source) = SOURCE_DATA_FP;
595 *((float *)target) = TARGET_DATA_FP;
596 sz = fi_atomic(ep[0], source, 1,
597 loc_mr[0], gni_addr[1],
598 _REM_ADDR(fi, target, target), mr_key[1],
599 FI_FLOAT, FI_MIN, target);
600 cr_assert_eq(sz, 0);
601
602 /* reset cqe */
603 cqe.op_context = cqe.buf = (void *) -1;
604 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
605 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
606 pthread_yield();
607 }
608
609 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
610 dump_cq_error(send_cq[0], target, 0));
611 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
612 rdm_atomic_check_cntrs(w, r, w_e, r_e);
613
614 dbg_printf("got write context event!\n");
615
616 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
617 SOURCE_DATA_FP : TARGET_DATA_FP;
618 ret = *((float *)target) == min_fp;
619 cr_assert(ret, "Data mismatch");
620
621 /* double */
622 *((double *)source) = SOURCE_DATA_FP;
623 *((double *)target) = TARGET_DATA_FP;
624 sz = fi_atomic(ep[0], source, 1,
625 loc_mr[0], gni_addr[1],
626 _REM_ADDR(fi, target, target), mr_key[1],
627 FI_DOUBLE, FI_MIN, target);
628 cr_assert_eq(sz, 0);
629
630 /* reset cqe */
631 cqe.op_context = cqe.buf = (void *) -1;
632 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
633 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
634 pthread_yield();
635 }
636
637 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
638 dump_cq_error(send_cq[0], target, 0));
639 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
640 rdm_atomic_check_cntrs(w, r, w_e, r_e);
641
642 dbg_printf("got write context event!\n");
643
644 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
645 SOURCE_DATA_FP : TARGET_DATA_FP;
646 ret = *((double *)target) == min_dp;
647 cr_assert(ret, "Data mismatch");
648 }
649
Test(rdm_atomic_basic,min)650 Test(rdm_atomic_basic, min)
651 {
652 rdm_atomic_xfer_for_each_size(do_min, 1, 1);
653 }
654
Test(rdm_atomic_scalable,min)655 Test(rdm_atomic_scalable, min)
656 {
657 rdm_atomic_xfer_for_each_size(do_min, 1, 1);
658 }
659
do_max(int len)660 void do_max(int len)
661 {
662 int ret;
663 ssize_t sz;
664 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
665 (void *) -1, UINT_MAX, UINT_MAX };
666 uint64_t min;
667 float min_fp;
668 double min_dp;
669 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
670 uint64_t r_e[NUMEPS] = {0};
671
672 /* i64 */
673 *((int64_t *)source) = SOURCE_DATA;
674 *((int64_t *)target) = TARGET_DATA;
675 sz = fi_atomic(ep[0], source, 1,
676 loc_mr[0], gni_addr[1],
677 _REM_ADDR(fi, target, target), mr_key[1],
678 FI_INT64, FI_MAX, target);
679 cr_assert_eq(sz, 0);
680
681 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
682 pthread_yield();
683 }
684
685 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
686 dump_cq_error(send_cq[0], target, 0));
687 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
688
689 w[0] = 1;
690 rdm_atomic_check_cntrs(w, r, w_e, r_e);
691
692 dbg_printf("got write context event!\n");
693
694 min = ((int64_t)SOURCE_DATA > (int64_t)TARGET_DATA) ?
695 SOURCE_DATA : TARGET_DATA;
696 ret = *((int64_t *)target) == min;
697 cr_assert(ret, "Data mismatch");
698
699 /* i32 */
700 *((int64_t *)source) = SOURCE_DATA;
701 *((int64_t *)target) = TARGET_DATA;
702 sz = fi_atomic(ep[0], source, 1,
703 loc_mr[0], gni_addr[1],
704 _REM_ADDR(fi, target, target), mr_key[1],
705 FI_INT32, FI_MAX, target);
706 cr_assert_eq(sz, 0);
707
708 /* reset cqe */
709 cqe.op_context = cqe.buf = (void *) -1;
710 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
711 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
712 pthread_yield();
713 }
714
715 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
716 dump_cq_error(send_cq[0], target, 0));
717 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
718 rdm_atomic_check_cntrs(w, r, w_e, r_e);
719
720 dbg_printf("got write context event!\n");
721
722 min = ((int32_t)SOURCE_DATA > (int32_t)TARGET_DATA) ?
723 SOURCE_DATA : TARGET_DATA;
724 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
725 ret = *((int64_t *)target) == min;
726 cr_assert(ret, "Data mismatch");
727
728 /* float */
729 *((float *)source) = SOURCE_DATA_FP;
730 *((float *)target) = TARGET_DATA_FP;
731 sz = fi_atomic(ep[0], source, 1,
732 loc_mr[0], gni_addr[1],
733 _REM_ADDR(fi, target, target), mr_key[1],
734 FI_FLOAT, FI_MAX, target);
735 cr_assert_eq(sz, 0);
736
737 /* reset cqe */
738 cqe.op_context = cqe.buf = (void *) -1;
739 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
740 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
741 pthread_yield();
742 }
743
744 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
745 dump_cq_error(send_cq[0], target, 0));
746 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
747 rdm_atomic_check_cntrs(w, r, w_e, r_e);
748
749 dbg_printf("got write context event!\n");
750
751 min_fp = (float)SOURCE_DATA_FP > (float)TARGET_DATA_FP ?
752 SOURCE_DATA_FP : TARGET_DATA_FP;
753 ret = *((float *)target) == min_fp;
754 cr_assert(ret, "Data mismatch");
755
756 /* double */
757 *((double *)source) = SOURCE_DATA_FP;
758 *((double *)target) = TARGET_DATA_FP;
759 sz = fi_atomic(ep[0], source, 1,
760 loc_mr[0], gni_addr[1],
761 _REM_ADDR(fi, target, target), mr_key[1],
762 FI_DOUBLE, FI_MAX, target);
763 cr_assert_eq(sz, 0);
764
765 /* reset cqe */
766 cqe.op_context = cqe.buf = (void *) -1;
767 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
768 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
769 pthread_yield();
770 }
771
772 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
773 dump_cq_error(send_cq[0], target, 0));
774 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
775 rdm_atomic_check_cntrs(w, r, w_e, r_e);
776
777 dbg_printf("got write context event!\n");
778
779 min_dp = (double)SOURCE_DATA_FP > (double)TARGET_DATA_FP ?
780 SOURCE_DATA_FP : TARGET_DATA_FP;
781 ret = *((double *)target) == min_dp;
782 cr_assert(ret, "Data mismatch");
783 }
784
Test(rdm_atomic_basic,max)785 Test(rdm_atomic_basic, max)
786 {
787 rdm_atomic_xfer_for_each_size(do_max, 1, 1);
788 }
789
Test(rdm_atomic_scalable,max)790 Test(rdm_atomic_scalable, max)
791 {
792 rdm_atomic_xfer_for_each_size(do_max, 1, 1);
793 }
794
do_sum(int len)795 void do_sum(int len)
796 {
797 int ret;
798 ssize_t sz;
799 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
800 (void *) -1, UINT_MAX, UINT_MAX };
801 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
802 uint64_t r_e[NUMEPS] = {0};
803
804 /* u64 */
805 *((uint64_t *)source) = SOURCE_DATA;
806 *((uint64_t *)target) = TARGET_DATA;
807 sz = fi_atomic(ep[0], source, 1,
808 loc_mr[0], gni_addr[1],
809 _REM_ADDR(fi, target, target), mr_key[1],
810 FI_UINT64, FI_SUM, target);
811 cr_assert_eq(sz, 0);
812
813 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
814 pthread_yield();
815 }
816
817 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
818 dump_cq_error(send_cq[0], target, 0));
819 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
820
821 w[0] = 1;
822 rdm_atomic_check_cntrs(w, r, w_e, r_e);
823
824 dbg_printf("got write context event!\n");
825
826 ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
827 cr_assert(ret, "Data mismatch");
828
829 /* U32 */
830 *((uint64_t *)source) = SOURCE_DATA;
831 *((uint64_t *)target) = TARGET_DATA;
832 sz = fi_atomic(ep[0], source, 1,
833 loc_mr[0], gni_addr[1],
834 _REM_ADDR(fi, target, target), mr_key[1],
835 FI_UINT32, FI_SUM, target);
836 cr_assert_eq(sz, 0);
837
838 /* reset cqe */
839 cqe.op_context = cqe.buf = (void *) -1;
840 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
841 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
842 pthread_yield();
843 }
844
845 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
846 dump_cq_error(send_cq[0], target, 0));
847 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
848 rdm_atomic_check_cntrs(w, r, w_e, r_e);
849
850 dbg_printf("got write context event!\n");
851
852 ret = *((uint64_t *)target) ==
853 (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
854 cr_assert(ret, "Data mismatch");
855
856 /* i64 */
857 *((uint64_t *)source) = SOURCE_DATA;
858 *((uint64_t *)target) = TARGET_DATA;
859 sz = fi_atomic(ep[0], source, 1,
860 loc_mr[0], gni_addr[1],
861 _REM_ADDR(fi, target, target), mr_key[1],
862 FI_INT64, FI_SUM, target);
863 cr_assert_eq(sz, 0);
864
865 /* reset cqe */
866 cqe.op_context = cqe.buf = (void *) -1;
867 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
868 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
869 pthread_yield();
870 }
871
872 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
873 dump_cq_error(send_cq[0], target, 0));
874 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
875 rdm_atomic_check_cntrs(w, r, w_e, r_e);
876
877 dbg_printf("got write context event!\n");
878
879 ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
880 cr_assert(ret, "Data mismatch");
881
882 /* i32 */
883 *((uint64_t *)source) = SOURCE_DATA;
884 *((uint64_t *)target) = TARGET_DATA;
885 sz = fi_atomic(ep[0], source, 1,
886 loc_mr[0], gni_addr[1],
887 _REM_ADDR(fi, target, target), mr_key[1],
888 FI_INT32, FI_SUM, target);
889 cr_assert_eq(sz, 0);
890
891 /* reset cqe */
892 cqe.op_context = cqe.buf = (void *) -1;
893 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
894 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
895 pthread_yield();
896 }
897
898 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
899 dump_cq_error(send_cq[0], target, 0));
900 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
901 rdm_atomic_check_cntrs(w, r, w_e, r_e);
902
903 dbg_printf("got write context event!\n");
904
905 ret = *((uint64_t *)target) ==
906 (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
907 cr_assert(ret, "Data mismatch");
908
909 /* float */
910 *((float *)source) = SOURCE_DATA_FP;
911 *((float *)target) = TARGET_DATA_FP;
912 sz = fi_atomic(ep[0], source, 1,
913 loc_mr[0], gni_addr[1],
914 _REM_ADDR(fi, target, target), mr_key[1],
915 FI_FLOAT, FI_SUM, target);
916 cr_assert_eq(sz, 0);
917
918 /* reset cqe */
919 cqe.op_context = cqe.buf = (void *) -1;
920 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
921 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
922 pthread_yield();
923 }
924
925 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
926 dump_cq_error(send_cq[0], target, 0));
927 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
928 rdm_atomic_check_cntrs(w, r, w_e, r_e);
929
930 dbg_printf("got write context event!\n");
931
932 ret = *((float *)target) ==
933 (float)(SOURCE_DATA_FP + TARGET_DATA_FP);
934 cr_assert(ret, "Data mismatch");
935 }
936
Test(rdm_atomic_basic,sum)937 Test(rdm_atomic_basic, sum)
938 {
939 rdm_atomic_xfer_for_each_size(do_sum, 1, 1);
940 }
941
Test(rdm_atomic_scalable,sum)942 Test(rdm_atomic_scalable, sum)
943 {
944 rdm_atomic_xfer_for_each_size(do_sum, 1, 1);
945 }
946
do_bor(int len)947 void do_bor(int len)
948 {
949 int ret;
950 ssize_t sz;
951 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
952 (void *) -1, UINT_MAX, UINT_MAX };
953 uint64_t res;
954 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
955 uint64_t r_e[NUMEPS] = {0};
956
957 /* u64 */
958 *((uint64_t *)source) = SOURCE_DATA;
959 *((uint64_t *)target) = TARGET_DATA;
960 sz = fi_atomic(ep[0], source, 1,
961 loc_mr[0], gni_addr[1],
962 _REM_ADDR(fi, target, target), mr_key[1],
963 FI_UINT64, FI_BOR, target);
964 cr_assert_eq(sz, 0);
965
966 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
967 pthread_yield();
968 }
969
970 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
971 dump_cq_error(send_cq[0], target, 0));
972 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
973
974 w[0] = 1;
975 rdm_atomic_check_cntrs(w, r, w_e, r_e);
976
977 dbg_printf("got write context event!\n");
978
979 res = SOURCE_DATA | TARGET_DATA;
980 ret = *((uint64_t *)target) == res;
981 cr_assert(ret, "Data mismatch");
982
983 /* U32 */
984 *((uint64_t *)source) = SOURCE_DATA;
985 *((uint64_t *)target) = TARGET_DATA;
986 sz = fi_atomic(ep[0], source, 1,
987 loc_mr[0], gni_addr[1],
988 _REM_ADDR(fi, target, target), mr_key[1],
989 FI_UINT32, FI_BOR, target);
990 cr_assert_eq(sz, 0);
991
992 /* reset cqe */
993 cqe.op_context = cqe.buf = (void *) -1;
994 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
995 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
996 pthread_yield();
997 }
998
999 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1000 dump_cq_error(send_cq[0], target, 0));
1001 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1002 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1003
1004 dbg_printf("got write context event!\n");
1005
1006 res = SOURCE_DATA | TARGET_DATA;
1007 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1008 ret = *((uint64_t *)target) == res;
1009 cr_assert(ret, "Data mismatch");
1010
1011 /* i64 */
1012 *((uint64_t *)source) = SOURCE_DATA;
1013 *((uint64_t *)target) = TARGET_DATA;
1014 sz = fi_atomic(ep[0], source, 1,
1015 loc_mr[0], gni_addr[1],
1016 _REM_ADDR(fi, target, target), mr_key[1],
1017 FI_INT64, FI_BOR, target);
1018 cr_assert_eq(sz, 0);
1019
1020 /* reset cqe */
1021 cqe.op_context = cqe.buf = (void *) -1;
1022 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1023 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1024 pthread_yield();
1025 }
1026
1027 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1028 dump_cq_error(send_cq[0], target, 0));
1029 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1030 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1031
1032 dbg_printf("got write context event!\n");
1033
1034 res = SOURCE_DATA | TARGET_DATA;
1035 ret = *((uint64_t *)target) == res;
1036 cr_assert(ret, "Data mismatch");
1037
1038 /* i32 */
1039 *((uint64_t *)source) = SOURCE_DATA;
1040 *((uint64_t *)target) = TARGET_DATA;
1041 sz = fi_atomic(ep[0], source, 1,
1042 loc_mr[0], gni_addr[1],
1043 _REM_ADDR(fi, target, target), mr_key[1],
1044 FI_INT32, FI_BOR, target);
1045 cr_assert_eq(sz, 0);
1046
1047 /* reset cqe */
1048 cqe.op_context = cqe.buf = (void *) -1;
1049 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1050 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1051 pthread_yield();
1052 }
1053
1054 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1055 dump_cq_error(send_cq[0], target, 0));
1056 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1057 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1058
1059 dbg_printf("got write context event!\n");
1060
1061 res = SOURCE_DATA | TARGET_DATA;
1062 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1063 ret = *((uint64_t *)target) == res;
1064 cr_assert(ret, "Data mismatch");
1065 }
1066
Test(rdm_atomic_basic,bor)1067 Test(rdm_atomic_basic, bor)
1068 {
1069 rdm_atomic_xfer_for_each_size(do_bor, 1, 1);
1070 }
1071
Test(rdm_atomic_scalable,bor)1072 Test(rdm_atomic_scalable, bor)
1073 {
1074 rdm_atomic_xfer_for_each_size(do_bor, 1, 1);
1075 }
1076
do_band(int len)1077 void do_band(int len)
1078 {
1079 int ret;
1080 ssize_t sz;
1081 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
1082 (void *) -1, UINT_MAX, UINT_MAX };
1083 uint64_t res;
1084 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
1085 uint64_t r_e[NUMEPS] = {0};
1086
1087 /* u64 */
1088 *((uint64_t *)source) = SOURCE_DATA;
1089 *((uint64_t *)target) = TARGET_DATA;
1090 sz = fi_atomic(ep[0], source, 1,
1091 loc_mr[0], gni_addr[1],
1092 _REM_ADDR(fi, target, target), mr_key[1],
1093 FI_UINT64, FI_BAND, target);
1094 cr_assert_eq(sz, 0);
1095
1096 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1097 pthread_yield();
1098 }
1099
1100 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1101 dump_cq_error(send_cq[0], target, 0));
1102 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1103
1104 w[0] = 1;
1105 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1106
1107 dbg_printf("got write context event!\n");
1108
1109 res = SOURCE_DATA & TARGET_DATA;
1110 ret = *((uint64_t *)target) == res;
1111 cr_assert(ret, "Data mismatch");
1112
1113 /* U32 */
1114 *((uint64_t *)source) = SOURCE_DATA;
1115 *((uint64_t *)target) = TARGET_DATA;
1116 sz = fi_atomic(ep[0], source, 1,
1117 loc_mr[0], gni_addr[1],
1118 _REM_ADDR(fi, target, target), mr_key[1],
1119 FI_UINT32, FI_BAND, target);
1120 cr_assert_eq(sz, 0);
1121
1122 /* reset cqe */
1123 cqe.op_context = cqe.buf = (void *) -1;
1124 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1125 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1126 pthread_yield();
1127 }
1128
1129 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1130 dump_cq_error(send_cq[0], target, 0));
1131 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1132 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1133
1134 dbg_printf("got write context event!\n");
1135
1136 res = SOURCE_DATA & TARGET_DATA;
1137 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1138 ret = *((uint64_t *)target) == res;
1139 cr_assert(ret, "Data mismatch");
1140
1141 /* i64 */
1142 *((uint64_t *)source) = SOURCE_DATA;
1143 *((uint64_t *)target) = TARGET_DATA;
1144 sz = fi_atomic(ep[0], source, 1,
1145 loc_mr[0], gni_addr[1],
1146 _REM_ADDR(fi, target, target), mr_key[1],
1147 FI_INT64, FI_BAND, target);
1148 cr_assert_eq(sz, 0);
1149
1150 /* reset cqe */
1151 cqe.op_context = cqe.buf = (void *) -1;
1152 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1153 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1154 pthread_yield();
1155 }
1156
1157 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1158 dump_cq_error(send_cq[0], target, 0));
1159 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1160 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1161
1162 dbg_printf("got write context event!\n");
1163
1164 res = SOURCE_DATA & TARGET_DATA;
1165 ret = *((uint64_t *)target) == res;
1166 cr_assert(ret, "Data mismatch");
1167
1168 /* i32 */
1169 *((uint64_t *)source) = SOURCE_DATA;
1170 *((uint64_t *)target) = TARGET_DATA;
1171 sz = fi_atomic(ep[0], source, 1,
1172 loc_mr[0], gni_addr[1],
1173 _REM_ADDR(fi, target, target), mr_key[1],
1174 FI_INT32, FI_BAND, target);
1175 cr_assert_eq(sz, 0);
1176
1177 /* reset cqe */
1178 cqe.op_context = cqe.buf = (void *) -1;
1179 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1180 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1181 pthread_yield();
1182 }
1183
1184 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1185 dump_cq_error(send_cq[0], target, 0));
1186 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1187 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1188
1189 dbg_printf("got write context event!\n");
1190
1191 res = SOURCE_DATA & TARGET_DATA;
1192 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1193 ret = *((uint64_t *)target) == res;
1194 cr_assert(ret, "Data mismatch");
1195 }
1196
Test(rdm_atomic_basic,band)1197 Test(rdm_atomic_basic, band)
1198 {
1199 rdm_atomic_xfer_for_each_size(do_band, 1, 1);
1200 }
1201
Test(rdm_atomic_scalable,band)1202 Test(rdm_atomic_scalable, band)
1203 {
1204 rdm_atomic_xfer_for_each_size(do_band, 1, 1);
1205 }
1206
do_bxor(int len)1207 void do_bxor(int len)
1208 {
1209 int ret;
1210 ssize_t sz;
1211 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
1212 (void *) -1, UINT_MAX, UINT_MAX };
1213 uint64_t res;
1214 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
1215 uint64_t r_e[NUMEPS] = {0};
1216
1217 /* u64 */
1218 *((uint64_t *)source) = SOURCE_DATA;
1219 *((uint64_t *)target) = TARGET_DATA;
1220 sz = fi_atomic(ep[0], source, 1,
1221 loc_mr[0], gni_addr[1],
1222 _REM_ADDR(fi, target, target), mr_key[1],
1223 FI_UINT64, FI_BXOR, target);
1224 cr_assert_eq(sz, 0);
1225
1226 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1227 pthread_yield();
1228 }
1229
1230 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1231 dump_cq_error(send_cq[0], target, 0));
1232 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1233
1234 w[0] = 1;
1235 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1236
1237 dbg_printf("got write context event!\n");
1238
1239 res = SOURCE_DATA ^ TARGET_DATA;
1240 ret = *((uint64_t *)target) == res;
1241 cr_assert(ret, "Data mismatch");
1242
1243 /* U32 */
1244 *((uint64_t *)source) = SOURCE_DATA;
1245 *((uint64_t *)target) = TARGET_DATA;
1246 sz = fi_atomic(ep[0], source, 1,
1247 loc_mr[0], gni_addr[1],
1248 _REM_ADDR(fi, target, target), mr_key[1],
1249 FI_UINT32, FI_BXOR, target);
1250 cr_assert_eq(sz, 0);
1251
1252 /* reset cqe */
1253 cqe.op_context = cqe.buf = (void *) -1;
1254 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1255 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1256 pthread_yield();
1257 }
1258
1259 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1260 dump_cq_error(send_cq[0], target, 0));
1261 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1262 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1263
1264 dbg_printf("got write context event!\n");
1265
1266 res = SOURCE_DATA ^ TARGET_DATA;
1267 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1268 ret = *((uint64_t *)target) == res;
1269 cr_assert(ret, "Data mismatch");
1270
1271 /* i64 */
1272 *((uint64_t *)source) = SOURCE_DATA;
1273 *((uint64_t *)target) = TARGET_DATA;
1274 sz = fi_atomic(ep[0], source, 1,
1275 loc_mr[0], gni_addr[1],
1276 _REM_ADDR(fi, target, target),
1277 mr_key[1],
1278 FI_INT64, FI_BXOR, target);
1279 cr_assert_eq(sz, 0);
1280
1281 /* reset cqe */
1282 cqe.op_context = cqe.buf = (void *) -1;
1283 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1284 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1285 pthread_yield();
1286 }
1287
1288 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1289 dump_cq_error(send_cq[0], target, 0));
1290 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1291 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1292
1293 dbg_printf("got write context event!\n");
1294
1295 res = SOURCE_DATA ^ TARGET_DATA;
1296 ret = *((uint64_t *)target) == res;
1297 cr_assert(ret, "Data mismatch");
1298
1299 /* i32 */
1300 *((uint64_t *)source) = SOURCE_DATA;
1301 *((uint64_t *)target) = TARGET_DATA;
1302 sz = fi_atomic(ep[0], source, 1,
1303 loc_mr[0], gni_addr[1],
1304 _REM_ADDR(fi, target, target),
1305 mr_key[1],
1306 FI_INT32, FI_BXOR, target);
1307 cr_assert_eq(sz, 0);
1308
1309 /* reset cqe */
1310 cqe.op_context = cqe.buf = (void *) -1;
1311 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1312 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1313 pthread_yield();
1314 }
1315
1316 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1317 dump_cq_error(send_cq[0], target, 0));
1318 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1319 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1320
1321 dbg_printf("got write context event!\n");
1322
1323 res = SOURCE_DATA ^ TARGET_DATA;
1324 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1325 ret = *((uint64_t *)target) == res;
1326 cr_assert(ret, "Data mismatch");
1327 }
1328
Test(rdm_atomic_basic,bxor)1329 Test(rdm_atomic_basic, bxor)
1330 {
1331 rdm_atomic_xfer_for_each_size(do_bxor, 1, 1);
1332 }
1333
Test(rdm_atomic_scalable,bxor)1334 Test(rdm_atomic_scalable, bxor)
1335 {
1336 rdm_atomic_xfer_for_each_size(do_bxor, 1, 1);
1337 }
1338
1339 #define AX_S_MASK 0x00000000FFFFFFFFUL
1340 #define AX_OP1 0x000000000FF0FFFFUL
1341 #define AX_OP2 0xFFFF0000FFFF0000UL
1342 #define AX_TGT_DATA 0x00FFFF0000FFFF00UL
do_axor(int len)1343 void do_axor(int len)
1344 {
1345 int ret;
1346 ssize_t sz;
1347 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
1348 (void *) -1, UINT_MAX, UINT_MAX };
1349 uint64_t exp;
1350 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
1351 uint64_t r_e[NUMEPS] = {0};
1352 struct fi_gni_ops_ep *ep_ops;
1353 uint64_t operand[2];
1354
1355 ret = fi_open_ops(&ep[0]->fid, FI_GNI_EP_OPS_1, 0,
1356 (void **) &ep_ops, NULL);
1357 cr_assert(!ret, "fi_open_ops endpoint");
1358
1359 /* u64 */
1360 operand[0] = AX_OP1; /* AND operand */
1361 operand[1] = AX_OP2; /* XOR operand */
1362 *((uint64_t *)target) = AX_TGT_DATA;
1363 dbg_printf("initial %016lx\n", *((uint64_t *)target));
1364
1365 sz = ep_ops->native_amo(ep[0], operand, 1, NULL, NULL,
1366 loc_mr[0], gni_addr[1],
1367 _REM_ADDR(fi, target, target),
1368 mr_key[1], FI_LONG_DOUBLE,
1369 GNIX_FAB_RQ_NAMO_AX, target);
1370 cr_assert_eq(sz, 0);
1371
1372 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1373 pthread_yield();
1374 }
1375
1376 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1377 dump_cq_error(send_cq[0], target, 0));
1378 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1379
1380 w[0] = 1;
1381 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1382
1383 dbg_printf("result %016lx\n", *((uint64_t *)target));
1384 exp = (AX_OP1 & AX_TGT_DATA) ^ AX_OP2;
1385 ret = *((uint64_t *)target) == exp;
1386 cr_assert(ret, "Data mismatch");
1387
1388 /* U32 */
1389 operand[0] = AX_OP1;
1390 operand[1] = AX_OP2;
1391 *((uint64_t *)target) = AX_TGT_DATA;
1392
1393 dbg_printf("initial %016lx\n", *((uint64_t *)target));
1394 dbg_printf("AX_OP1 %016lx\n", AX_OP1);
1395 dbg_printf("AX_OP2 %016lx\n", AX_OP2);
1396
1397 sz = ep_ops->native_amo(ep[0], operand, 1, NULL, NULL,
1398 loc_mr[0], gni_addr[1],
1399 _REM_ADDR(fi, target, target),
1400 mr_key[1], FI_UINT64,
1401 GNIX_FAB_RQ_NAMO_AX_S, target);
1402 cr_assert_eq(sz, 0);
1403
1404 /* reset cqe */
1405 cqe.op_context = cqe.buf = (void *) -1;
1406 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1407 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1408 pthread_yield();
1409 }
1410
1411 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1412 dump_cq_error(send_cq[0], target, 0));
1413 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1414
1415 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1416
1417 dbg_printf("AX_TGT_DATA & (AX_OP1 | ~AX_S_MASK) %016lx\n",
1418 AX_TGT_DATA & (AX_OP1 | ~AX_S_MASK));
1419 dbg_printf("AX_OP2 %016lx\n", AX_OP2);
1420 exp = (AX_TGT_DATA & (AX_OP1 | ~AX_S_MASK)) ^ (AX_OP2 & AX_S_MASK);
1421 ret = *((uint64_t *)target) == exp;
1422 cr_assert(ret, "Data mismatch expected %016lx: result %016lx",
1423 exp, *((uint64_t *)target));
1424 dbg_printf("result %016lx\n", *((uint64_t *)target));
1425
1426 /* fetching u64 */
1427 operand[0] = AX_OP1; /* AND operand */
1428 operand[1] = AX_OP2; /* XOR operand */
1429 *((uint64_t *)target) = AX_TGT_DATA;
1430 *((uint64_t *)source) = 0;
1431 dbg_printf("initial %016lx\n", *((uint64_t *)target));
1432
1433 sz = ep_ops->native_amo(ep[0], operand, 1, NULL, source,
1434 loc_mr[0], gni_addr[1],
1435 _REM_ADDR(fi, target, target),
1436 mr_key[1], FI_UINT64,
1437 GNIX_FAB_RQ_NAMO_FAX, target);
1438 cr_assert_eq(sz, 0);
1439
1440 /* reset cqe */
1441 cqe.op_context = cqe.buf = (void *) -1;
1442 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1443 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1444 pthread_yield();
1445 }
1446
1447 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1448 dump_cq_error(send_cq[0], target, 0));
1449 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1450
1451 w[0] = 0;
1452 r[0] = 1;
1453 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1454
1455 dbg_printf("result %016lx\n", *((uint64_t *)target));
1456 exp = (AX_OP1 & AX_TGT_DATA) ^ AX_OP2;
1457 ret = *((uint64_t *)target) == exp;
1458 cr_assert(ret, "Data mismatch");
1459 ret = *((uint64_t *)source) == AX_TGT_DATA;
1460 dbg_printf("fetchv %016lx\n", *((uint64_t *)source));
1461 cr_assert(ret, "Data mismatch expected %016lx: fetchv %016lx",
1462 AX_TGT_DATA, *((uint64_t *)source));
1463 cr_assert(ret, "Data mismatch");
1464
1465 /* fetching U32 */
1466 operand[0] = AX_OP1;
1467 operand[1] = AX_OP2;
1468 *((uint64_t *)target) = AX_TGT_DATA;
1469 *((uint64_t *)source) = 0;
1470
1471 dbg_printf("initial %016lx\n", *((uint64_t *)target));
1472 dbg_printf("source %016lx\n", *((uint64_t *)source));
1473 dbg_printf("AX_OP1 %016lx\n", AX_OP1);
1474 dbg_printf("AX_OP2 %016lx\n", AX_OP2);
1475
1476 sz = ep_ops->native_amo(ep[0], operand, 1, NULL, source,
1477 loc_mr[0], gni_addr[1],
1478 _REM_ADDR(fi, target, target),
1479 mr_key[1], FI_UINT32,
1480 GNIX_FAB_RQ_NAMO_FAX_S, target);
1481 cr_assert_eq(sz, 0);
1482
1483 /* reset cqe */
1484 cqe.op_context = cqe.buf = (void *) -1;
1485 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1486 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1487 pthread_yield();
1488 }
1489
1490 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1491 dump_cq_error(send_cq[0], target, 0));
1492 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1493
1494 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1495
1496 dbg_printf("AX_TGT_DATA & (AX_OP1 | ~AX_S_MASK) %016lx\n",
1497 AX_TGT_DATA & (AX_OP1 | ~AX_S_MASK));
1498 dbg_printf("AX_OP2 %016lx\n", AX_OP2);
1499 exp = (AX_TGT_DATA & (AX_OP1 | ~AX_S_MASK)) ^ (AX_OP2 & AX_S_MASK);
1500 ret = *((uint64_t *)target) == exp;
1501 cr_assert(ret, "Data mismatch expected %016lx: result %016lx",
1502 exp, *((uint64_t *)target));
1503 dbg_printf("result %016lx\n", *((uint64_t *)target));
1504 /* 32 bit fetch */
1505 ret = *((uint64_t *)source) == (AX_TGT_DATA & AX_S_MASK);
1506 dbg_printf("fetchv %016lx\n", *((uint64_t *)source));
1507 cr_assert(ret, "Data mismatch expected %016lx: fetchv %016lx",
1508 AX_TGT_DATA & AX_S_MASK, *((uint64_t *)source));
1509 }
1510
Test(rdm_atomic_basic,axor)1511 Test(rdm_atomic_basic, axor)
1512 {
1513 rdm_atomic_xfer_for_each_size(do_axor, 1, 1);
1514 }
1515
Test(rdm_atomic_scalable,axor)1516 Test(rdm_atomic_scalable, axor)
1517 {
1518 rdm_atomic_xfer_for_each_size(do_axor, 1, 1);
1519 }
1520
do_atomic_write(int len)1521 void do_atomic_write(int len)
1522 {
1523 int ret;
1524 ssize_t sz;
1525 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
1526 (void *) -1, UINT_MAX, UINT_MAX };
1527 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
1528 uint64_t r_e[NUMEPS] = {0};
1529
1530 /* u64 */
1531 *((uint64_t *)source) = SOURCE_DATA;
1532 *((uint64_t *)target) = TARGET_DATA;
1533 sz = fi_atomic(ep[0], source, 1,
1534 loc_mr[0], gni_addr[1],
1535 _REM_ADDR(fi, target, target), mr_key[1],
1536 FI_UINT64, FI_ATOMIC_WRITE, target);
1537 cr_assert_eq(sz, 0);
1538
1539 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1540 pthread_yield();
1541 }
1542
1543 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1544 dump_cq_error(send_cq[0], target, 0));
1545 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1546
1547 w[0] = 1;
1548 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1549
1550 dbg_printf("got write context event!\n");
1551
1552 ret = *((uint64_t *)target) == SOURCE_DATA;
1553 cr_assert(ret, "Data mismatch");
1554
1555 /* U32 */
1556 *((uint64_t *)source) = SOURCE_DATA;
1557 *((uint64_t *)target) = TARGET_DATA;
1558 sz = fi_atomic(ep[0], source, 1,
1559 loc_mr[0], gni_addr[1],
1560 _REM_ADDR(fi, target, target), mr_key[1],
1561 FI_UINT32, FI_ATOMIC_WRITE, target);
1562 cr_assert_eq(sz, 0);
1563
1564 /* reset cqe */
1565 cqe.op_context = cqe.buf = (void *) -1;
1566 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1567 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1568 pthread_yield();
1569 }
1570
1571 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1572 dump_cq_error(send_cq[0], target, 0));
1573 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1574 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1575
1576 dbg_printf("got write context event!\n");
1577
1578 ret = *((uint64_t *)target) ==
1579 (uint64_t)((SOURCE_DATA & U32_MASK) |
1580 (TARGET_DATA & (U32_MASK << 32)));
1581 cr_assert(ret, "Data mismatch");
1582
1583 /* i64 */
1584 *((uint64_t *)source) = SOURCE_DATA;
1585 *((uint64_t *)target) = TARGET_DATA;
1586 sz = fi_atomic(ep[0], source, 1,
1587 loc_mr[0], gni_addr[1],
1588 _REM_ADDR(fi, target, target), mr_key[1],
1589 FI_INT64, FI_ATOMIC_WRITE, target);
1590 cr_assert_eq(sz, 0);
1591
1592 /* reset cqe */
1593 cqe.op_context = cqe.buf = (void *) -1;
1594 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1595 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1596 pthread_yield();
1597 }
1598
1599 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1600 dump_cq_error(send_cq[0], target, 0));
1601 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1602 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1603
1604 dbg_printf("got write context event!\n");
1605
1606 ret = *((uint64_t *)target) == SOURCE_DATA;
1607 cr_assert(ret, "Data mismatch");
1608
1609 /* i32 */
1610 *((uint64_t *)source) = SOURCE_DATA;
1611 *((uint64_t *)target) = TARGET_DATA;
1612 sz = fi_atomic(ep[0], source, 1,
1613 loc_mr[0], gni_addr[1],
1614 _REM_ADDR(fi, target, target), mr_key[1],
1615 FI_INT32, FI_ATOMIC_WRITE, target);
1616 cr_assert_eq(sz, 0);
1617
1618 /* reset cqe */
1619 cqe.op_context = cqe.buf = (void *) -1;
1620 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1621 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1622 pthread_yield();
1623 }
1624
1625 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1626 dump_cq_error(send_cq[0], target, 0));
1627 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1628 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1629
1630 dbg_printf("got write context event!\n");
1631
1632 ret = *((uint64_t *)target) ==
1633 (uint64_t)((SOURCE_DATA & U32_MASK) |
1634 (TARGET_DATA & (U32_MASK << 32)));
1635 cr_assert(ret, "Data mismatch");
1636
1637 /* float */
1638 *((float *)source) = SOURCE_DATA_FP;
1639 *((float *)target) = TARGET_DATA_FP;
1640 sz = fi_atomic(ep[0], source, 1,
1641 loc_mr[0], gni_addr[1],
1642 _REM_ADDR(fi, target, target), mr_key[1],
1643 FI_FLOAT, FI_ATOMIC_WRITE, target);
1644 cr_assert_eq(sz, 0);
1645
1646 /* reset cqe */
1647 cqe.op_context = cqe.buf = (void *) -1;
1648 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1649 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1650 pthread_yield();
1651 }
1652
1653 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1654 dump_cq_error(send_cq[0], target, 0));
1655 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1656 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1657
1658 dbg_printf("got write context event!\n");
1659
1660 ret = *((float *)target) == (float)SOURCE_DATA_FP;
1661 cr_assert(ret, "Data mismatch");
1662
1663 /* double */
1664 *((double *)source) = SOURCE_DATA_FP;
1665 *((double *)target) = TARGET_DATA_FP;
1666 sz = fi_atomic(ep[0], source, 1,
1667 loc_mr[0], gni_addr[1],
1668 _REM_ADDR(fi, target, target), mr_key[1],
1669 FI_DOUBLE, FI_ATOMIC_WRITE, target);
1670 cr_assert_eq(sz, 0);
1671
1672 /* reset cqe */
1673 cqe.op_context = cqe.buf = (void *) -1;
1674 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1675 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1676 pthread_yield();
1677 }
1678
1679 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1680 dump_cq_error(send_cq[0], target, 0));
1681 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1682 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1683
1684 dbg_printf("got write context event!\n");
1685
1686 ret = *((double *)target) == (double)SOURCE_DATA_FP;
1687 cr_assert(ret, "Data mismatch");
1688 }
1689
Test(rdm_atomic_basic,write)1690 Test(rdm_atomic_basic, write)
1691 {
1692 rdm_atomic_xfer_for_each_size(do_atomic_write, 1, 1);
1693 }
1694
Test(rdm_atomic_scalable,write)1695 Test(rdm_atomic_scalable, write)
1696 {
1697 rdm_atomic_xfer_for_each_size(do_atomic_write, 1, 1);
1698 }
1699
do_min_buf(void * s,void * t)1700 void do_min_buf(void *s, void *t)
1701 {
1702 int ret;
1703 ssize_t sz;
1704 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
1705 (void *) -1, UINT_MAX, UINT_MAX };
1706 uint64_t min;
1707 float min_fp;
1708 double min_dp;
1709 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
1710 uint64_t r_e[NUMEPS] = {0};
1711
1712 /* i64 */
1713 *((int64_t *)s) = SOURCE_DATA;
1714 *((int64_t *)t) = TARGET_DATA;
1715 sz = fi_atomic(ep[0], s, 1,
1716 loc_mr[0], gni_addr[1],
1717 _REM_ADDR(fi, target, t), mr_key[1],
1718 FI_INT64, FI_MIN, t);
1719 if ((uint64_t)t & 0x7) {
1720 cr_assert_eq(sz, -FI_EINVAL);
1721 } else {
1722 cr_assert_eq(sz, 0);
1723
1724 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1725 pthread_yield();
1726 }
1727
1728 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1729 dump_cq_error(send_cq[0], t, 0));
1730 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
1731
1732 w[0] = 1;
1733 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1734
1735 dbg_printf("got write context event!\n");
1736
1737 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
1738 SOURCE_DATA : TARGET_DATA;
1739 ret = *((int64_t *)t) == min;
1740 cr_assert(ret, "Data mismatch");
1741 }
1742
1743 /* i32 */
1744 *((int64_t *)s) = SOURCE_DATA;
1745 *((int64_t *)t) = TARGET_DATA;
1746 sz = fi_atomic(ep[0], s, 1,
1747 loc_mr[0], gni_addr[1],
1748 _REM_ADDR(fi, target, t), mr_key[1],
1749 FI_INT32, FI_MIN, t);
1750 if ((uint64_t)t & 0x3) {
1751 cr_assert_eq(sz, -FI_EINVAL);
1752 } else {
1753 cr_assert_eq(sz, 0);
1754
1755 /* reset cqe */
1756 cqe.op_context = cqe.buf = (void *) -1;
1757 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1758 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1759 pthread_yield();
1760 }
1761
1762 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1763 dump_cq_error(send_cq[0], t, 0));
1764 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
1765
1766 w[0] = 1;
1767 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1768
1769 dbg_printf("got write context event!\n");
1770
1771 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
1772 SOURCE_DATA : TARGET_DATA;
1773 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1774 ret = *((int64_t *)t) == min;
1775 cr_assert(ret, "Data mismatch");
1776 }
1777
1778 /* float */
1779 *((float *)s) = SOURCE_DATA_FP;
1780 *((float *)t) = TARGET_DATA_FP;
1781 sz = fi_atomic(ep[0], s, 1,
1782 loc_mr[0], gni_addr[1],
1783 _REM_ADDR(fi, target, t), mr_key[1],
1784 FI_FLOAT, FI_MIN, t);
1785 if ((uint64_t)t & 0x3) {
1786 cr_assert_eq(sz, -FI_EINVAL);
1787 } else {
1788 cr_assert_eq(sz, 0);
1789
1790 /* reset cqe */
1791 cqe.op_context = cqe.buf = (void *) -1;
1792 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1793 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1794 pthread_yield();
1795 }
1796
1797 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1798 dump_cq_error(send_cq[0], t, 0));
1799 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
1800
1801 w[0] = 1;
1802 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1803
1804 dbg_printf("got write context event!\n");
1805
1806 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
1807 SOURCE_DATA_FP : TARGET_DATA_FP;
1808 ret = *((float *)t) == min_fp;
1809 cr_assert(ret, "Data mismatch");
1810 }
1811
1812 /* double */
1813 *((double *)s) = SOURCE_DATA_FP;
1814 *((double *)t) = TARGET_DATA_FP;
1815 sz = fi_atomic(ep[0], s, 1,
1816 loc_mr[0], gni_addr[1],
1817 _REM_ADDR(fi, target, t), mr_key[1],
1818 FI_DOUBLE, FI_MIN, t);
1819 if ((uint64_t)t & 0x7) {
1820 cr_assert_eq(sz, -FI_EINVAL);
1821 } else {
1822 cr_assert_eq(sz, 0);
1823
1824 /* reset cqe */
1825 cqe.op_context = cqe.buf = (void *) -1;
1826 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1827 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1828 pthread_yield();
1829 }
1830
1831 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1832 dump_cq_error(send_cq[0], t, 0));
1833 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_WRITE, 0);
1834
1835 w[0] = 1;
1836 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1837
1838 dbg_printf("got write context event!\n");
1839
1840 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
1841 SOURCE_DATA_FP : TARGET_DATA_FP;
1842 ret = *((double *)t) == min_dp;
1843 cr_assert(ret, "Data mismatch");
1844 }
1845 }
1846
__atomic_alignment(void)1847 static inline void __atomic_alignment(void)
1848 {
1849 int s_off, t_off;
1850
1851 for (s_off = 0; s_off < 7; s_off++) {
1852 for (t_off = 0; t_off < 7; t_off++) {
1853 do_min_buf(source + s_off, target + t_off);
1854 }
1855 }
1856 }
1857
Test(rdm_atomic_basic,atomic_alignment)1858 Test(rdm_atomic_basic, atomic_alignment)
1859 {
1860 __atomic_alignment();
1861 }
1862
Test(rdm_atomic_scalable,atomic_alignment)1863 Test(rdm_atomic_scalable, atomic_alignment)
1864 {
1865 __atomic_alignment();
1866 }
1867
__atomicv(void)1868 static inline void __atomicv(void)
1869 {
1870 int ret;
1871 ssize_t sz;
1872 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
1873 (void *) -1, UINT_MAX, UINT_MAX };
1874 uint64_t min;
1875 float min_fp;
1876 double min_dp;
1877 struct fi_ioc iov;
1878 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
1879 uint64_t r_e[NUMEPS] = {0};
1880
1881 iov.addr = source;
1882 iov.count = 1;
1883
1884 /* i64 */
1885 *((int64_t *)source) = SOURCE_DATA;
1886 *((int64_t *)target) = TARGET_DATA;
1887 sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
1888 _REM_ADDR(fi, target, target),
1889 mr_key[1], FI_INT64, FI_MIN, target);
1890 cr_assert_eq(sz, 0);
1891
1892 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1893 pthread_yield();
1894 }
1895
1896 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1897 dump_cq_error(send_cq[0], target, 0));
1898 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1899
1900 w[0] = 1;
1901 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1902
1903 dbg_printf("got write context event!\n");
1904
1905 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
1906 SOURCE_DATA : TARGET_DATA;
1907 ret = *((int64_t *)target) == min;
1908 cr_assert(ret, "Data mismatch");
1909
1910 /* i32 */
1911 *((int64_t *)source) = SOURCE_DATA;
1912 *((int64_t *)target) = TARGET_DATA;
1913 sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
1914 _REM_ADDR(fi, target, target),
1915 mr_key[1], FI_INT32, FI_MIN, target);
1916 cr_assert_eq(sz, 0);
1917
1918 /* reset cqe */
1919 cqe.op_context = cqe.buf = (void *) -1;
1920 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1921 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1922 pthread_yield();
1923 }
1924
1925 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1926 dump_cq_error(send_cq[0], target, 0));
1927 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1928 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1929
1930 dbg_printf("got write context event!\n");
1931
1932 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
1933 SOURCE_DATA : TARGET_DATA;
1934 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
1935 ret = *((int64_t *)target) == min;
1936 cr_assert(ret, "Data mismatch");
1937
1938 /* float */
1939 *((float *)source) = SOURCE_DATA_FP;
1940 *((float *)target) = TARGET_DATA_FP;
1941 sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
1942 _REM_ADDR(fi, target, target),
1943 mr_key[1], FI_FLOAT, FI_MIN, target);
1944 cr_assert_eq(sz, 0);
1945
1946 /* reset cqe */
1947 cqe.op_context = cqe.buf = (void *) -1;
1948 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1949 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1950 pthread_yield();
1951 }
1952
1953 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1954 dump_cq_error(send_cq[0], target, 0));
1955 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1956 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1957
1958 dbg_printf("got write context event!\n");
1959
1960 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
1961 SOURCE_DATA_FP : TARGET_DATA_FP;
1962 ret = *((float *)target) == min_fp;
1963 cr_assert(ret, "Data mismatch");
1964
1965 /* double */
1966 *((double *)source) = SOURCE_DATA_FP;
1967 *((double *)target) = TARGET_DATA_FP;
1968 sz = fi_atomicv(ep[0], &iov, (void **)loc_mr, 1, gni_addr[1],
1969 _REM_ADDR(fi, target, target),
1970 mr_key[1], FI_DOUBLE, FI_MIN, target);
1971 cr_assert_eq(sz, 0);
1972
1973 /* reset cqe */
1974 cqe.op_context = cqe.buf = (void *) -1;
1975 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
1976 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
1977 pthread_yield();
1978 }
1979
1980 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
1981 dump_cq_error(send_cq[0], target, 0));
1982 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
1983 rdm_atomic_check_cntrs(w, r, w_e, r_e);
1984
1985 dbg_printf("got write context event!\n");
1986
1987 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
1988 SOURCE_DATA_FP : TARGET_DATA_FP;
1989 ret = *((double *)target) == min_dp;
1990 cr_assert(ret, "Data mismatch");
1991 }
1992
Test(rdm_atomic_basic,atomicv)1993 Test(rdm_atomic_basic, atomicv)
1994 {
1995 __atomicv();
1996 }
1997
Test(rdm_atomic_scalable,atomicv)1998 Test(rdm_atomic_scalable, atomicv)
1999 {
2000 __atomicv();
2001 }
2002
__atomicmsg(void)2003 static inline void __atomicmsg(void)
2004 {
2005 int ret;
2006 ssize_t sz;
2007 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
2008 (void *) -1, UINT_MAX, UINT_MAX };
2009 uint64_t min;
2010 float min_fp;
2011 double min_dp;
2012 struct fi_msg_atomic msg;
2013 struct fi_ioc msg_iov;
2014 struct fi_rma_ioc rma_iov;
2015 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
2016 uint64_t r_e[NUMEPS] = {0};
2017
2018 msg_iov.addr = source;
2019 msg_iov.count = 1;
2020 msg.msg_iov = &msg_iov;
2021 msg.desc = (void **)loc_mr;
2022 msg.iov_count = 1;
2023 msg.addr = gni_addr[1];
2024 rma_iov.addr = _REM_ADDR(fi, target, target);
2025 rma_iov.count = 1;
2026 rma_iov.key = mr_key[1];
2027 msg.rma_iov = &rma_iov;
2028 msg.context = target;
2029 msg.op = FI_MIN;
2030
2031 /* i64 */
2032 *((int64_t *)source) = SOURCE_DATA;
2033 *((int64_t *)target) = TARGET_DATA;
2034 msg.datatype = FI_INT64;
2035 sz = fi_atomicmsg(ep[0], &msg, 0);
2036 cr_assert_eq(sz, 0);
2037
2038 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2039 pthread_yield();
2040 }
2041
2042 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2043 dump_cq_error(send_cq[0], NULL, 0));
2044 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
2045
2046 w[0] = 1;
2047 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2048
2049 dbg_printf("got write context event!\n");
2050
2051 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
2052 SOURCE_DATA : TARGET_DATA;
2053 ret = *((int64_t *)target) == min;
2054 cr_assert(ret, "Data mismatch");
2055
2056 /* i32 */
2057 *((int64_t *)source) = SOURCE_DATA;
2058 *((int64_t *)target) = TARGET_DATA;
2059 msg.datatype = FI_INT32;
2060 sz = fi_atomicmsg(ep[0], &msg, 0);
2061 cr_assert_eq(sz, 0);
2062
2063 /* reset cqe */
2064 cqe.op_context = cqe.buf = (void *) -1;
2065 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2066 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2067 pthread_yield();
2068 }
2069
2070 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2071 dump_cq_error(send_cq[0], NULL, 0));
2072 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
2073 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2074
2075 dbg_printf("got write context event!\n");
2076
2077 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
2078 SOURCE_DATA : TARGET_DATA;
2079 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
2080 ret = *((int64_t *)target) == min;
2081 cr_assert(ret, "Data mismatch");
2082
2083 /* float */
2084 *((float *)source) = SOURCE_DATA_FP;
2085 *((float *)target) = TARGET_DATA_FP;
2086 msg.datatype = FI_FLOAT;
2087 sz = fi_atomicmsg(ep[0], &msg, 0);
2088 cr_assert_eq(sz, 0);
2089
2090 /* reset cqe */
2091 cqe.op_context = cqe.buf = (void *) -1;
2092 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2093 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2094 pthread_yield();
2095 }
2096
2097 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2098 dump_cq_error(send_cq[0], NULL, 0));
2099 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
2100 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2101
2102 dbg_printf("got write context event!\n");
2103
2104 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
2105 SOURCE_DATA_FP : TARGET_DATA_FP;
2106 ret = *((float *)target) == min_fp;
2107 cr_assert(ret, "Data mismatch");
2108
2109 /* double */
2110 *((double *)source) = SOURCE_DATA_FP;
2111 *((double *)target) = TARGET_DATA_FP;
2112 msg.datatype = FI_DOUBLE;
2113 sz = fi_atomicmsg(ep[0], &msg, 0);
2114 cr_assert_eq(sz, 0);
2115
2116 /* reset cqe */
2117 cqe.op_context = cqe.buf = (void *) -1;
2118 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2119 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2120 pthread_yield();
2121 }
2122
2123 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2124 dump_cq_error(send_cq[0], NULL, 0));
2125 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_WRITE, 0);
2126 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2127
2128 dbg_printf("got write context event!\n");
2129
2130 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
2131 SOURCE_DATA_FP : TARGET_DATA_FP;
2132 ret = *((double *)target) == min_dp;
2133 cr_assert(ret, "Data mismatch");
2134 }
2135
Test(rdm_atomic_basic,atomicmsg)2136 Test(rdm_atomic_basic, atomicmsg)
2137 {
2138 __atomicmsg();
2139 }
2140
Test(rdm_atomic_scalable,atomicmsg)2141 Test(rdm_atomic_scalable, atomicmsg)
2142 {
2143 __atomicmsg();
2144 }
2145
__atomicinject(void)2146 static inline void __atomicinject(void)
2147 {
2148 int ret, loops;
2149 ssize_t sz;
2150 struct fi_cq_tagged_entry cqe;
2151 uint64_t min;
2152 float min_fp;
2153 double min_dp;
2154 static gnix_mr_cache_t *cache;
2155 struct gnix_fid_ep *ep_priv;
2156 int already_registered = 0;
2157
2158 /* i64 */
2159 *((int64_t *)source) = SOURCE_DATA;
2160 *((int64_t *)target) = TARGET_DATA;
2161
2162
2163 if (!USING_SCALABLE(fi)) {
2164 ep_priv = container_of(ep[0], struct gnix_fid_ep, ep_fid);
2165 cache = GET_DOMAIN_RW_CACHE(ep_priv->domain);
2166 cr_assert(cache != NULL);
2167 already_registered = ofi_atomic_get32(&cache->inuse.elements);
2168 }
2169
2170 sz = fi_inject_atomic(ep[0], source, 1,
2171 gni_addr[1],
2172 _REM_ADDR(fi, target, target), mr_key[1],
2173 FI_INT64, FI_MIN);
2174 cr_assert_eq(sz, 0);
2175
2176 if (!USING_SCALABLE(fi)) {
2177 /*
2178 * shouldn't have registered the source buffer,
2179 * trust but verify
2180 */
2181 cr_assert(ofi_atomic_get32(&cache->inuse.elements)
2182 == already_registered);
2183 }
2184
2185 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
2186 SOURCE_DATA : TARGET_DATA;
2187 loops = 0;
2188 while (*((int64_t *)target) != min) {
2189 ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
2190 cr_assert(ret == -FI_EAGAIN,
2191 "Received unexpected event\n");
2192
2193 pthread_yield();
2194 cr_assert(++loops < 10000, "Data mismatch");
2195 }
2196
2197 /* i32 */
2198 *((int64_t *)source) = SOURCE_DATA;
2199 *((int64_t *)target) = TARGET_DATA;
2200 sz = fi_inject_atomic(ep[0], source, 1,
2201 gni_addr[1],
2202 _REM_ADDR(fi, target, target), mr_key[1],
2203 FI_INT32, FI_MIN);
2204 cr_assert_eq(sz, 0);
2205
2206 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
2207 SOURCE_DATA : TARGET_DATA;
2208 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
2209 loops = 0;
2210 while (*((int64_t *)target) != min) {
2211 ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
2212 cr_assert(ret == -FI_EAGAIN,
2213 "Received unexpected event\n");
2214
2215 pthread_yield();
2216 cr_assert(++loops < 10000, "Data mismatch");
2217 }
2218
2219 /* float */
2220 *((float *)source) = SOURCE_DATA_FP;
2221 *((float *)target) = TARGET_DATA_FP;
2222 sz = fi_inject_atomic(ep[0], source, 1,
2223 gni_addr[1],
2224 _REM_ADDR(fi, target, target), mr_key[1],
2225 FI_FLOAT, FI_MIN);
2226 cr_assert_eq(sz, 0);
2227
2228 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
2229 SOURCE_DATA_FP : TARGET_DATA_FP;
2230 loops = 0;
2231 while (*((float *)target) != min_fp) {
2232 ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
2233 cr_assert(ret == -FI_EAGAIN,
2234 "Received unexpected event\n");
2235
2236 pthread_yield();
2237 cr_assert(++loops < 10000, "Data mismatch");
2238 }
2239
2240 /* double */
2241 *((double *)source) = SOURCE_DATA_FP;
2242 *((double *)target) = TARGET_DATA_FP;
2243 sz = fi_inject_atomic(ep[0], source, 1,
2244 gni_addr[1],
2245 _REM_ADDR(fi, target, target), mr_key[1],
2246 FI_DOUBLE, FI_MIN);
2247 cr_assert_eq(sz, 0);
2248
2249 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
2250 SOURCE_DATA_FP : TARGET_DATA_FP;
2251 loops = 0;
2252 while (*((double *)target) != min_dp) {
2253 ret = fi_cq_read(send_cq[0], &cqe, 1); /* for progress */
2254 cr_assert(ret == -FI_EAGAIN,
2255 "Received unexpected event\n");
2256
2257 pthread_yield();
2258 cr_assert(++loops < 10000, "Data mismatch");
2259 }
2260 }
2261
Test(rdm_atomic_basic,atomicinject)2262 Test(rdm_atomic_basic, atomicinject)
2263 {
2264 __atomicinject();
2265 }
2266
Test(rdm_atomic_scalable,atomicinject)2267 Test(rdm_atomic_scalable, atomicinject)
2268 {
2269 __atomicinject();
2270 }
2271
2272 /******************************************************************************
2273 *
2274 * Fetching atomics
2275 *
2276 *****************************************************************************/
2277
2278 int supported_fetch_atomic_ops[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
2279 [FI_MIN] = { 0,0,0,0,1,0,1,0,1,1 },
2280 [FI_MAX] = { 0,0,0,0,1,0,1,0,1,1 },
2281 [FI_SUM] = { 0,0,0,0,1,1,1,1,1,0 }, /* GNI DP sum is broken */
2282 [FI_PROD] = NO_DATATYPES_SUPPORTED,
2283 [FI_LOR] = NO_DATATYPES_SUPPORTED,
2284 [FI_LAND] = NO_DATATYPES_SUPPORTED,
2285 [FI_BOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
2286 [FI_BAND] = GNI_DATATYPES_NO_FP_SUPPORTED,
2287 [FI_LXOR] = NO_DATATYPES_SUPPORTED,
2288 [FI_BXOR] = GNI_DATATYPES_NO_FP_SUPPORTED,
2289 [FI_ATOMIC_READ] = ALL_GNI_DATATYPES_SUPPORTED,
2290 [FI_ATOMIC_WRITE] = ALL_GNI_DATATYPES_SUPPORTED,
2291 [FI_CSWAP] = NO_DATATYPES_SUPPORTED,
2292 [FI_CSWAP_NE] = NO_DATATYPES_SUPPORTED,
2293 [FI_CSWAP_LE] = NO_DATATYPES_SUPPORTED,
2294 [FI_CSWAP_LT] = NO_DATATYPES_SUPPORTED,
2295 [FI_CSWAP_GE] = NO_DATATYPES_SUPPORTED,
2296 [FI_CSWAP_GT] = NO_DATATYPES_SUPPORTED,
2297 [FI_MSWAP] = NO_DATATYPES_SUPPORTED,
2298 };
2299
do_invalid_fetch_atomic(enum fi_datatype dt,enum fi_op op)2300 void do_invalid_fetch_atomic(enum fi_datatype dt, enum fi_op op)
2301 {
2302 ssize_t sz;
2303 size_t count;
2304 uint64_t operand;
2305
2306 if (!supported_fetch_atomic_ops[op][dt]) {
2307 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2308 source, loc_mr[0],
2309 gni_addr[1],
2310 _REM_ADDR(fi, target, target),
2311 mr_key[1],
2312 dt, op, target);
2313 cr_assert(sz == -FI_EOPNOTSUPP);
2314
2315 sz = fi_fetch_atomicvalid(ep[0], dt, op, &count);
2316 cr_assert(sz == -FI_EOPNOTSUPP, "fi_atomicvalid() succeeded\n");
2317 } else {
2318 sz = fi_fetch_atomicvalid(ep[0], dt, op, &count);
2319 cr_assert(!sz, "fi_atomicvalid() failed\n");
2320 cr_assert(count == 1, "fi_atomicvalid(): bad count \n");
2321 }
2322 }
2323
Test(rdm_atomic_default,invalid_fetch_atomic)2324 Test(rdm_atomic_default, invalid_fetch_atomic)
2325 {
2326 int i, j;
2327
2328 for(i = 0; i < FI_ATOMIC_OP_LAST; i++) {
2329 for(j = 0; j < FI_DATATYPE_LAST; j++) {
2330 do_invalid_fetch_atomic(j, i);
2331 }
2332 }
2333 }
2334
do_fetch_min(int len)2335 void do_fetch_min(int len)
2336 {
2337 int ret;
2338 ssize_t sz;
2339 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
2340 (void *) -1, UINT_MAX, UINT_MAX };
2341 uint64_t min;
2342 float min_fp;
2343 double min_dp;
2344 uint64_t operand = SOURCE_DATA;
2345 float operand_fp;
2346 double operand_dp;
2347 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
2348 uint64_t r_e[NUMEPS] = {0};
2349
2350 /* i64 */
2351 *((int64_t *)source) = FETCH_SOURCE_DATA;
2352 *((int64_t *)target) = TARGET_DATA;
2353 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2354 source, loc_mr[0], gni_addr[1],
2355 _REM_ADDR(fi, target, target),
2356 mr_key[1], FI_INT64, FI_MIN, target);
2357 cr_assert_eq(sz, 0);
2358
2359 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2360 pthread_yield();
2361 }
2362
2363 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2364 dump_cq_error(send_cq[0], target, 0));
2365 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2366
2367 r[0] = 1;
2368 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2369
2370 dbg_printf("got write context event!\n");
2371
2372 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
2373 SOURCE_DATA : TARGET_DATA;
2374 ret = *((int64_t *)target) == min;
2375 cr_assert(ret, "Data mismatch");
2376 ret = *((int64_t *)source) == TARGET_DATA;
2377 cr_assert(ret, "Fetch data mismatch");
2378
2379 /* i32 */
2380 *((int64_t *)source) = FETCH_SOURCE_DATA;
2381 *((int64_t *)target) = TARGET_DATA;
2382 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2383 source, loc_mr[0], gni_addr[1],
2384 _REM_ADDR(fi, target, target),
2385 mr_key[1], FI_INT32, FI_MIN, target);
2386 cr_assert_eq(sz, 0);
2387
2388 /* reset cqe */
2389 cqe.op_context = cqe.buf = (void *) -1;
2390 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2391 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2392 pthread_yield();
2393 }
2394
2395 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2396 dump_cq_error(send_cq[0], target, 0));
2397 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2398 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2399
2400 dbg_printf("got write context event!\n");
2401
2402 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
2403 SOURCE_DATA : TARGET_DATA;
2404 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
2405 ret = *((int64_t *)target) == min;
2406 cr_assert(ret, "Data mismatch");
2407 min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
2408 ret = *((int64_t *)source) == min;
2409 cr_assert(ret, "Fetch data mismatch");
2410
2411 /* float */
2412 *((float *)&operand_fp) = SOURCE_DATA_FP;
2413 *((float *)source) = FETCH_SOURCE_DATA;
2414 *((float *)target) = TARGET_DATA_FP;
2415 sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
2416 source, loc_mr[0], gni_addr[1],
2417 _REM_ADDR(fi, target, target),
2418 mr_key[1], FI_FLOAT, FI_MIN, target);
2419 cr_assert_eq(sz, 0);
2420
2421 /* reset cqe */
2422 cqe.op_context = cqe.buf = (void *) -1;
2423 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2424 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2425 pthread_yield();
2426 }
2427
2428 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2429 dump_cq_error(send_cq[0], target, 0));
2430 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2431 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2432
2433 dbg_printf("got write context event!\n");
2434
2435 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
2436 SOURCE_DATA_FP : TARGET_DATA_FP;
2437 ret = *((float *)target) == min_fp;
2438 cr_assert(ret, "Data mismatch");
2439 ret = *((float *)source) == (float)TARGET_DATA_FP;
2440 cr_assert(ret, "Fetch data mismatch");
2441
2442 /* double */
2443 *((double *)&operand_dp) = SOURCE_DATA_FP;
2444 *((double *)source) = SOURCE_DATA_FP;
2445 *((double *)target) = TARGET_DATA_FP;
2446 sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
2447 source, loc_mr[0], gni_addr[1],
2448 _REM_ADDR(fi, target, target),
2449 mr_key[1], FI_DOUBLE, FI_MIN, target);
2450 cr_assert_eq(sz, 0);
2451
2452 /* reset cqe */
2453 cqe.op_context = cqe.buf = (void *) -1;
2454 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2455 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2456 pthread_yield();
2457 }
2458
2459 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2460 dump_cq_error(send_cq[0], target, 0));
2461 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2462 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2463
2464 dbg_printf("got write context event!\n");
2465
2466 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
2467 SOURCE_DATA_FP : TARGET_DATA_FP;
2468 ret = *((double *)target) == min_dp;
2469 cr_assert(ret, "Data mismatch");
2470 ret = *((double *)source) == (double)TARGET_DATA_FP;
2471 cr_assert(ret, "Fetch data mismatch");
2472 }
2473
Test(rdm_atomic_basic,fetch_min)2474 Test(rdm_atomic_basic, fetch_min)
2475 {
2476 rdm_atomic_xfer_for_each_size(do_fetch_min, 1, 1);
2477 }
2478
Test(rdm_atomic_scalable,fetch_min)2479 Test(rdm_atomic_scalable, fetch_min)
2480 {
2481 rdm_atomic_xfer_for_each_size(do_fetch_min, 1, 1);
2482 }
2483
do_fetch_max(int len)2484 void do_fetch_max(int len)
2485 {
2486 int ret;
2487 ssize_t sz;
2488 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
2489 (void *) -1, UINT_MAX, UINT_MAX };
2490 uint64_t min;
2491 float min_fp;
2492 double min_dp;
2493 uint64_t operand = SOURCE_DATA;
2494 float operand_fp;
2495 double operand_dp;
2496 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
2497 uint64_t r_e[NUMEPS] = {0};
2498
2499 /* i64 */
2500 *((int64_t *)source) = FETCH_SOURCE_DATA;
2501 *((int64_t *)target) = TARGET_DATA;
2502 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2503 source, loc_mr[0], gni_addr[1],
2504 _REM_ADDR(fi, target, target),
2505 mr_key[1], FI_INT64, FI_MAX, target);
2506 cr_assert_eq(sz, 0);
2507
2508 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2509 pthread_yield();
2510 }
2511
2512 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2513 dump_cq_error(send_cq[0], target, 0));
2514 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2515
2516 r[0] = 1;
2517 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2518
2519 dbg_printf("got write context event!\n");
2520
2521 min = ((int64_t)SOURCE_DATA > (int64_t)TARGET_DATA) ?
2522 SOURCE_DATA : TARGET_DATA;
2523 ret = *((int64_t *)target) == min;
2524 cr_assert(ret, "Data mismatch");
2525 ret = *((int64_t *)source) == TARGET_DATA;
2526 cr_assert(ret, "Fetch data mismatch");
2527
2528 /* i32 */
2529 *((int64_t *)source) = FETCH_SOURCE_DATA;
2530 *((int64_t *)target) = TARGET_DATA;
2531 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2532 source, loc_mr[0], gni_addr[1],
2533 _REM_ADDR(fi, target, target),
2534 mr_key[1], FI_INT32, FI_MAX, target);
2535 cr_assert_eq(sz, 0);
2536
2537 /* reset cqe */
2538 cqe.op_context = cqe.buf = (void *) -1;
2539 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2540 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2541 pthread_yield();
2542 }
2543
2544 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2545 dump_cq_error(send_cq[0], target, 0));
2546 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2547 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2548
2549 dbg_printf("got write context event!\n");
2550
2551 min = ((int32_t)SOURCE_DATA > (int32_t)TARGET_DATA) ?
2552 SOURCE_DATA : TARGET_DATA;
2553 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
2554 ret = *((int64_t *)target) == min;
2555 cr_assert(ret, "Data mismatch");
2556 min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
2557 ret = *((int64_t *)source) == min;
2558 cr_assert(ret, "Fetch data mismatch");
2559
2560 /* float */
2561 *((float *)&operand_fp) = SOURCE_DATA_FP;
2562 *((float *)source) = FETCH_SOURCE_DATA;
2563 *((float *)target) = TARGET_DATA_FP;
2564 sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
2565 source, loc_mr[0], gni_addr[1],
2566 _REM_ADDR(fi, target, target),
2567 mr_key[1], FI_FLOAT, FI_MAX, target);
2568 cr_assert_eq(sz, 0);
2569
2570 /* reset cqe */
2571 cqe.op_context = cqe.buf = (void *) -1;
2572 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2573 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2574 pthread_yield();
2575 }
2576
2577 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2578 dump_cq_error(send_cq[0], target, 0));
2579 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2580 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2581
2582 dbg_printf("got write context event!\n");
2583
2584 min_fp = (float)SOURCE_DATA_FP > (float)TARGET_DATA_FP ?
2585 SOURCE_DATA_FP : TARGET_DATA_FP;
2586 ret = *((float *)target) == min_fp;
2587 cr_assert(ret, "Data mismatch");
2588 ret = *((float *)source) == (float)TARGET_DATA_FP;
2589 cr_assert(ret, "Fetch data mismatch");
2590
2591 /* double */
2592 *((double *)&operand_dp) = SOURCE_DATA_FP;
2593 *((double *)source) = SOURCE_DATA_FP;
2594 *((double *)target) = TARGET_DATA_FP;
2595 sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
2596 source, loc_mr[0], gni_addr[1],
2597 _REM_ADDR(fi, target, target),
2598 mr_key[1], FI_DOUBLE, FI_MAX, target);
2599 cr_assert_eq(sz, 0);
2600
2601 /* reset cqe */
2602 cqe.op_context = cqe.buf = (void *) -1;
2603 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2604 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2605 pthread_yield();
2606 }
2607
2608 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2609 dump_cq_error(send_cq[0], target, 0));
2610 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2611 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2612
2613 dbg_printf("got write context event!\n");
2614
2615 min_dp = (double)SOURCE_DATA_FP > (double)TARGET_DATA_FP ?
2616 SOURCE_DATA_FP : TARGET_DATA_FP;
2617 ret = *((double *)target) == min_dp;
2618 cr_assert(ret, "Data mismatch");
2619 ret = *((double *)source) == (double)TARGET_DATA_FP;
2620 cr_assert(ret, "Fetch data mismatch");
2621 }
2622
Test(rdm_atomic_basic,fetch_max)2623 Test(rdm_atomic_basic, fetch_max)
2624 {
2625 rdm_atomic_xfer_for_each_size(do_fetch_max, 1, 1);
2626 }
2627
Test(rdm_atomic_scalable,fetch_max)2628 Test(rdm_atomic_scalable, fetch_max)
2629 {
2630 rdm_atomic_xfer_for_each_size(do_fetch_max, 1, 1);
2631 }
2632
do_fetch_sum(int len)2633 void do_fetch_sum(int len)
2634 {
2635 int ret;
2636 ssize_t sz;
2637 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
2638 (void *) -1, UINT_MAX, UINT_MAX };
2639 uint64_t operand = SOURCE_DATA;
2640 float operand_fp;
2641 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
2642 uint64_t r_e[NUMEPS] = {0};
2643
2644 /* u64 */
2645 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2646 *((uint64_t *)target) = TARGET_DATA;
2647 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2648 source, loc_mr[0], gni_addr[1],
2649 _REM_ADDR(fi, target, target),
2650 mr_key[1], FI_UINT64, FI_SUM, target);
2651 cr_assert_eq(sz, 0);
2652
2653 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2654 pthread_yield();
2655 }
2656
2657 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2658 dump_cq_error(send_cq[0], target, 0));
2659 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2660
2661 r[0] = 1;
2662 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2663
2664 dbg_printf("got write context event!\n");
2665
2666 ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
2667 cr_assert(ret, "Data mismatch");
2668 ret = *((uint64_t *)source) == TARGET_DATA;
2669 cr_assert(ret, "Fetch data mismatch");
2670
2671 /* U32 */
2672 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2673 *((uint64_t *)target) = TARGET_DATA;
2674 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2675 source, loc_mr[0], gni_addr[1],
2676 _REM_ADDR(fi, target, target),
2677 mr_key[1], FI_UINT32, FI_SUM, target);
2678 cr_assert_eq(sz, 0);
2679
2680 /* reset cqe */
2681 cqe.op_context = cqe.buf = (void *) -1;
2682 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2683 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2684 pthread_yield();
2685 }
2686
2687 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2688 dump_cq_error(send_cq[0], target, 0));
2689 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2690 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2691
2692 dbg_printf("got write context event!\n");
2693
2694 ret = *((uint64_t *)target) ==
2695 (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
2696 cr_assert(ret, "Data mismatch");
2697 ret = *((uint64_t *)source) ==
2698 (uint64_t)((TARGET_DATA & U32_MASK) |
2699 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
2700 cr_assert(ret, "Fetch data mismatch");
2701
2702 /* i64 */
2703 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2704 *((uint64_t *)target) = TARGET_DATA;
2705 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2706 source, loc_mr[0], gni_addr[1],
2707 _REM_ADDR(fi, target, target),
2708 mr_key[1], FI_INT64, FI_SUM, target);
2709 cr_assert_eq(sz, 0);
2710
2711 /* reset cqe */
2712 cqe.op_context = cqe.buf = (void *) -1;
2713 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2714 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2715 pthread_yield();
2716 }
2717
2718 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2719 dump_cq_error(send_cq[0], target, 0));
2720 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2721 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2722
2723 dbg_printf("got write context event!\n");
2724
2725 ret = *((uint64_t *)target) == (SOURCE_DATA + TARGET_DATA);
2726 cr_assert(ret, "Data mismatch");
2727 ret = *((uint64_t *)source) == TARGET_DATA;
2728 cr_assert(ret, "Fetch data mismatch");
2729
2730 /* i32 */
2731 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2732 *((uint64_t *)target) = TARGET_DATA;
2733 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2734 source, loc_mr[0], gni_addr[1],
2735 _REM_ADDR(fi, target, target),
2736 mr_key[1], FI_INT32, FI_SUM, target);
2737 cr_assert_eq(sz, 0);
2738
2739 /* reset cqe */
2740 cqe.op_context = cqe.buf = (void *) -1;
2741 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2742 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2743 pthread_yield();
2744 }
2745
2746 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2747 dump_cq_error(send_cq[0], target, 0));
2748 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2749 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2750
2751 dbg_printf("got write context event!\n");
2752
2753 ret = *((uint64_t *)target) ==
2754 (uint64_t)((SOURCE_DATA & U32_MASK) + TARGET_DATA);
2755 cr_assert(ret, "Data mismatch");
2756 ret = *((uint64_t *)source) ==
2757 (uint64_t)((TARGET_DATA & U32_MASK) |
2758 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
2759 cr_assert(ret, "Fetch data mismatch");
2760
2761 /* float */
2762 *((float *)&operand_fp) = SOURCE_DATA_FP;
2763 *((float *)source) = FETCH_SOURCE_DATA;
2764 *((float *)target) = TARGET_DATA_FP;
2765 sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
2766 source, loc_mr[0], gni_addr[1],
2767 _REM_ADDR(fi, target, target),
2768 mr_key[1], FI_FLOAT, FI_SUM, target);
2769 cr_assert_eq(sz, 0);
2770
2771 /* reset cqe */
2772 cqe.op_context = cqe.buf = (void *) -1;
2773 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2774 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2775 pthread_yield();
2776 }
2777
2778 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2779 dump_cq_error(send_cq[0], target, 0));
2780 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2781 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2782
2783 dbg_printf("got write context event!\n");
2784
2785 ret = *((float *)target) ==
2786 (float)(SOURCE_DATA_FP + TARGET_DATA_FP);
2787 cr_assert(ret, "Data mismatch");
2788 ret = *((float *)source) == (float)TARGET_DATA_FP;
2789 cr_assert(ret, "Fetch data mismatch");
2790 }
2791
Test(rdm_atomic_basic,fetch_sum)2792 Test(rdm_atomic_basic, fetch_sum)
2793 {
2794 rdm_atomic_xfer_for_each_size(do_fetch_sum, 1, 1);
2795 }
2796
Test(rdm_atomic_scalable,fetch_sum)2797 Test(rdm_atomic_scalable, fetch_sum)
2798 {
2799 rdm_atomic_xfer_for_each_size(do_fetch_sum, 1, 1);
2800 }
2801
do_fetch_bor(int len)2802 void do_fetch_bor(int len)
2803 {
2804 int ret;
2805 ssize_t sz;
2806 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
2807 (void *) -1, UINT_MAX, UINT_MAX };
2808 uint64_t res;
2809 uint64_t operand = SOURCE_DATA;
2810 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
2811 uint64_t r_e[NUMEPS] = {0};
2812
2813 /* u64 */
2814 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2815 *((uint64_t *)target) = TARGET_DATA;
2816 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2817 source, loc_mr[0], gni_addr[1],
2818 _REM_ADDR(fi, target, target),
2819 mr_key[1], FI_UINT64, FI_BOR, target);
2820 cr_assert_eq(sz, 0);
2821
2822 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2823 pthread_yield();
2824 }
2825
2826 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2827 dump_cq_error(send_cq[0], target, 0));
2828 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2829
2830 r[0] = 1;
2831 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2832
2833 dbg_printf("got write context event!\n");
2834
2835 res = SOURCE_DATA | TARGET_DATA;
2836 ret = *((uint64_t *)target) == res;
2837 cr_assert(ret, "Data mismatch");
2838 ret = *((uint64_t *)source) == TARGET_DATA;
2839 cr_assert(ret, "Fetch data mismatch");
2840
2841 /* U32 */
2842 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2843 *((uint64_t *)target) = TARGET_DATA;
2844 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2845 source, loc_mr[0], gni_addr[1],
2846 _REM_ADDR(fi, target, target),
2847 mr_key[1], FI_UINT32, FI_BOR, target);
2848 cr_assert_eq(sz, 0);
2849
2850 /* reset cqe */
2851 cqe.op_context = cqe.buf = (void *) -1;
2852 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2853 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2854 pthread_yield();
2855 }
2856
2857 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2858 dump_cq_error(send_cq[0], target, 0));
2859 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2860 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2861
2862 dbg_printf("got write context event!\n");
2863
2864 res = SOURCE_DATA | TARGET_DATA;
2865 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
2866 ret = *((uint64_t *)target) == res;
2867 cr_assert(ret, "Data mismatch");
2868 res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
2869 ret = *((uint64_t *)source) == res;
2870 cr_assert(ret, "Fetch data mismatch");
2871
2872 /* i64 */
2873 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2874 *((uint64_t *)target) = TARGET_DATA;
2875 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2876 source, loc_mr[0], gni_addr[1],
2877 _REM_ADDR(fi, target, target),
2878 mr_key[1], FI_INT64, FI_BOR, target);
2879 cr_assert_eq(sz, 0);
2880
2881 /* reset cqe */
2882 cqe.op_context = cqe.buf = (void *) -1;
2883 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2884 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2885 pthread_yield();
2886 }
2887
2888 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2889 dump_cq_error(send_cq[0], target, 0));
2890 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2891 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2892
2893 dbg_printf("got write context event!\n");
2894
2895 res = SOURCE_DATA | TARGET_DATA;
2896 ret = *((uint64_t *)target) == res;
2897 cr_assert(ret, "Data mismatch");
2898 ret = *((uint64_t *)source) == TARGET_DATA;
2899 cr_assert(ret, "Fetch data mismatch");
2900
2901 /* i32 */
2902 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2903 *((uint64_t *)target) = TARGET_DATA;
2904 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2905 source, loc_mr[0], gni_addr[1],
2906 _REM_ADDR(fi, target, target),
2907 mr_key[1], FI_INT32, FI_BOR, target);
2908 cr_assert_eq(sz, 0);
2909
2910 /* reset cqe */
2911 cqe.op_context = cqe.buf = (void *) -1;
2912 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2913 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2914 pthread_yield();
2915 }
2916
2917 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2918 dump_cq_error(send_cq[0], target, 0));
2919 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2920 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2921
2922 dbg_printf("got write context event!\n");
2923
2924 res = SOURCE_DATA | TARGET_DATA;
2925 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
2926 ret = *((uint64_t *)target) == res;
2927 cr_assert(ret, "Data mismatch");
2928 res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
2929 ret = *((uint64_t *)source) == res;
2930 cr_assert(ret, "Fetch data mismatch");
2931 }
2932
Test(rdm_atomic_basic,fetch_bor)2933 Test(rdm_atomic_basic, fetch_bor)
2934 {
2935 rdm_atomic_xfer_for_each_size(do_fetch_bor, 1, 1);
2936 }
2937
Test(rdm_atomic_scalable,fetch_bor)2938 Test(rdm_atomic_scalable, fetch_bor)
2939 {
2940 rdm_atomic_xfer_for_each_size(do_fetch_bor, 1, 1);
2941 }
2942
do_fetch_band(int len)2943 void do_fetch_band(int len)
2944 {
2945 int ret;
2946 ssize_t sz;
2947 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
2948 (void *) -1, UINT_MAX, UINT_MAX };
2949 uint64_t res;
2950 uint64_t operand = SOURCE_DATA;
2951 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
2952 uint64_t r_e[NUMEPS] = {0};
2953
2954 /* u64 */
2955 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2956 *((uint64_t *)target) = TARGET_DATA;
2957 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2958 source, loc_mr[0], gni_addr[1],
2959 _REM_ADDR(fi, target, target),
2960 mr_key[1], FI_UINT64, FI_BAND, target);
2961 cr_assert_eq(sz, 0);
2962
2963 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2964 pthread_yield();
2965 }
2966
2967 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2968 dump_cq_error(send_cq[0], target, 0));
2969 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
2970
2971 r[0] = 1;
2972 rdm_atomic_check_cntrs(w, r, w_e, r_e);
2973
2974 dbg_printf("got write context event!\n");
2975
2976 res = SOURCE_DATA & TARGET_DATA;
2977 ret = *((uint64_t *)target) == res;
2978 cr_assert(ret, "Data mismatch");
2979 ret = *((uint64_t *)source) == TARGET_DATA;
2980 cr_assert(ret, "Fetch data mismatch");
2981
2982 /* U32 */
2983 *((uint64_t *)source) = FETCH_SOURCE_DATA;
2984 *((uint64_t *)target) = TARGET_DATA;
2985 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
2986 source, loc_mr[0], gni_addr[1],
2987 _REM_ADDR(fi, target, target),
2988 mr_key[1], FI_UINT32, FI_BAND, target);
2989 cr_assert_eq(sz, 0);
2990
2991 /* reset cqe */
2992 cqe.op_context = cqe.buf = (void *) -1;
2993 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
2994 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
2995 pthread_yield();
2996 }
2997
2998 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
2999 dump_cq_error(send_cq[0], target, 0));
3000 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3001 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3002
3003 dbg_printf("got write context event!\n");
3004
3005 res = SOURCE_DATA & TARGET_DATA;
3006 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
3007 ret = *((uint64_t *)target) == res;
3008 cr_assert(ret, "Data mismatch");
3009 res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
3010 ret = *((uint64_t *)source) == res;
3011 cr_assert(ret, "Fetch data mismatch");
3012
3013 /* i64 */
3014 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3015 *((uint64_t *)target) = TARGET_DATA;
3016 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3017 source, loc_mr[0], gni_addr[1],
3018 _REM_ADDR(fi, target, target),
3019 mr_key[1], FI_INT64, FI_BAND, target);
3020 cr_assert_eq(sz, 0);
3021
3022 /* reset cqe */
3023 cqe.op_context = cqe.buf = (void *) -1;
3024 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3025 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3026 pthread_yield();
3027 }
3028
3029 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3030 dump_cq_error(send_cq[0], target, 0));
3031 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3032 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3033
3034 dbg_printf("got write context event!\n");
3035
3036 res = SOURCE_DATA & TARGET_DATA;
3037 ret = *((uint64_t *)target) == res;
3038 cr_assert(ret, "Data mismatch");
3039 ret = *((uint64_t *)source) == TARGET_DATA;
3040 cr_assert(ret, "Fetch data mismatch");
3041
3042 /* i32 */
3043 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3044 *((uint64_t *)target) = TARGET_DATA;
3045 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3046 source, loc_mr[0], gni_addr[1],
3047 _REM_ADDR(fi, target, target),
3048 mr_key[1], FI_INT32, FI_BAND, target);
3049 cr_assert_eq(sz, 0);
3050
3051 /* reset cqe */
3052 cqe.op_context = cqe.buf = (void *) -1;
3053 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3054 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3055 pthread_yield();
3056 }
3057
3058 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3059 dump_cq_error(send_cq[0], target, 0));
3060 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3061 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3062
3063 dbg_printf("got write context event!\n");
3064
3065 res = SOURCE_DATA & TARGET_DATA;
3066 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
3067 ret = *((uint64_t *)target) == res;
3068 cr_assert(ret, "Data mismatch");
3069 res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
3070 ret = *((uint64_t *)source) == res;
3071 cr_assert(ret, "Fetch data mismatch");
3072 }
3073
Test(rdm_atomic_basic,fetch_band)3074 Test(rdm_atomic_basic, fetch_band)
3075 {
3076 rdm_atomic_xfer_for_each_size(do_fetch_band, 1, 1);
3077 }
3078
Test(rdm_atomic_scalable,fetch_band)3079 Test(rdm_atomic_scalable, fetch_band)
3080 {
3081 rdm_atomic_xfer_for_each_size(do_fetch_band, 1, 1);
3082 }
3083
do_fetch_bxor(int len)3084 void do_fetch_bxor(int len)
3085 {
3086 int ret;
3087 ssize_t sz;
3088 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
3089 (void *) -1, UINT_MAX, UINT_MAX };
3090 uint64_t res;
3091 uint64_t operand = SOURCE_DATA;
3092 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
3093 uint64_t r_e[NUMEPS] = {0};
3094
3095 /* u64 */
3096 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3097 *((uint64_t *)target) = TARGET_DATA;
3098 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3099 source, loc_mr[0], gni_addr[1],
3100 _REM_ADDR(fi, target, target),
3101 mr_key[1], FI_UINT64, FI_BXOR, target);
3102 cr_assert_eq(sz, 0);
3103
3104 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3105 pthread_yield();
3106 }
3107
3108 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3109 dump_cq_error(send_cq[0], target, 0));
3110 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3111
3112 r[0] = 1;
3113 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3114
3115 dbg_printf("got write context event!\n");
3116
3117 res = SOURCE_DATA ^ TARGET_DATA;
3118 ret = *((uint64_t *)target) == res;
3119 cr_assert(ret, "Data mismatch");
3120 ret = *((uint64_t *)source) == TARGET_DATA;
3121 cr_assert(ret, "Fetch data mismatch");
3122
3123 /* U32 */
3124 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3125 *((uint64_t *)target) = TARGET_DATA;
3126 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3127 source, loc_mr[0], gni_addr[1],
3128 _REM_ADDR(fi, target, target),
3129 mr_key[1], FI_UINT32, FI_BXOR, target);
3130 cr_assert_eq(sz, 0);
3131
3132 /* reset cqe */
3133 cqe.op_context = cqe.buf = (void *) -1;
3134 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3135 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3136 pthread_yield();
3137 }
3138
3139 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3140 dump_cq_error(send_cq[0], target, 0));
3141 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3142 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3143
3144 dbg_printf("got write context event!\n");
3145
3146 res = SOURCE_DATA ^ TARGET_DATA;
3147 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
3148 ret = *((uint64_t *)target) == res;
3149 cr_assert(ret, "Data mismatch");
3150 res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
3151 ret = *((uint64_t *)source) == res;
3152 cr_assert(ret, "Fetch data mismatch");
3153
3154 /* i64 */
3155 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3156 *((uint64_t *)target) = TARGET_DATA;
3157 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3158 source, loc_mr[0], gni_addr[1],
3159 _REM_ADDR(fi, target, target),
3160 mr_key[1], FI_INT64, FI_BXOR, target);
3161 cr_assert_eq(sz, 0);
3162
3163 /* reset cqe */
3164 cqe.op_context = cqe.buf = (void *) -1;
3165 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3166 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3167 pthread_yield();
3168 }
3169
3170 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3171 dump_cq_error(send_cq[0], target, 0));
3172 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3173 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3174
3175 dbg_printf("got write context event!\n");
3176
3177 res = SOURCE_DATA ^ TARGET_DATA;
3178 ret = *((uint64_t *)target) == res;
3179 cr_assert(ret, "Data mismatch");
3180 ret = *((uint64_t *)source) == TARGET_DATA;
3181 cr_assert(ret, "Fetch data mismatch");
3182
3183 /* i32 */
3184 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3185 *((uint64_t *)target) = TARGET_DATA;
3186 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3187 source, loc_mr[0], gni_addr[1],
3188 _REM_ADDR(fi, target, target),
3189 mr_key[1], FI_INT32, FI_BXOR, target);
3190 cr_assert_eq(sz, 0);
3191
3192 /* reset cqe */
3193 cqe.op_context = cqe.buf = (void *) -1;
3194 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3195 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3196 pthread_yield();
3197 }
3198
3199 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3200 dump_cq_error(send_cq[0], target, 0));
3201 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3202 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3203
3204 dbg_printf("got write context event!\n");
3205
3206 res = SOURCE_DATA ^ TARGET_DATA;
3207 res = (res & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
3208 ret = *((uint64_t *)target) == res;
3209 cr_assert(ret, "Data mismatch");
3210 res = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
3211 ret = *((uint64_t *)source) == res;
3212 cr_assert(ret, "Fetch data mismatch");
3213 }
3214
Test(rdm_atomic_basic,fetch_bxor)3215 Test(rdm_atomic_basic, fetch_bxor)
3216 {
3217 rdm_atomic_xfer_for_each_size(do_fetch_bxor, 1, 1);
3218 }
3219
Test(rdm_atomic_scalable,fetch_bxor)3220 Test(rdm_atomic_scalable, fetch_bxor)
3221 {
3222 rdm_atomic_xfer_for_each_size(do_fetch_bxor, 1, 1);
3223 }
3224
do_fetch_atomic_write(int len)3225 void do_fetch_atomic_write(int len)
3226 {
3227 int ret;
3228 ssize_t sz;
3229 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
3230 (void *) -1, UINT_MAX, UINT_MAX };
3231 uint64_t operand = SOURCE_DATA;
3232 float operand_fp;
3233 double operand_dp;
3234 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
3235 uint64_t r_e[NUMEPS] = {0};
3236
3237 /* u64 */
3238 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3239 *((uint64_t *)target) = TARGET_DATA;
3240 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3241 source, loc_mr[0], gni_addr[1],
3242 _REM_ADDR(fi, target, target),
3243 mr_key[1], FI_UINT64, FI_ATOMIC_WRITE, target);
3244 cr_assert_eq(sz, 0);
3245
3246 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3247 pthread_yield();
3248 }
3249
3250 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3251 dump_cq_error(send_cq[0], target, 0));
3252 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3253
3254 r[0] = 1;
3255 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3256
3257 dbg_printf("got write context event!\n");
3258
3259 ret = *((uint64_t *)target) == SOURCE_DATA;
3260 cr_assert(ret, "Data mismatch");
3261 ret = *((uint64_t *)source) == TARGET_DATA;
3262 cr_assert(ret, "Fetch data mismatch");
3263
3264 /* U32 */
3265 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3266 *((uint64_t *)target) = TARGET_DATA;
3267 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3268 source, loc_mr[0], gni_addr[1],
3269 _REM_ADDR(fi, target, target),
3270 mr_key[1], FI_UINT32, FI_ATOMIC_WRITE, target);
3271 cr_assert_eq(sz, 0);
3272
3273 /* reset cqe */
3274 cqe.op_context = cqe.buf = (void *) -1;
3275 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3276 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3277 pthread_yield();
3278 }
3279
3280 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3281 dump_cq_error(send_cq[0], target, 0));
3282 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3283 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3284
3285 dbg_printf("got write context event!\n");
3286
3287 ret = *((uint64_t *)target) ==
3288 (uint64_t)((SOURCE_DATA & U32_MASK) |
3289 (TARGET_DATA & (U32_MASK << 32)));
3290 cr_assert(ret, "Data mismatch");
3291 ret = *((uint64_t *)source) ==
3292 (uint64_t)((TARGET_DATA & U32_MASK) |
3293 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
3294 cr_assert(ret, "Fetch data mismatch");
3295
3296 /* i64 */
3297 *((uint64_t *)source) = 0;
3298 *((uint64_t *)target) = TARGET_DATA;
3299 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3300 source, loc_mr[0], gni_addr[1],
3301 _REM_ADDR(fi, target, target),
3302 mr_key[1], FI_INT64, FI_ATOMIC_WRITE, target);
3303 cr_assert_eq(sz, 0);
3304
3305 /* reset cqe */
3306 cqe.op_context = cqe.buf = (void *) -1;
3307 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3308 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3309 pthread_yield();
3310 }
3311
3312 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3313 dump_cq_error(send_cq[0], target, 0));
3314 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3315 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3316
3317 dbg_printf("got write context event!\n");
3318
3319 ret = *((uint64_t *)target) == SOURCE_DATA;
3320 cr_assert(ret, "Data mismatch");
3321 ret = *((uint64_t *)source) == TARGET_DATA;
3322 cr_assert(ret, "Fetch data mismatch");
3323
3324 /* i32 */
3325 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3326 *((uint64_t *)target) = TARGET_DATA;
3327 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3328 source, loc_mr[0], gni_addr[1],
3329 _REM_ADDR(fi, target, target),
3330 mr_key[1], FI_INT32, FI_ATOMIC_WRITE, target);
3331 cr_assert_eq(sz, 0);
3332
3333 /* reset cqe */
3334 cqe.op_context = cqe.buf = (void *) -1;
3335 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3336 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3337 pthread_yield();
3338 }
3339
3340 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3341 dump_cq_error(send_cq[0], target, 0));
3342 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3343 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3344
3345 dbg_printf("got write context event!\n");
3346
3347 ret = *((uint64_t *)target) ==
3348 (uint64_t)((SOURCE_DATA & U32_MASK) |
3349 (TARGET_DATA & (U32_MASK << 32)));
3350 cr_assert(ret, "Data mismatch");
3351 ret = *((uint64_t *)source) ==
3352 (uint64_t)((TARGET_DATA & U32_MASK) |
3353 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
3354 cr_assert(ret, "Fetch data mismatch");
3355
3356 /* float */
3357 *((float *)&operand_fp) = SOURCE_DATA_FP;
3358 *((float *)source) = FETCH_SOURCE_DATA;
3359 *((float *)target) = TARGET_DATA_FP;
3360 sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
3361 source, loc_mr[0], gni_addr[1],
3362 _REM_ADDR(fi, target, target),
3363 mr_key[1], FI_FLOAT, FI_ATOMIC_WRITE, target);
3364 cr_assert_eq(sz, 0);
3365
3366 /* reset cqe */
3367 cqe.op_context = cqe.buf = (void *) -1;
3368 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3369 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3370 pthread_yield();
3371 }
3372
3373 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3374 dump_cq_error(send_cq[0], target, 0));
3375 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3376 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3377
3378 dbg_printf("got write context event!\n");
3379
3380 ret = *((float *)target) == (float)SOURCE_DATA_FP;
3381 cr_assert(ret, "Data mismatch");
3382 ret = *((float *)source) == (float)TARGET_DATA_FP;
3383 cr_assert(ret, "Fetch data mismatch");
3384
3385 /* double */
3386 *(double *)&operand_dp = SOURCE_DATA_FP;
3387 *((double *)source) = FETCH_SOURCE_DATA;
3388 *((double *)target) = TARGET_DATA_FP;
3389 sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
3390 source, loc_mr[0], gni_addr[1],
3391 _REM_ADDR(fi, target, target),
3392 mr_key[1], FI_DOUBLE, FI_ATOMIC_WRITE, target);
3393 cr_assert_eq(sz, 0);
3394
3395 /* reset cqe */
3396 cqe.op_context = cqe.buf = (void *) -1;
3397 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3398 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3399 pthread_yield();
3400 }
3401
3402 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3403 dump_cq_error(send_cq[0], target, 0));
3404 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3405 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3406
3407 dbg_printf("got write context event!\n");
3408
3409 ret = *((double *)target) == (double)SOURCE_DATA_FP;
3410 cr_assert(ret, "Data mismatch");
3411 ret = *((double *)source) == (double)TARGET_DATA_FP;
3412 cr_assert(ret, "Fetch data mismatch");
3413 }
3414
Test(rdm_atomic_basic,fetch_atomic_write)3415 Test(rdm_atomic_basic, fetch_atomic_write)
3416 {
3417 rdm_atomic_xfer_for_each_size(do_fetch_atomic_write, 1, 1);
3418 }
3419
Test(rdm_atomic_scalable,fetch_atomic_write)3420 Test(rdm_atomic_scalable, fetch_atomic_write)
3421 {
3422 rdm_atomic_xfer_for_each_size(do_fetch_atomic_write, 1, 1);
3423 }
3424
do_fetch_atomic_read(int len)3425 void do_fetch_atomic_read(int len)
3426 {
3427 int ret;
3428 ssize_t sz;
3429 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
3430 (void *) -1, UINT_MAX, UINT_MAX };
3431 float operand_fp;
3432 double operand_dp;
3433 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
3434 uint64_t r_e[NUMEPS] = {0};
3435
3436 /* u64 */
3437 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3438 *((uint64_t *)target) = TARGET_DATA;
3439 sz = fi_fetch_atomic(ep[0], NULL, 1, NULL,
3440 source, loc_mr[0], gni_addr[1],
3441 _REM_ADDR(fi, target, target),
3442 mr_key[1], FI_UINT64, FI_ATOMIC_READ, target);
3443 cr_assert_eq(sz, 0);
3444
3445 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3446 pthread_yield();
3447 }
3448
3449 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3450 dump_cq_error(send_cq[0], target, 0));
3451 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3452
3453 r[0] = 1;
3454 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3455
3456 dbg_printf("got write context event!\n");
3457
3458 ret = *((uint64_t *)target) == TARGET_DATA;
3459 cr_assert(ret, "Data mismatch");
3460 ret = *((uint64_t *)source) == (uint64_t)TARGET_DATA;
3461 cr_assert(ret, "Fetch data mismatch");
3462
3463 /* U32 */
3464 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3465 *((uint64_t *)target) = TARGET_DATA;
3466 sz = fi_fetch_atomic(ep[0], NULL, 1, NULL,
3467 source, loc_mr[0], gni_addr[1],
3468 _REM_ADDR(fi, target, target),
3469 mr_key[1], FI_UINT32, FI_ATOMIC_READ, target);
3470 cr_assert_eq(sz, 0);
3471
3472 /* reset cqe */
3473 cqe.op_context = cqe.buf = (void *) -1;
3474 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3475 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3476 pthread_yield();
3477 }
3478
3479 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3480 dump_cq_error(send_cq[0], target, 0));
3481 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3482 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3483
3484 dbg_printf("got write context event!\n");
3485
3486 ret = *((uint64_t *)target) == TARGET_DATA;
3487 cr_assert(ret, "Data mismatch");
3488 ret = *((uint64_t *)source) ==
3489 (uint64_t)((TARGET_DATA & U32_MASK) |
3490 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
3491 cr_assert(ret, "Fetch data mismatch");
3492
3493 /* i64 */
3494 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3495 *((uint64_t *)target) = TARGET_DATA;
3496 sz = fi_fetch_atomic(ep[0], NULL, 1, NULL,
3497 source, loc_mr[0], gni_addr[1],
3498 _REM_ADDR(fi, target, target),
3499 mr_key[1], FI_INT64, FI_ATOMIC_READ, target);
3500 cr_assert_eq(sz, 0);
3501
3502 /* reset cqe */
3503 cqe.op_context = cqe.buf = (void *) -1;
3504 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3505 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3506 pthread_yield();
3507 }
3508
3509 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3510 dump_cq_error(send_cq[0], target, 0));
3511 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3512 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3513
3514 dbg_printf("got write context event!\n");
3515
3516 ret = *((uint64_t *)target) == TARGET_DATA;
3517 cr_assert(ret, "Data mismatch");
3518 ret = *((uint64_t *)source) == TARGET_DATA;
3519 cr_assert(ret, "Fetch data mismatch");
3520
3521 /* i32 */
3522 *((uint64_t *)source) = FETCH_SOURCE_DATA;
3523 *((uint64_t *)target) = TARGET_DATA;
3524 sz = fi_fetch_atomic(ep[0], NULL, 1, NULL,
3525 source, loc_mr[0], gni_addr[1],
3526 _REM_ADDR(fi, target, target),
3527 mr_key[1], FI_INT32, FI_ATOMIC_READ, target);
3528 cr_assert_eq(sz, 0);
3529
3530 /* reset cqe */
3531 cqe.op_context = cqe.buf = (void *) -1;
3532 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3533 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3534 pthread_yield();
3535 }
3536
3537 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3538 dump_cq_error(send_cq[0], target, 0));
3539 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3540 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3541
3542 dbg_printf("got write context event!\n");
3543
3544 ret = *((uint64_t *)target) == TARGET_DATA;
3545 cr_assert(ret, "Data mismatch");
3546 ret = *((uint64_t *)source) ==
3547 (uint64_t)((TARGET_DATA & U32_MASK) |
3548 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
3549 cr_assert(ret, "Fetch data mismatch");
3550
3551 /* float */
3552 *((float *)&operand_fp) = SOURCE_DATA_FP;
3553 *((float *)source) = FETCH_SOURCE_DATA;
3554 *((float *)target) = TARGET_DATA_FP;
3555 sz = fi_fetch_atomic(ep[0], NULL, 1, NULL,
3556 source, loc_mr[0], gni_addr[1],
3557 _REM_ADDR(fi, target, target),
3558 mr_key[1], FI_FLOAT, FI_ATOMIC_READ, target);
3559 cr_assert_eq(sz, 0);
3560
3561 /* reset cqe */
3562 cqe.op_context = cqe.buf = (void *) -1;
3563 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3564 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3565 pthread_yield();
3566 }
3567
3568 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3569 dump_cq_error(send_cq[0], target, 0));
3570 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3571 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3572
3573 dbg_printf("got write context event!\n");
3574
3575 ret = *((float *)target) == (float)TARGET_DATA_FP;
3576 cr_assert(ret, "Data mismatch");
3577 ret = *((float *)source) == (float)TARGET_DATA_FP;
3578 cr_assert(ret, "Fetch data mismatch");
3579
3580 /* double */
3581 *(double *)&operand_dp = SOURCE_DATA_FP;
3582 *((double *)source) = FETCH_SOURCE_DATA;
3583 *((double *)target) = TARGET_DATA_FP;
3584 sz = fi_fetch_atomic(ep[0], NULL, 1, NULL,
3585 source, loc_mr[0], gni_addr[1],
3586 _REM_ADDR(fi, target, target),
3587 mr_key[1], FI_DOUBLE, FI_ATOMIC_READ, target);
3588 cr_assert_eq(sz, 0);
3589
3590 /* reset cqe */
3591 cqe.op_context = cqe.buf = (void *) -1;
3592 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3593 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3594 pthread_yield();
3595 }
3596
3597 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3598 dump_cq_error(send_cq[0], target, 0));
3599 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3600 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3601
3602 dbg_printf("got write context event!\n");
3603
3604 ret = *((double *)target) == (double)TARGET_DATA_FP;
3605 cr_assert(ret, "Data mismatch");
3606 ret = *((double *)source) == (double)TARGET_DATA_FP;
3607 cr_assert(ret, "Fetch data mismatch");
3608 }
3609
Test(rdm_atomic_basic,fetch_atomic_read)3610 Test(rdm_atomic_basic, fetch_atomic_read)
3611 {
3612 rdm_atomic_xfer_for_each_size(do_fetch_atomic_read, 1, 1);
3613 }
3614
Test(rdm_atomic_scalable,fetch_atomic_read)3615 Test(rdm_atomic_scalable, fetch_atomic_read)
3616 {
3617 rdm_atomic_xfer_for_each_size(do_fetch_atomic_read, 1, 1);
3618 }
3619
do_fetch_min_buf(void * s,void * t)3620 void do_fetch_min_buf(void *s, void *t)
3621 {
3622 int ret;
3623 ssize_t sz;
3624 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
3625 (void *) -1, UINT_MAX, UINT_MAX };
3626 uint64_t min;
3627 float min_fp;
3628 double min_dp;
3629 uint64_t operand = SOURCE_DATA;
3630 float operand_fp;
3631 double operand_dp;
3632 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
3633 uint64_t r_e[NUMEPS] = {0};
3634
3635 /* i64 */
3636 *((int64_t *)s) = FETCH_SOURCE_DATA;
3637 *((int64_t *)t) = TARGET_DATA;
3638 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3639 s, loc_mr[0], gni_addr[1],
3640 _REM_ADDR(fi, target, t), mr_key[1],
3641 FI_INT64, FI_MIN, t);
3642 if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
3643 cr_assert_eq(sz, -FI_EINVAL);
3644 } else {
3645 cr_assert_eq(sz, 0);
3646
3647 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3648 pthread_yield();
3649 }
3650
3651 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3652 dump_cq_error(send_cq[0], t, 0));
3653 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
3654
3655 r[0] = 1;
3656 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3657
3658 dbg_printf("got write context event!\n");
3659
3660 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
3661 SOURCE_DATA : TARGET_DATA;
3662 ret = *((int64_t *)t) == min;
3663 cr_assert(ret, "Data mismatch");
3664 ret = *((int64_t *)s) == TARGET_DATA;
3665 cr_assert(ret, "Fetch data mismatch");
3666 }
3667
3668 /* i32 */
3669 *((int64_t *)s) = FETCH_SOURCE_DATA;
3670 *((int64_t *)t) = TARGET_DATA;
3671 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
3672 s, loc_mr[0], gni_addr[1],
3673 _REM_ADDR(fi, target, t), mr_key[1],
3674 FI_INT32, FI_MIN, t);
3675 if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
3676 cr_assert_eq(sz, -FI_EINVAL);
3677 } else {
3678 cr_assert_eq(sz, 0);
3679
3680 /* reset cqe */
3681 cqe.op_context = cqe.buf = (void *) -1;
3682 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3683 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3684 pthread_yield();
3685 }
3686
3687 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3688 dump_cq_error(send_cq[0], t, 0));
3689 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
3690
3691 r[0] = 1;
3692 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3693
3694 dbg_printf("got write context event!\n");
3695
3696 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
3697 SOURCE_DATA : TARGET_DATA;
3698 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
3699 ret = *((int64_t *)t) == min;
3700 cr_assert(ret, "Data mismatch");
3701 min = (TARGET_DATA & U32_MASK) |
3702 (FETCH_SOURCE_DATA & (U32_MASK << 32));
3703 ret = *((int64_t *)s) == min;
3704 cr_assert(ret, "Fetch data mismatch");
3705 }
3706
3707 /* float */
3708 *((float *)&operand_fp) = SOURCE_DATA_FP;
3709 *((float *)s) = FETCH_SOURCE_DATA;
3710 *((float *)t) = TARGET_DATA_FP;
3711 sz = fi_fetch_atomic(ep[0], &operand_fp, 1, NULL,
3712 s, loc_mr[0], gni_addr[1],
3713 _REM_ADDR(fi, target, t), mr_key[1],
3714 FI_FLOAT, FI_MIN, t);
3715 if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
3716 cr_assert_eq(sz, -FI_EINVAL);
3717 } else {
3718 cr_assert_eq(sz, 0);
3719
3720 /* reset cqe */
3721 cqe.op_context = cqe.buf = (void *) -1;
3722 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3723 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3724 pthread_yield();
3725 }
3726
3727 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3728 dump_cq_error(send_cq[0], t, 0));
3729 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
3730
3731 r[0] = 1;
3732 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3733
3734 dbg_printf("got write context event!\n");
3735
3736 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
3737 SOURCE_DATA_FP : TARGET_DATA_FP;
3738 ret = *((float *)t) == min_fp;
3739 cr_assert(ret, "Data mismatch");
3740 ret = *((float *)s) == (float)TARGET_DATA_FP;
3741 cr_assert(ret, "Fetch data mismatch");
3742 }
3743
3744 /* double */
3745 *((double *)&operand_dp) = SOURCE_DATA_FP;
3746 *((double *)s) = SOURCE_DATA_FP;
3747 *((double *)t) = TARGET_DATA_FP;
3748 sz = fi_fetch_atomic(ep[0], &operand_dp, 1, NULL,
3749 s, loc_mr[0], gni_addr[1],
3750 _REM_ADDR(fi, target, t), mr_key[1],
3751 FI_DOUBLE, FI_MIN, t);
3752 if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
3753 cr_assert_eq(sz, -FI_EINVAL);
3754 } else {
3755 cr_assert_eq(sz, 0);
3756
3757 /* reset cqe */
3758 cqe.op_context = cqe.buf = (void *) -1;
3759 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3760 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3761 pthread_yield();
3762 }
3763
3764 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3765 dump_cq_error(send_cq[0], t, 0));
3766 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
3767
3768 r[0] = 1;
3769 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3770
3771 dbg_printf("got write context event!\n");
3772
3773 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
3774 SOURCE_DATA_FP : TARGET_DATA_FP;
3775 ret = *((double *)t) == min_dp;
3776 cr_assert(ret, "Data mismatch");
3777 ret = *((double *)s) == (double)TARGET_DATA_FP;
3778 cr_assert(ret, "Fetch data mismatch");
3779 }
3780 }
3781
__atomic_fetch_alignment(void)3782 static inline void __atomic_fetch_alignment(void)
3783 {
3784 int s_off, t_off;
3785
3786 for (s_off = 0; s_off < 7; s_off++) {
3787 for (t_off = 0; t_off < 7; t_off++) {
3788 do_fetch_min_buf(source + s_off, target + t_off);
3789 }
3790 }
3791 }
3792
Test(rdm_atomic_basic,atomic_fetch_alignment)3793 Test(rdm_atomic_basic, atomic_fetch_alignment)
3794 {
3795 __atomic_fetch_alignment();
3796 }
3797
Test(rdm_atomic_scalable,atomic_fetch_alignment)3798 Test(rdm_atomic_scalable, atomic_fetch_alignment)
3799 {
3800 __atomic_fetch_alignment();
3801 }
3802
__fatomicv(void)3803 static inline void __fatomicv(void)
3804 {
3805 int ret;
3806 ssize_t sz;
3807 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
3808 (void *) -1, UINT_MAX, UINT_MAX };
3809 uint64_t min;
3810 float min_fp;
3811 double min_dp;
3812 uint64_t operand = SOURCE_DATA;
3813 float operand_fp;
3814 double operand_dp;
3815 struct fi_ioc iov, r_iov;
3816 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
3817 uint64_t r_e[NUMEPS] = {0};
3818
3819 iov.count = 1;
3820 r_iov.count = 1;
3821
3822 /* i64 */
3823 *((int64_t *)source) = FETCH_SOURCE_DATA;
3824 *((int64_t *)target) = TARGET_DATA;
3825 iov.addr = &operand;
3826 r_iov.addr = source;
3827 sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
3828 &r_iov, (void **)loc_mr, 1,
3829 gni_addr[1],
3830 _REM_ADDR(fi, target, target), mr_key[1],
3831 FI_INT64, FI_MIN, target);
3832 cr_assert_eq(sz, 0);
3833
3834 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3835 pthread_yield();
3836 }
3837
3838 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3839 dump_cq_error(send_cq[0], target, 0));
3840 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3841
3842 r[0] = 1;
3843 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3844
3845 dbg_printf("got write context event!\n");
3846
3847 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
3848 SOURCE_DATA : TARGET_DATA;
3849 ret = *((int64_t *)target) == min;
3850 cr_assert(ret, "Data mismatch");
3851 ret = *((int64_t *)source) == TARGET_DATA;
3852 cr_assert(ret, "Fetch data mismatch");
3853
3854 /* i32 */
3855 *((int64_t *)source) = FETCH_SOURCE_DATA;
3856 *((int64_t *)target) = TARGET_DATA;
3857 iov.addr = &operand;
3858 r_iov.addr = source;
3859 sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
3860 &r_iov, (void **)loc_mr, 1,
3861 gni_addr[1],
3862 _REM_ADDR(fi, target, target), mr_key[1],
3863 FI_INT32, FI_MIN, target);
3864 cr_assert_eq(sz, 0);
3865
3866 /* reset cqe */
3867 cqe.op_context = cqe.buf = (void *) -1;
3868 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3869 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3870 pthread_yield();
3871 }
3872
3873 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3874 dump_cq_error(send_cq[0], NULL, 0));
3875 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3876 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3877
3878 dbg_printf("got write context event!\n");
3879
3880 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
3881 SOURCE_DATA : TARGET_DATA;
3882 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
3883 ret = *((int64_t *)target) == min;
3884 cr_assert(ret, "Data mismatch");
3885 min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
3886 ret = *((int64_t *)source) == min;
3887 cr_assert(ret, "Fetch data mismatch");
3888
3889 /* float */
3890 *((float *)&operand_fp) = SOURCE_DATA_FP;
3891 *((float *)source) = FETCH_SOURCE_DATA;
3892 *((float *)target) = TARGET_DATA_FP;
3893 iov.addr = &operand_fp;
3894 r_iov.addr = source;
3895 sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
3896 &r_iov, (void **)loc_mr, 1,
3897 gni_addr[1],
3898 _REM_ADDR(fi, target, target), mr_key[1],
3899 FI_FLOAT, FI_MIN, target);
3900 cr_assert_eq(sz, 0);
3901
3902 /* reset cqe */
3903 cqe.op_context = cqe.buf = (void *) -1;
3904 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3905 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3906 pthread_yield();
3907 }
3908
3909 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3910 dump_cq_error(send_cq[0], target, 0));
3911 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3912 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3913
3914 dbg_printf("got write context event!\n");
3915
3916 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
3917 SOURCE_DATA_FP : TARGET_DATA_FP;
3918 ret = *((float *)target) == min_fp;
3919 cr_assert(ret, "Data mismatch");
3920 ret = *((float *)source) == (float)TARGET_DATA_FP;
3921 cr_assert(ret, "Fetch data mismatch");
3922
3923 /* double */
3924 *((double *)&operand_dp) = SOURCE_DATA_FP;
3925 *((double *)source) = SOURCE_DATA_FP;
3926 *((double *)target) = TARGET_DATA_FP;
3927 iov.addr = &operand_dp;
3928 r_iov.addr = source;
3929 sz = fi_fetch_atomicv(ep[0], &iov, NULL, 1,
3930 &r_iov, (void **)loc_mr, 1,
3931 gni_addr[1],
3932 _REM_ADDR(fi, target, target), mr_key[1],
3933 FI_DOUBLE, FI_MIN, target);
3934 cr_assert_eq(sz, 0);
3935
3936 /* reset cqe */
3937 cqe.op_context = cqe.buf = (void *) -1;
3938 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
3939 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
3940 pthread_yield();
3941 }
3942
3943 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
3944 dump_cq_error(send_cq[0], target, 0));
3945 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
3946 rdm_atomic_check_cntrs(w, r, w_e, r_e);
3947
3948 dbg_printf("got write context event!\n");
3949
3950 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
3951 SOURCE_DATA_FP : TARGET_DATA_FP;
3952 ret = *((double *)target) == min_dp;
3953 cr_assert(ret, "Data mismatch");
3954 ret = *((double *)source) == (double)TARGET_DATA_FP;
3955 cr_assert(ret, "Fetch data mismatch");
3956 }
3957
Test(rdm_atomic_basic,fatomicv)3958 Test(rdm_atomic_basic, fatomicv)
3959 {
3960 __fatomicv();
3961 }
3962
Test(rdm_atomic_scalable,fatomicv)3963 Test(rdm_atomic_scalable, fatomicv)
3964 {
3965 __fatomicv();
3966 }
3967
__fatomicmsg(void)3968 static inline void __fatomicmsg(void)
3969 {
3970 int ret;
3971 ssize_t sz;
3972 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
3973 (void *) -1, UINT_MAX, UINT_MAX };
3974 uint64_t min;
3975 float min_fp;
3976 double min_dp;
3977 uint64_t operand = SOURCE_DATA;
3978 float operand_fp;
3979 double operand_dp;
3980 struct fi_msg_atomic msg;
3981 struct fi_ioc msg_iov, res_iov;
3982 struct fi_rma_ioc rma_iov;
3983 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
3984 uint64_t r_e[NUMEPS] = {0};
3985
3986 msg_iov.count = 1;
3987 msg.msg_iov = &msg_iov;
3988 msg.desc = (void **)loc_mr;
3989 msg.iov_count = 1;
3990 msg.addr = gni_addr[1];
3991 rma_iov.addr = _REM_ADDR(fi, target, target);
3992 rma_iov.count = 1;
3993 rma_iov.key = mr_key[1];
3994 msg.rma_iov = &rma_iov;
3995 msg.context = target;
3996 msg.op = FI_MIN;
3997
3998 res_iov.addr = source;
3999 res_iov.count = 1;
4000
4001 /* i64 */
4002 *((int64_t *)source) = FETCH_SOURCE_DATA;
4003 *((int64_t *)target) = TARGET_DATA;
4004 msg_iov.addr = &operand;
4005 msg.datatype = FI_INT64;
4006 sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
4007 cr_assert_eq(sz, 0);
4008
4009 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4010 pthread_yield();
4011 }
4012
4013 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4014 dump_cq_error(send_cq[0], NULL, 0));
4015 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4016
4017 r[0] = 1;
4018 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4019
4020 dbg_printf("got write context event!\n");
4021
4022 min = ((int64_t)SOURCE_DATA < (int64_t)TARGET_DATA) ?
4023 SOURCE_DATA : TARGET_DATA;
4024 ret = *((int64_t *)target) == min;
4025 cr_assert(ret, "Data mismatch");
4026 ret = *((int64_t *)source) == TARGET_DATA;
4027 cr_assert(ret, "Fetch data mismatch");
4028
4029 /* i32 */
4030 *((int64_t *)source) = FETCH_SOURCE_DATA;
4031 *((int64_t *)target) = TARGET_DATA;
4032 msg_iov.addr = &operand;
4033 msg.datatype = FI_INT32;
4034 sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
4035 cr_assert_eq(sz, 0);
4036
4037 /* reset cqe */
4038 cqe.op_context = cqe.buf = (void *) -1;
4039 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4040 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4041 pthread_yield();
4042 }
4043
4044 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4045 dump_cq_error(send_cq[0], NULL, 0));
4046 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4047 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4048
4049 dbg_printf("got write context event!\n");
4050
4051 min = ((int32_t)SOURCE_DATA < (int32_t)TARGET_DATA) ?
4052 SOURCE_DATA : TARGET_DATA;
4053 min = (min & U32_MASK) | (TARGET_DATA & (U32_MASK << 32));
4054 ret = *((int64_t *)target) == min;
4055 cr_assert(ret, "Data mismatch");
4056 min = (TARGET_DATA & U32_MASK) | (FETCH_SOURCE_DATA & (U32_MASK << 32));
4057 ret = *((int64_t *)source) == min;
4058 cr_assert(ret, "Fetch data mismatch");
4059
4060 /* float */
4061 *((float *)&operand_fp) = SOURCE_DATA_FP;
4062 *((float *)source) = FETCH_SOURCE_DATA;
4063 *((float *)target) = TARGET_DATA_FP;
4064 msg_iov.addr = &operand_fp;
4065 msg.datatype = FI_FLOAT;
4066 sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
4067 cr_assert_eq(sz, 0);
4068
4069 /* reset cqe */
4070 cqe.op_context = cqe.buf = (void *) -1;
4071 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4072 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4073 pthread_yield();
4074 }
4075
4076 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4077 dump_cq_error(send_cq[0], NULL, 0));
4078 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4079 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4080
4081 dbg_printf("got write context event!\n");
4082
4083 min_fp = (float)SOURCE_DATA_FP < (float)TARGET_DATA_FP ?
4084 SOURCE_DATA_FP : TARGET_DATA_FP;
4085 ret = *((float *)target) == min_fp;
4086 cr_assert(ret, "Data mismatch");
4087 ret = *((float *)source) == (float)TARGET_DATA_FP;
4088 cr_assert(ret, "Fetch data mismatch");
4089
4090 /* double */
4091 *((double *)&operand_dp) = SOURCE_DATA_FP;
4092 *((double *)source) = SOURCE_DATA_FP;
4093 *((double *)target) = TARGET_DATA_FP;
4094 msg_iov.addr = &operand_dp;
4095 msg.datatype = FI_DOUBLE;
4096 sz = fi_fetch_atomicmsg(ep[0], &msg, &res_iov, (void **)loc_mr, 1, 0);
4097 cr_assert_eq(sz, 0);
4098
4099 /* reset cqe */
4100 cqe.op_context = cqe.buf = (void *) -1;
4101 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4102 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4103 pthread_yield();
4104 }
4105
4106 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4107 dump_cq_error(send_cq[0], NULL, 0));
4108 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4109 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4110
4111 dbg_printf("got write context event!\n");
4112
4113 min_dp = (double)SOURCE_DATA_FP < (double)TARGET_DATA_FP ?
4114 SOURCE_DATA_FP : TARGET_DATA_FP;
4115 ret = *((double *)target) == min_dp;
4116 cr_assert(ret, "Data mismatch");
4117 ret = *((double *)source) == (double)TARGET_DATA_FP;
4118 cr_assert(ret, "Fetch data mismatch");
4119 }
4120
Test(rdm_atomic_basic,fatomicmsg)4121 Test(rdm_atomic_basic, fatomicmsg)
4122 {
4123 __fatomicmsg();
4124 }
4125
Test(rdm_atomic_scalable,fatomicmsg)4126 Test(rdm_atomic_scalable, fatomicmsg)
4127 {
4128 __fatomicmsg();
4129 }
4130
4131 /******************************************************************************
4132 *
4133 * Compare atomics
4134 *
4135 *****************************************************************************/
4136
4137 int supported_compare_atomic_ops[FI_ATOMIC_OP_LAST][FI_DATATYPE_LAST] = {
4138 [FI_MIN] = NO_DATATYPES_SUPPORTED,
4139 [FI_MAX] = NO_DATATYPES_SUPPORTED,
4140 [FI_SUM] = NO_DATATYPES_SUPPORTED,
4141 [FI_PROD] = NO_DATATYPES_SUPPORTED,
4142 [FI_LOR] = NO_DATATYPES_SUPPORTED,
4143 [FI_LAND] = NO_DATATYPES_SUPPORTED,
4144 [FI_BOR] = NO_DATATYPES_SUPPORTED,
4145 [FI_BAND] = NO_DATATYPES_SUPPORTED,
4146 [FI_LXOR] = NO_DATATYPES_SUPPORTED,
4147 [FI_BXOR] = NO_DATATYPES_SUPPORTED,
4148 [FI_ATOMIC_READ] = NO_DATATYPES_SUPPORTED,
4149 [FI_ATOMIC_WRITE] = NO_DATATYPES_SUPPORTED,
4150 [FI_CSWAP] = ALL_GNI_DATATYPES_SUPPORTED,
4151 [FI_CSWAP_NE] = NO_DATATYPES_SUPPORTED,
4152 [FI_CSWAP_LE] = NO_DATATYPES_SUPPORTED,
4153 [FI_CSWAP_LT] = NO_DATATYPES_SUPPORTED,
4154 [FI_CSWAP_GE] = NO_DATATYPES_SUPPORTED,
4155 [FI_CSWAP_GT] = NO_DATATYPES_SUPPORTED,
4156 [FI_MSWAP] = ALL_GNI_DATATYPES_SUPPORTED,
4157 };
4158
do_invalid_compare_atomic(enum fi_datatype dt,enum fi_op op)4159 void do_invalid_compare_atomic(enum fi_datatype dt, enum fi_op op)
4160 {
4161 ssize_t sz;
4162 size_t count;
4163 uint64_t operand, op2;
4164
4165 if (!supported_compare_atomic_ops[op][dt]) {
4166 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4167 source, loc_mr,
4168 gni_addr[1],
4169 _REM_ADDR(fi, target, target),
4170 mr_key[1],
4171 dt, op, target);
4172 cr_assert(sz == -FI_EOPNOTSUPP);
4173
4174 sz = fi_compare_atomicvalid(ep[0], dt, op, &count);
4175 cr_assert(sz == -FI_EOPNOTSUPP, "fi_atomicvalid() succeeded\n");
4176 } else {
4177 sz = fi_compare_atomicvalid(ep[0], dt, op, &count);
4178 cr_assert(!sz, "fi_atomicvalid() failed\n");
4179 cr_assert(count == 1, "fi_atomicvalid(): bad count \n");
4180 }
4181 }
4182
Test(rdm_atomic_default,invalid_compare_atomic)4183 Test(rdm_atomic_default, invalid_compare_atomic)
4184 {
4185 int i, j;
4186
4187 for(i = 0; i < FI_ATOMIC_OP_LAST; i++) {
4188 for(j = 0; j < FI_DATATYPE_LAST; j++) {
4189 do_invalid_compare_atomic(j, i);
4190 }
4191 }
4192 }
4193
do_cswap(int len)4194 void do_cswap(int len)
4195 {
4196 int ret;
4197 ssize_t sz;
4198 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
4199 (void *) -1, UINT_MAX, UINT_MAX };
4200 uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
4201 float operand_fp, op2_fp;
4202 double operand_dp, op2_dp;
4203 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
4204 uint64_t r_e[NUMEPS] = {0};
4205
4206 /* u64 */
4207 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4208 *((uint64_t *)target) = TARGET_DATA;
4209 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4210 source, loc_mr[0], gni_addr[1],
4211 _REM_ADDR(fi, target, target),
4212 mr_key[1], FI_UINT64, FI_CSWAP, target);
4213 cr_assert_eq(sz, 0);
4214
4215 /* reset cqe */
4216 cqe.op_context = cqe.buf = (void *) -1;
4217 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4218 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4219 pthread_yield();
4220 }
4221
4222 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4223 dump_cq_error(send_cq[0], target, 0));
4224 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4225
4226 r[0] = 1;
4227 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4228
4229 dbg_printf("got write context event!\n");
4230
4231 ret = *((uint64_t *)target) == SOURCE_DATA;
4232 cr_assert(ret, "Data mismatch");
4233 ret = *((uint64_t *)source) == TARGET_DATA;
4234 cr_assert(ret, "Fetch data mismatch");
4235
4236 /* U32 */
4237 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4238 *((uint64_t *)target) = TARGET_DATA;
4239 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4240 source, loc_mr[0], gni_addr[1],
4241 _REM_ADDR(fi, target, target),
4242 mr_key[1], FI_UINT32, FI_CSWAP, target);
4243 cr_assert_eq(sz, 0);
4244
4245 /* reset cqe */
4246 cqe.op_context = cqe.buf = (void *) -1;
4247 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4248 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4249 pthread_yield();
4250 }
4251
4252 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4253 dump_cq_error(send_cq[0], target, 0));
4254 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4255 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4256
4257 dbg_printf("got write context event!\n");
4258
4259 ret = *((uint64_t *)target) ==
4260 (uint64_t)((SOURCE_DATA & U32_MASK) |
4261 (TARGET_DATA & (U32_MASK << 32)));
4262 cr_assert(ret, "Data mismatch");
4263 ret = *((uint64_t *)source) ==
4264 (uint64_t)((TARGET_DATA & U32_MASK) |
4265 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
4266 cr_assert(ret, "Fetch data mismatch");
4267
4268 /* i64 */
4269 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4270 *((uint64_t *)target) = TARGET_DATA;
4271 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4272 source, loc_mr[0], gni_addr[1],
4273 _REM_ADDR(fi, target, target),
4274 mr_key[1], FI_INT64, FI_CSWAP, target);
4275 cr_assert_eq(sz, 0);
4276
4277 /* reset cqe */
4278 cqe.op_context = cqe.buf = (void *) -1;
4279 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4280 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4281 pthread_yield();
4282 }
4283
4284 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4285 dump_cq_error(send_cq[0], target, 0));
4286 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4287 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4288
4289 dbg_printf("got write context event!\n");
4290
4291 ret = *((uint64_t *)target) == SOURCE_DATA;
4292 cr_assert(ret, "Data mismatch");
4293 ret = *((uint64_t *)source) == TARGET_DATA;
4294 cr_assert(ret, "Fetch data mismatch");
4295
4296 /* i32 */
4297 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4298 *((uint64_t *)target) = TARGET_DATA;
4299 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4300 source, loc_mr[0], gni_addr[1],
4301 _REM_ADDR(fi, target, target),
4302 mr_key[1], FI_INT32, FI_CSWAP, target);
4303 cr_assert_eq(sz, 0);
4304
4305 /* reset cqe */
4306 cqe.op_context = cqe.buf = (void *) -1;
4307 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4308 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4309 pthread_yield();
4310 }
4311
4312 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4313 dump_cq_error(send_cq[0], target, 0));
4314 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4315 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4316
4317 dbg_printf("got write context event!\n");
4318
4319 ret = *((uint64_t *)target) ==
4320 (uint64_t)((SOURCE_DATA & U32_MASK) |
4321 (TARGET_DATA & (U32_MASK << 32)));
4322 cr_assert(ret, "Data mismatch");
4323 ret = *((uint64_t *)source) ==
4324 (uint64_t)((TARGET_DATA & U32_MASK) |
4325 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
4326 cr_assert(ret, "Fetch data mismatch");
4327
4328 /* float */
4329 *((float *)&operand_fp) = SOURCE_DATA_FP;
4330 *((float *)&op2_fp) = TARGET_DATA_FP;
4331 *((float *)source) = FETCH_SOURCE_DATA;
4332 *((float *)target) = TARGET_DATA_FP;
4333 sz = fi_compare_atomic(ep[0], &operand_fp, 1, NULL, &op2_fp, NULL,
4334 source, loc_mr[0], gni_addr[1],
4335 _REM_ADDR(fi, target, target),
4336 mr_key[1], FI_FLOAT, FI_CSWAP, target);
4337 cr_assert_eq(sz, 0);
4338
4339 /* reset cqe */
4340 cqe.op_context = cqe.buf = (void *) -1;
4341 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4342 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4343 pthread_yield();
4344 }
4345
4346 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4347 dump_cq_error(send_cq[0], target, 0));
4348 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4349 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4350
4351 dbg_printf("got write context event!\n");
4352
4353 ret = *((float *)target) == (float)SOURCE_DATA_FP;
4354 cr_assert(ret, "Data mismatch");
4355 ret = *((float *)source) == (float)TARGET_DATA_FP;
4356 cr_assert(ret, "Fetch data mismatch");
4357
4358 /* double */
4359 *((double *)&operand_dp) = SOURCE_DATA_FP;
4360 *((double *)&op2_dp) = TARGET_DATA_FP;
4361 *((double *)source) = FETCH_SOURCE_DATA;
4362 *((double *)target) = TARGET_DATA_FP;
4363 sz = fi_compare_atomic(ep[0], &operand_dp, 1, NULL, &op2_dp, NULL,
4364 source, loc_mr[0], gni_addr[1],
4365 _REM_ADDR(fi, target, target),
4366 mr_key[1], FI_DOUBLE, FI_CSWAP, target);
4367 cr_assert_eq(sz, 0);
4368
4369 /* reset cqe */
4370 cqe.op_context = cqe.buf = (void *) -1;
4371 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4372 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4373 pthread_yield();
4374 }
4375
4376 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4377 dump_cq_error(send_cq[0], target, 0));
4378 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4379 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4380
4381 dbg_printf("got write context event!\n");
4382
4383 ret = *((double *)target) == (double)SOURCE_DATA_FP;
4384 cr_assert(ret, "Data mismatch");
4385 ret = *((double *)source) == (double)TARGET_DATA_FP;
4386 cr_assert(ret, "Fetch data mismatch");
4387 }
4388
Test(rdm_atomic_basic,cswap)4389 Test(rdm_atomic_basic, cswap)
4390 {
4391 rdm_atomic_xfer_for_each_size(do_cswap, 1, 1);
4392 }
4393
Test(rdm_atomic_scalable,cswap)4394 Test(rdm_atomic_scalable, cswap)
4395 {
4396 rdm_atomic_xfer_for_each_size(do_cswap, 1, 1);
4397 }
4398
do_mswap(int len)4399 void do_mswap(int len)
4400 {
4401 int ret;
4402 ssize_t sz;
4403 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
4404 (void *) -1, UINT_MAX, UINT_MAX };
4405 uint64_t res;
4406 uint64_t operand = SOURCE_DATA, op2 = DATA_MASK;
4407 float operand_fp, op2_fp;
4408 double operand_dp, op2_dp;
4409 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
4410 uint64_t r_e[NUMEPS] = {0};
4411
4412 /* u64 */
4413 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4414 *((uint64_t *)target) = TARGET_DATA;
4415 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4416 source, loc_mr[0], gni_addr[1],
4417 _REM_ADDR(fi, target, target),
4418 mr_key[1], FI_UINT64, FI_MSWAP, target);
4419 cr_assert_eq(sz, 0);
4420
4421 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4422 pthread_yield();
4423 }
4424
4425 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4426 dump_cq_error(send_cq[0], target, 0));
4427 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4428
4429 r[0] = 1;
4430 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4431
4432 dbg_printf("got write context event!\n");
4433
4434 res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
4435 ret = *((uint64_t *)target) == res;
4436 cr_assert(ret, "Data mismatch");
4437 ret = *((uint64_t *)source) == TARGET_DATA;
4438 cr_assert(ret, "Fetch data mismatch");
4439
4440 /* U32 */
4441 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4442 *((uint64_t *)target) = TARGET_DATA;
4443 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4444 source, loc_mr[0], gni_addr[1],
4445 _REM_ADDR(fi, target, target),
4446 mr_key[1], FI_UINT32, FI_MSWAP, target);
4447 cr_assert_eq(sz, 0);
4448
4449 /* reset cqe */
4450 cqe.op_context = cqe.buf = (void *) -1;
4451 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4452 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4453 pthread_yield();
4454 }
4455
4456 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4457 dump_cq_error(send_cq[0], target, 0));
4458 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4459 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4460
4461 dbg_printf("got write context event!\n");
4462
4463 res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
4464 ret = *((uint64_t *)target) ==
4465 (uint64_t)((res & U32_MASK) |
4466 (TARGET_DATA & (U32_MASK << 32)));
4467 cr_assert(ret, "Data mismatch");
4468 ret = *((uint64_t *)source) ==
4469 (uint64_t)((TARGET_DATA & U32_MASK) |
4470 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
4471 cr_assert(ret, "Fetch data mismatch");
4472
4473 /* i64 */
4474 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4475 *((uint64_t *)target) = TARGET_DATA;
4476 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4477 source, loc_mr[0], gni_addr[1],
4478 _REM_ADDR(fi, target, target),
4479 mr_key[1], FI_INT64, FI_MSWAP, target);
4480 cr_assert_eq(sz, 0);
4481
4482 /* reset cqe */
4483 cqe.op_context = cqe.buf = (void *) -1;
4484 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4485 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4486 pthread_yield();
4487 }
4488
4489 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4490 dump_cq_error(send_cq[0], target, 0));
4491 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4492 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4493
4494 dbg_printf("got write context event!\n");
4495
4496 res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
4497 ret = *((uint64_t *)target) == res;
4498 cr_assert(ret, "Data mismatch");
4499 ret = *((uint64_t *)source) == TARGET_DATA;
4500 cr_assert(ret, "Fetch data mismatch");
4501
4502 /* i32 */
4503 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4504 *((uint64_t *)target) = TARGET_DATA;
4505 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4506 source, loc_mr[0], gni_addr[1],
4507 _REM_ADDR(fi, target, target),
4508 mr_key[1], FI_INT32, FI_MSWAP, target);
4509 cr_assert_eq(sz, 0);
4510
4511 /* reset cqe */
4512 cqe.op_context = cqe.buf = (void *) -1;
4513 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4514 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4515 pthread_yield();
4516 }
4517
4518 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4519 dump_cq_error(send_cq[0], target, 0));
4520 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4521 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4522
4523 dbg_printf("got write context event!\n");
4524
4525 res = (SOURCE_DATA & DATA_MASK) | (TARGET_DATA & ~DATA_MASK);
4526 ret = *((uint64_t *)target) ==
4527 (uint64_t)((res & U32_MASK) |
4528 (TARGET_DATA & (U32_MASK << 32)));
4529 cr_assert(ret, "Data mismatch");
4530 ret = *((uint64_t *)source) ==
4531 (uint64_t)((TARGET_DATA & U32_MASK) |
4532 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
4533 cr_assert(ret, "Fetch data mismatch");
4534
4535 /* float */
4536 *((float *)&operand_fp) = SOURCE_DATA_FP;
4537 *((float *)&op2_fp) = TARGET_DATA_FP;
4538 *((float *)source) = FETCH_SOURCE_DATA;
4539 *((float *)target) = TARGET_DATA_FP;
4540 sz = fi_compare_atomic(ep[0], &operand_fp, 1, NULL, &op2_fp, NULL,
4541 source, loc_mr[0], gni_addr[1],
4542 _REM_ADDR(fi, target, target),
4543 mr_key[1], FI_FLOAT, FI_MSWAP, target);
4544 cr_assert_eq(sz, 0);
4545
4546 /* reset cqe */
4547 cqe.op_context = cqe.buf = (void *) -1;
4548 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4549 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4550 pthread_yield();
4551 }
4552
4553 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4554 dump_cq_error(send_cq[0], target, 0));
4555 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4556 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4557
4558 dbg_printf("got write context event!\n");
4559
4560 ret = *((float *)target) == (float)SOURCE_DATA_FP;
4561 cr_assert(ret, "Data mismatch");
4562 ret = *((float *)source) == (float)TARGET_DATA_FP;
4563 cr_assert(ret, "Fetch data mismatch");
4564
4565 /* double */
4566 *((double *)&operand_dp) = SOURCE_DATA_FP;
4567 *((double *)&op2_dp) = TARGET_DATA_FP;
4568 *((double *)source) = FETCH_SOURCE_DATA;
4569 *((double *)target) = TARGET_DATA_FP;
4570 sz = fi_compare_atomic(ep[0], &operand_dp, 1, NULL, &op2_dp, NULL,
4571 source, loc_mr[0], gni_addr[1],
4572 _REM_ADDR(fi, target, target),
4573 mr_key[1], FI_DOUBLE, FI_MSWAP, target);
4574 cr_assert_eq(sz, 0);
4575
4576 /* reset cqe */
4577 cqe.op_context = cqe.buf = (void *) -1;
4578 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4579 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4580 pthread_yield();
4581 }
4582
4583 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4584 dump_cq_error(send_cq[0], target, 0));
4585 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4586 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4587
4588 dbg_printf("got write context event!\n");
4589
4590 ret = *((double *)target) == (double)SOURCE_DATA_FP;
4591 cr_assert(ret, "Data mismatch");
4592 ret = *((double *)source) == (double)TARGET_DATA_FP;
4593 cr_assert(ret, "Fetch data mismatch");
4594 }
4595
Test(rdm_atomic_basic,mswap)4596 Test(rdm_atomic_basic, mswap)
4597 {
4598 rdm_atomic_xfer_for_each_size(do_mswap, 1, 1);
4599 }
4600
Test(rdm_atomic_scalable,mswap)4601 Test(rdm_atomic_scalable, mswap)
4602 {
4603 rdm_atomic_xfer_for_each_size(do_mswap, 1, 1);
4604 }
4605
do_cswap_buf(void * s,void * t)4606 void do_cswap_buf(void *s, void *t)
4607 {
4608 int ret;
4609 ssize_t sz;
4610 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
4611 (void *) -1, UINT_MAX, UINT_MAX };
4612 uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
4613 float operand_fp, op2_fp;
4614 double operand_dp, op2_dp;
4615 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
4616 uint64_t r_e[NUMEPS] = {0};
4617
4618 /* u64 */
4619 *((uint64_t *)s) = FETCH_SOURCE_DATA;
4620 *((uint64_t *)t) = TARGET_DATA;
4621 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4622 s, loc_mr[0], gni_addr[1],
4623 _REM_ADDR(fi, target, t),
4624 mr_key[1], FI_UINT64, FI_CSWAP, t);
4625 if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
4626 cr_assert_eq(sz, -FI_EINVAL);
4627 } else {
4628 cr_assert_eq(sz, 0);
4629
4630 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4631 pthread_yield();
4632 }
4633
4634 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4635 dump_cq_error(send_cq[0], t, 0));
4636 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
4637
4638 r[0] = 1;
4639 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4640
4641 dbg_printf("got write context event!\n");
4642
4643 ret = *((uint64_t *)t) == SOURCE_DATA;
4644 cr_assert(ret, "Data mismatch");
4645 ret = *((uint64_t *)s) == TARGET_DATA;
4646 cr_assert(ret, "Fetch data mismatch");
4647 }
4648
4649 /* U32 */
4650 *((uint64_t *)s) = FETCH_SOURCE_DATA;
4651 *((uint64_t *)t) = TARGET_DATA;
4652 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4653 s, loc_mr[0], gni_addr[1],
4654 _REM_ADDR(fi, target, t),
4655 mr_key[1], FI_UINT32, FI_CSWAP, t);
4656 if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
4657 cr_assert_eq(sz, -FI_EINVAL);
4658 } else {
4659 cr_assert_eq(sz, 0);
4660
4661 /* reset cqe */
4662 cqe.op_context = cqe.buf = (void *) -1;
4663 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4664 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4665 pthread_yield();
4666 }
4667
4668 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4669 dump_cq_error(send_cq[0], t, 0));
4670 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
4671
4672 r[0] = 1;
4673 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4674
4675 dbg_printf("got write context event!\n");
4676
4677 ret = *((uint64_t *)t) ==
4678 (uint64_t)((SOURCE_DATA & U32_MASK) |
4679 (TARGET_DATA & (U32_MASK << 32)));
4680 cr_assert(ret, "Data mismatch");
4681 ret = *((uint64_t *)s) ==
4682 (uint64_t)((TARGET_DATA & U32_MASK) |
4683 (FETCH_SOURCE_DATA &
4684 (U32_MASK << 32)));
4685 cr_assert(ret, "Fetch data mismatch");
4686 }
4687
4688 /* i64 */
4689 *((uint64_t *)s) = FETCH_SOURCE_DATA;
4690 *((uint64_t *)t) = TARGET_DATA;
4691 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4692 s, loc_mr[0], gni_addr[1],
4693 _REM_ADDR(fi, target, t),
4694 mr_key[1], FI_INT64, FI_CSWAP, t);
4695 if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
4696 cr_assert_eq(sz, -FI_EINVAL);
4697 } else {
4698 cr_assert_eq(sz, 0);
4699
4700 /* reset cqe */
4701 cqe.op_context = cqe.buf = (void *) -1;
4702 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4703 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4704 pthread_yield();
4705 }
4706
4707 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4708 dump_cq_error(send_cq[0], t, 0));
4709 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
4710
4711 r[0] = 1;
4712 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4713
4714 dbg_printf("got write context event!\n");
4715
4716 ret = *((uint64_t *)t) == SOURCE_DATA;
4717 cr_assert(ret, "Data mismatch");
4718 ret = *((uint64_t *)s) == TARGET_DATA;
4719 cr_assert(ret, "Fetch data mismatch");
4720 }
4721
4722 /* i32 */
4723 *((uint64_t *)s) = FETCH_SOURCE_DATA;
4724 *((uint64_t *)t) = TARGET_DATA;
4725 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
4726 s, loc_mr[0], gni_addr[1],
4727 _REM_ADDR(fi, target, t),
4728 mr_key[1], FI_INT32, FI_CSWAP, t);
4729 if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
4730 cr_assert_eq(sz, -FI_EINVAL);
4731 } else {
4732 cr_assert_eq(sz, 0);
4733
4734 /* reset cqe */
4735 cqe.op_context = cqe.buf = (void *) -1;
4736 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4737 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4738 pthread_yield();
4739 }
4740
4741 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4742 dump_cq_error(send_cq[0], t, 0));
4743 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
4744
4745 r[0] = 1;
4746 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4747
4748 dbg_printf("got write context event!\n");
4749
4750 ret = *((uint64_t *)t) ==
4751 (uint64_t)((SOURCE_DATA & U32_MASK) |
4752 (TARGET_DATA & (U32_MASK << 32)));
4753 cr_assert(ret, "Data mismatch");
4754 ret = *((uint64_t *)s) ==
4755 (uint64_t)((TARGET_DATA & U32_MASK) |
4756 (FETCH_SOURCE_DATA &
4757 (U32_MASK << 32)));
4758 cr_assert(ret, "Fetch data mismatch");
4759 }
4760
4761 /* float */
4762 *((float *)&operand_fp) = SOURCE_DATA_FP;
4763 *((float *)&op2_fp) = TARGET_DATA_FP;
4764 *((float *)s) = FETCH_SOURCE_DATA;
4765 *((float *)t) = TARGET_DATA_FP;
4766 sz = fi_compare_atomic(ep[0], &operand_fp, 1, NULL, &op2_fp, NULL,
4767 s, loc_mr[0], gni_addr[1],
4768 _REM_ADDR(fi, target, t),
4769 mr_key[1], FI_FLOAT, FI_CSWAP, t);
4770 if ((uint64_t)s & 0x3 || (uint64_t)t & 0x3) {
4771 cr_assert_eq(sz, -FI_EINVAL);
4772 } else {
4773 cr_assert_eq(sz, 0);
4774
4775 /* reset cqe */
4776 cqe.op_context = cqe.buf = (void *) -1;
4777 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4778 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4779 pthread_yield();
4780 }
4781
4782 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4783 dump_cq_error(send_cq[0], t, 0));
4784 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
4785
4786 r[0] = 1;
4787 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4788
4789 dbg_printf("got write context event!\n");
4790
4791 ret = *((float *)t) == (float)SOURCE_DATA_FP;
4792 cr_assert(ret, "Data mismatch");
4793 ret = *((float *)s) == (float)TARGET_DATA_FP;
4794 cr_assert(ret, "Fetch data mismatch");
4795 }
4796
4797 /* double */
4798 *((double *)&operand_dp) = SOURCE_DATA_FP;
4799 *((double *)&op2_dp) = TARGET_DATA_FP;
4800 *((double *)s) = FETCH_SOURCE_DATA;
4801 *((double *)t) = TARGET_DATA_FP;
4802 sz = fi_compare_atomic(ep[0], &operand_dp, 1, NULL, &op2_dp, NULL,
4803 s, loc_mr[0], gni_addr[1],
4804 _REM_ADDR(fi, target, t),
4805 mr_key[1], FI_DOUBLE, FI_CSWAP, t);
4806 if ((uint64_t)s & 0x7 || (uint64_t)t & 0x7) {
4807 cr_assert_eq(sz, -FI_EINVAL);
4808 } else {
4809 cr_assert_eq(sz, 0);
4810
4811 /* reset cqe */
4812 cqe.op_context = cqe.buf = (void *) -1;
4813 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4814 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4815 pthread_yield();
4816 }
4817
4818 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4819 dump_cq_error(send_cq[0], t, 0));
4820 rdm_atomic_check_tcqe(&cqe, t, FI_ATOMIC | FI_READ, 0);
4821
4822 r[0] = 1;
4823 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4824
4825 dbg_printf("got write context event!\n");
4826
4827 ret = *((double *)t) == (double)SOURCE_DATA_FP;
4828 cr_assert(ret, "Data mismatch");
4829 ret = *((double *)s) == (double)TARGET_DATA_FP;
4830 cr_assert(ret, "Fetch data mismatch");
4831 }
4832 }
4833
__atomic_compare_alignment(void)4834 static inline void __atomic_compare_alignment(void)
4835 {
4836 int s_off, t_off;
4837
4838 for (s_off = 0; s_off < 7; s_off++) {
4839 for (t_off = 0; t_off < 7; t_off++) {
4840 do_cswap_buf(source + s_off, target + t_off);
4841 }
4842 }
4843 }
4844
Test(rdm_atomic_basic,atomic_compare_alignment)4845 Test(rdm_atomic_basic, atomic_compare_alignment)
4846 {
4847 __atomic_compare_alignment();
4848 }
4849
Test(rdm_atomic_scalable,atomic_compare_alignment)4850 Test(rdm_atomic_scalable, atomic_compare_alignment)
4851 {
4852 __atomic_compare_alignment();
4853 }
4854
__catomicv(void)4855 static inline void __catomicv(void)
4856 {
4857 int ret;
4858 ssize_t sz;
4859 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
4860 (void *) -1, UINT_MAX, UINT_MAX };
4861 uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
4862 float operand_fp, op2_fp;
4863 double operand_dp, op2_dp;
4864 struct fi_ioc iov, r_iov, c_iov;
4865 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
4866 uint64_t r_e[NUMEPS] = {0};
4867
4868 iov.count = 1;
4869 r_iov.count = 1;
4870 c_iov.count = 1;
4871
4872 /* u64 */
4873 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4874 *((uint64_t *)target) = TARGET_DATA;
4875 iov.addr = &operand;
4876 r_iov.addr = source;
4877 c_iov.addr = &op2;
4878 sz = fi_compare_atomicv(ep[0],
4879 &iov, NULL, 1,
4880 &c_iov, NULL, 1,
4881 &r_iov, (void **)loc_mr, 1,
4882 gni_addr[1],
4883 _REM_ADDR(fi, target, target), mr_key[1],
4884 FI_UINT64, FI_CSWAP, target);
4885 cr_assert_eq(sz, 0);
4886
4887 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4888 pthread_yield();
4889 }
4890
4891 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4892 dump_cq_error(send_cq[0], target, 0));
4893 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4894
4895 r[0] = 1;
4896 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4897
4898 dbg_printf("got write context event!\n");
4899
4900 ret = *((uint64_t *)target) == SOURCE_DATA;
4901 cr_assert(ret, "Data mismatch");
4902 ret = *((uint64_t *)source) == TARGET_DATA;
4903 cr_assert(ret, "Fetch data mismatch");
4904
4905 /* U32 */
4906 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4907 *((uint64_t *)target) = TARGET_DATA;
4908 iov.addr = &operand;
4909 r_iov.addr = source;
4910 c_iov.addr = &op2;
4911 sz = fi_compare_atomicv(ep[0],
4912 &iov, NULL, 1,
4913 &c_iov, NULL, 1,
4914 &r_iov, (void **)loc_mr, 1,
4915 gni_addr[1],
4916 _REM_ADDR(fi, target, target), mr_key[1],
4917 FI_UINT32, FI_CSWAP, target);
4918 cr_assert_eq(sz, 0);
4919
4920 /* reset cqe */
4921 cqe.op_context = cqe.buf = (void *) -1;
4922 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4923 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4924 pthread_yield();
4925 }
4926
4927 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4928 dump_cq_error(send_cq[0], target, 0));
4929 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4930 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4931
4932 dbg_printf("got write context event!\n");
4933
4934 ret = *((uint64_t *)target) ==
4935 (uint64_t)((SOURCE_DATA & U32_MASK) |
4936 (TARGET_DATA & (U32_MASK << 32)));
4937 cr_assert(ret, "Data mismatch");
4938 ret = *((uint64_t *)source) ==
4939 (uint64_t)((TARGET_DATA & U32_MASK) |
4940 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
4941 cr_assert(ret, "Fetch data mismatch");
4942
4943 /* i64 */
4944 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4945 *((uint64_t *)target) = TARGET_DATA;
4946 iov.addr = &operand;
4947 r_iov.addr = source;
4948 c_iov.addr = &op2;
4949 sz = fi_compare_atomicv(ep[0],
4950 &iov, NULL, 1,
4951 &c_iov, NULL, 1,
4952 &r_iov, (void **)loc_mr, 1,
4953 gni_addr[1],
4954 _REM_ADDR(fi, target, target), mr_key[1],
4955 FI_INT64, FI_CSWAP, target);
4956 cr_assert_eq(sz, 0);
4957
4958 /* reset cqe */
4959 cqe.op_context = cqe.buf = (void *) -1;
4960 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4961 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4962 pthread_yield();
4963 }
4964
4965 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
4966 dump_cq_error(send_cq[0], target, 0));
4967 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
4968 rdm_atomic_check_cntrs(w, r, w_e, r_e);
4969
4970 dbg_printf("got write context event!\n");
4971
4972 ret = *((uint64_t *)target) == SOURCE_DATA;
4973 cr_assert(ret, "Data mismatch");
4974 ret = *((uint64_t *)source) == TARGET_DATA;
4975 cr_assert(ret, "Fetch data mismatch");
4976
4977 /* i32 */
4978 *((uint64_t *)source) = FETCH_SOURCE_DATA;
4979 *((uint64_t *)target) = TARGET_DATA;
4980 iov.addr = &operand;
4981 r_iov.addr = source;
4982 c_iov.addr = &op2;
4983 sz = fi_compare_atomicv(ep[0],
4984 &iov, NULL, 1,
4985 &c_iov, NULL, 1,
4986 &r_iov, (void **)loc_mr, 1,
4987 gni_addr[1],
4988 _REM_ADDR(fi, target, target), mr_key[1],
4989 FI_INT32, FI_CSWAP, target);
4990 cr_assert_eq(sz, 0);
4991
4992 /* reset cqe */
4993 cqe.op_context = cqe.buf = (void *) -1;
4994 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
4995 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
4996 pthread_yield();
4997 }
4998
4999 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5000 dump_cq_error(send_cq[0], target, 0));
5001 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5002 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5003
5004 dbg_printf("got write context event!\n");
5005
5006 ret = *((uint64_t *)target) ==
5007 (uint64_t)((SOURCE_DATA & U32_MASK) |
5008 (TARGET_DATA & (U32_MASK << 32)));
5009 cr_assert(ret, "Data mismatch");
5010 ret = *((uint64_t *)source) ==
5011 (uint64_t)((TARGET_DATA & U32_MASK) |
5012 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
5013 cr_assert(ret, "Fetch data mismatch");
5014
5015 /* float */
5016 *((float *)&operand_fp) = SOURCE_DATA_FP;
5017 *((float *)&op2_fp) = TARGET_DATA_FP;
5018 *((float *)source) = FETCH_SOURCE_DATA;
5019 *((float *)target) = TARGET_DATA_FP;
5020 iov.addr = &operand_fp;
5021 r_iov.addr = source;
5022 c_iov.addr = &op2_fp;
5023 sz = fi_compare_atomicv(ep[0],
5024 &iov, NULL, 1,
5025 &c_iov, NULL, 1,
5026 &r_iov, (void **)loc_mr, 1,
5027 gni_addr[1],
5028 _REM_ADDR(fi, target, target), mr_key[1],
5029 FI_FLOAT, FI_CSWAP, target);
5030 cr_assert_eq(sz, 0);
5031
5032 /* reset cqe */
5033 cqe.op_context = cqe.buf = (void *) -1;
5034 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5035 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5036 pthread_yield();
5037 }
5038
5039 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5040 dump_cq_error(send_cq[0], target, 0));
5041 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5042 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5043
5044 dbg_printf("got write context event!\n");
5045
5046 ret = *((float *)target) == (float)SOURCE_DATA_FP;
5047 cr_assert(ret, "Data mismatch");
5048 ret = *((float *)source) == (float)TARGET_DATA_FP;
5049 cr_assert(ret, "Fetch data mismatch");
5050
5051 /* double */
5052 *((double *)&operand_dp) = SOURCE_DATA_FP;
5053 *((double *)&op2_dp) = TARGET_DATA_FP;
5054 *((double *)source) = FETCH_SOURCE_DATA;
5055 *((double *)target) = TARGET_DATA_FP;
5056 iov.addr = &operand_dp;
5057 r_iov.addr = source;
5058 c_iov.addr = &op2_dp;
5059 sz = fi_compare_atomicv(ep[0],
5060 &iov, NULL, 1,
5061 &c_iov, NULL, 1,
5062 &r_iov, (void **)loc_mr, 1,
5063 gni_addr[1],
5064 _REM_ADDR(fi, target, target), mr_key[1],
5065 FI_DOUBLE, FI_CSWAP, target);
5066 cr_assert_eq(sz, 0);
5067
5068 /* reset cqe */
5069 cqe.op_context = cqe.buf = (void *) -1;
5070 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5071 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5072 pthread_yield();
5073 }
5074
5075 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5076 dump_cq_error(send_cq[0], target, 0));
5077 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5078 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5079
5080 dbg_printf("got write context event!\n");
5081
5082 ret = *((double *)target) == (double)SOURCE_DATA_FP;
5083 cr_assert(ret, "Data mismatch");
5084 ret = *((double *)source) == (double)TARGET_DATA_FP;
5085 cr_assert(ret, "Fetch data mismatch");
5086 }
5087
Test(rdm_atomic_basic,catomicv)5088 Test(rdm_atomic_basic, catomicv)
5089 {
5090 __catomicv();
5091 }
5092
Test(rdm_atomic_scalable,catomicv)5093 Test(rdm_atomic_scalable, catomicv)
5094 {
5095 __catomicv();
5096 }
5097
__catomicmsg(void)5098 static inline void __catomicmsg(void)
5099 {
5100 int ret;
5101 ssize_t sz;
5102 struct fi_cq_tagged_entry cqe = { (void *) -1, UINT_MAX, UINT_MAX,
5103 (void *) -1, UINT_MAX, UINT_MAX };
5104 uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
5105 float operand_fp, op2_fp;
5106 double operand_dp, op2_dp;
5107 struct fi_msg_atomic msg;
5108 struct fi_ioc msg_iov, res_iov, cmp_iov;
5109 struct fi_rma_ioc rma_iov;
5110 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
5111 uint64_t r_e[NUMEPS] = {0};
5112
5113 msg_iov.count = 1;
5114 msg.msg_iov = &msg_iov;
5115 msg.desc = (void **)loc_mr;
5116 msg.iov_count = 1;
5117 msg.addr = gni_addr[1];
5118 rma_iov.addr = _REM_ADDR(fi, target, target);
5119 rma_iov.count = 1;
5120 rma_iov.key = mr_key[1];
5121 msg.rma_iov = &rma_iov;
5122 msg.context = target;
5123 msg.op = FI_CSWAP;
5124
5125 res_iov.count = 1;
5126 cmp_iov.count = 1;
5127
5128 /* u64 */
5129 *((uint64_t *)source) = FETCH_SOURCE_DATA;
5130 *((uint64_t *)target) = TARGET_DATA;
5131 msg_iov.addr = &operand;
5132 msg.datatype = FI_UINT64;
5133 res_iov.addr = source;
5134 cmp_iov.addr = &op2;
5135 sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
5136 &res_iov, (void **)loc_mr, 1, 0);
5137 cr_assert_eq(sz, 0);
5138
5139 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5140 pthread_yield();
5141 }
5142
5143 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5144 dump_cq_error(send_cq[0], NULL, 0));
5145 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5146
5147 r[0] = 1;
5148 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5149
5150 dbg_printf("got write context event!\n");
5151
5152 ret = *((uint64_t *)target) == SOURCE_DATA;
5153 cr_assert(ret, "Data mismatch");
5154 ret = *((uint64_t *)source) == TARGET_DATA;
5155 cr_assert(ret, "Fetch data mismatch");
5156
5157 /* U32 */
5158 *((uint64_t *)source) = FETCH_SOURCE_DATA;
5159 *((uint64_t *)target) = TARGET_DATA;
5160 msg_iov.addr = &operand;
5161 msg.datatype = FI_UINT32;
5162 res_iov.addr = source;
5163 cmp_iov.addr = &op2;
5164 sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
5165 &res_iov, (void **)loc_mr, 1, 0);
5166 cr_assert_eq(sz, 0);
5167
5168 /* reset cqe */
5169 cqe.op_context = cqe.buf = (void *) -1;
5170 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5171 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5172 pthread_yield();
5173 }
5174
5175 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5176 dump_cq_error(send_cq[0], NULL, 0));
5177 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5178 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5179
5180 dbg_printf("got write context event!\n");
5181
5182 ret = *((uint64_t *)target) ==
5183 (uint64_t)((SOURCE_DATA & U32_MASK) |
5184 (TARGET_DATA & (U32_MASK << 32)));
5185 cr_assert(ret, "Data mismatch");
5186 ret = *((uint64_t *)source) ==
5187 (uint64_t)((TARGET_DATA & U32_MASK) |
5188 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
5189 cr_assert(ret, "Fetch data mismatch");
5190
5191 /* i64 */
5192 *((uint64_t *)source) = FETCH_SOURCE_DATA;
5193 *((uint64_t *)target) = TARGET_DATA;
5194 msg_iov.addr = &operand;
5195 msg.datatype = FI_INT64;
5196 res_iov.addr = source;
5197 cmp_iov.addr = &op2;
5198 sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
5199 &res_iov, (void **)loc_mr, 1, 0);
5200 cr_assert_eq(sz, 0);
5201
5202 /* reset cqe */
5203 cqe.op_context = cqe.buf = (void *) -1;
5204 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5205 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5206 pthread_yield();
5207 }
5208
5209 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5210 dump_cq_error(send_cq[0], NULL, 0));
5211 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5212 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5213
5214 dbg_printf("got write context event!\n");
5215
5216 ret = *((uint64_t *)target) == SOURCE_DATA;
5217 cr_assert(ret, "Data mismatch");
5218 ret = *((uint64_t *)source) == TARGET_DATA;
5219 cr_assert(ret, "Fetch data mismatch");
5220
5221 /* i32 */
5222 *((uint64_t *)source) = FETCH_SOURCE_DATA;
5223 *((uint64_t *)target) = TARGET_DATA;
5224 msg_iov.addr = &operand;
5225 msg.datatype = FI_INT32;
5226 res_iov.addr = source;
5227 cmp_iov.addr = &op2;
5228 sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
5229 &res_iov, (void **)loc_mr, 1, 0);
5230 cr_assert_eq(sz, 0);
5231
5232 /* reset cqe */
5233 cqe.op_context = cqe.buf = (void *) -1;
5234 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5235 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5236 pthread_yield();
5237 }
5238
5239 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5240 dump_cq_error(send_cq[0], NULL, 0));
5241 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5242 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5243
5244 dbg_printf("got write context event!\n");
5245
5246 ret = *((uint64_t *)target) ==
5247 (uint64_t)((SOURCE_DATA & U32_MASK) |
5248 (TARGET_DATA & (U32_MASK << 32)));
5249 cr_assert(ret, "Data mismatch");
5250 ret = *((uint64_t *)source) ==
5251 (uint64_t)((TARGET_DATA & U32_MASK) |
5252 (FETCH_SOURCE_DATA & (U32_MASK << 32)));
5253 cr_assert(ret, "Fetch data mismatch");
5254
5255 /* float */
5256 *((float *)&operand_fp) = SOURCE_DATA_FP;
5257 *((float *)&op2_fp) = TARGET_DATA_FP;
5258 *((float *)source) = FETCH_SOURCE_DATA;
5259 *((float *)target) = TARGET_DATA_FP;
5260 msg_iov.addr = &operand_fp;
5261 msg.datatype = FI_FLOAT;
5262 res_iov.addr = source;
5263 cmp_iov.addr = &op2_fp;
5264 sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
5265 &res_iov, (void **)loc_mr, 1, 0);
5266 cr_assert_eq(sz, 0);
5267
5268 /* reset cqe */
5269 cqe.op_context = cqe.buf = (void *) -1;
5270 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5271 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5272 pthread_yield();
5273 }
5274
5275 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5276 dump_cq_error(send_cq[0], NULL, 0));
5277 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5278 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5279
5280 dbg_printf("got write context event!\n");
5281
5282 ret = *((float *)target) == (float)SOURCE_DATA_FP;
5283 cr_assert(ret, "Data mismatch");
5284 ret = *((float *)source) == (float)TARGET_DATA_FP;
5285 cr_assert(ret, "Fetch data mismatch");
5286
5287 /* double */
5288 *((double *)&operand_dp) = SOURCE_DATA_FP;
5289 *((double *)&op2_dp) = TARGET_DATA_FP;
5290 *((double *)source) = FETCH_SOURCE_DATA;
5291 *((double *)target) = TARGET_DATA_FP;
5292 msg_iov.addr = &operand_dp;
5293 msg.datatype = FI_DOUBLE;
5294 res_iov.addr = source;
5295 cmp_iov.addr = &op2_dp;
5296 sz = fi_compare_atomicmsg(ep[0], &msg, &cmp_iov, NULL, 1,
5297 &res_iov, (void **)loc_mr, 1, 0);
5298 cr_assert_eq(sz, 0);
5299
5300 /* reset cqe */
5301 cqe.op_context = cqe.buf = (void *) -1;
5302 cqe.flags = cqe.len = cqe.data = cqe.tag = UINT_MAX;
5303 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5304 pthread_yield();
5305 }
5306
5307 cr_assert_eq(ret, 1, "fi_cq_read returned %d %d", ret,
5308 dump_cq_error(send_cq[0], NULL, 0));
5309 rdm_atomic_check_tcqe(&cqe, target, FI_ATOMIC | FI_READ, 0);
5310 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5311
5312 dbg_printf("got write context event!\n");
5313
5314 ret = *((double *)target) == (double)SOURCE_DATA_FP;
5315 cr_assert(ret, "Data mismatch");
5316 ret = *((double *)source) == (double)TARGET_DATA_FP;
5317 cr_assert(ret, "Fetch data mismatch");
5318 }
5319
Test(rdm_atomic_basic,catomicmsg)5320 Test(rdm_atomic_basic, catomicmsg)
5321 {
5322 __catomicmsg();
5323 }
5324
Test(rdm_atomic_scalable,catomicmsg)5325 Test(rdm_atomic_scalable, catomicmsg)
5326 {
5327 __catomicmsg();
5328 }
5329
5330 /******************************************************************************
5331 *
5332 * Other
5333 *
5334 *****************************************************************************/
5335
Test(rdm_atomic_default,atomic_err)5336 Test(rdm_atomic_default, atomic_err)
5337 {
5338 int ret;
5339 ssize_t sz;
5340 struct fi_cq_tagged_entry cqe;
5341 struct fi_cq_err_entry err_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
5342 (void *) -1, UINT_MAX, UINT_MAX,
5343 UINT_MAX, INT_MAX, INT_MAX,
5344 (void *) NULL, 0 };
5345 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
5346 uint64_t r_e[NUMEPS] = {0};
5347
5348 rdm_atomic_err_inject_enable();
5349
5350 /* i64 */
5351 *((int64_t *)source) = SOURCE_DATA;
5352 *((int64_t *)target) = TARGET_DATA;
5353 sz = fi_atomic(ep[0], source, 1,
5354 loc_mr[0], gni_addr[1],
5355 _REM_ADDR(fi, target, target), mr_key[1],
5356 FI_INT64, FI_MIN, target);
5357 cr_assert_eq(sz, 0);
5358
5359 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5360 pthread_yield();
5361 }
5362 cr_assert_eq(ret, -FI_EAVAIL);
5363 cr_assert_eq(err_cqe.err_data_size, 0);
5364 ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
5365 cr_assert_eq(ret, 1);
5366
5367 cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
5368 "Bad error context");
5369 cr_assert(err_cqe.flags == (FI_ATOMIC | FI_WRITE));
5370 cr_assert(err_cqe.len == 0, "Bad error len");
5371 cr_assert(err_cqe.buf == 0, "Bad error buf");
5372 cr_assert(err_cqe.data == 0, "Bad error data");
5373 cr_assert(err_cqe.tag == 0, "Bad error tag");
5374 cr_assert(err_cqe.olen == 0, "Bad error olen");
5375 cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
5376 cr_assert(err_cqe.prov_errno == gnixu_to_fi_errno(GNI_RC_TRANSACTION_ERROR),
5377 "Bad prov errno");
5378 cr_assert(err_cqe.err_data_size == 0);
5379 cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
5380
5381 w_e[0] = 1;
5382 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5383 }
5384
Test(rdm_atomic_default,fetch_atomic_err)5385 Test(rdm_atomic_default, fetch_atomic_err)
5386 {
5387 int ret;
5388 ssize_t sz;
5389 struct fi_cq_tagged_entry cqe;
5390 uint64_t operand = SOURCE_DATA;
5391 struct fi_cq_err_entry err_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
5392 (void *) -1, UINT_MAX, UINT_MAX,
5393 UINT_MAX, INT_MAX, INT_MAX,
5394 (void *) NULL, 0 };
5395 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
5396 uint64_t r_e[NUMEPS] = {0};
5397
5398 rdm_atomic_err_inject_enable();
5399
5400 /* i64 */
5401 *((int64_t *)source) = FETCH_SOURCE_DATA;
5402 *((int64_t *)target) = TARGET_DATA;
5403 sz = fi_fetch_atomic(ep[0], &operand, 1, NULL,
5404 source, loc_mr[0], gni_addr[1],
5405 _REM_ADDR(fi, target, target),
5406 mr_key[1], FI_INT64, FI_MIN, target);
5407 cr_assert_eq(sz, 0);
5408
5409 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5410 pthread_yield();
5411 }
5412 cr_assert_eq(ret, -FI_EAVAIL);
5413
5414 ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
5415 cr_assert_eq(ret, 1);
5416
5417 cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
5418 "Bad error context");
5419 cr_assert(err_cqe.flags == (FI_ATOMIC | FI_READ));
5420 cr_assert(err_cqe.len == 0, "Bad error len");
5421 cr_assert(err_cqe.buf == 0, "Bad error buf");
5422 cr_assert(err_cqe.data == 0, "Bad error data");
5423 cr_assert(err_cqe.tag == 0, "Bad error tag");
5424 cr_assert(err_cqe.olen == 0, "Bad error olen");
5425 cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
5426 cr_assert(err_cqe.prov_errno == gnixu_to_fi_errno(GNI_RC_TRANSACTION_ERROR),
5427 "Bad prov errno");
5428 cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
5429
5430 r_e[0] = 1;
5431 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5432 }
5433
Test(rdm_atomic_default,compare_atomic_err)5434 Test(rdm_atomic_default, compare_atomic_err)
5435 {
5436 int ret;
5437 ssize_t sz;
5438 struct fi_cq_tagged_entry cqe;
5439 uint64_t operand = SOURCE_DATA, op2 = TARGET_DATA;
5440 struct fi_cq_err_entry err_cqe = { (void *) -1, UINT_MAX, UINT_MAX,
5441 (void *) -1, UINT_MAX, UINT_MAX,
5442 UINT_MAX, INT_MAX, INT_MAX,
5443 (void *) NULL, 0};
5444 uint64_t w[NUMEPS] = {0}, r[NUMEPS] = {0}, w_e[NUMEPS] = {0};
5445 uint64_t r_e[NUMEPS] = {0};
5446
5447 rdm_atomic_err_inject_enable();
5448
5449 /* u64 */
5450 *((uint64_t *)source) = FETCH_SOURCE_DATA;
5451 *((uint64_t *)target) = TARGET_DATA;
5452 sz = fi_compare_atomic(ep[0], &operand, 1, NULL, &op2, NULL,
5453 source, loc_mr[0], gni_addr[1],
5454 _REM_ADDR(fi, target, target),
5455 mr_key[1], FI_UINT64, FI_CSWAP, target);
5456 cr_assert_eq(sz, 0);
5457
5458 while ((ret = fi_cq_read(send_cq[0], &cqe, 1)) == -FI_EAGAIN) {
5459 pthread_yield();
5460 }
5461 cr_assert_eq(ret, -FI_EAVAIL);
5462
5463 ret = fi_cq_readerr(send_cq[0], &err_cqe, 0);
5464 cr_assert_eq(ret, 1);
5465
5466 cr_assert((uint64_t)err_cqe.op_context == (uint64_t)target,
5467 "Bad error context");
5468 cr_assert(err_cqe.flags == (FI_ATOMIC | FI_READ));
5469 cr_assert(err_cqe.len == 0, "Bad error len");
5470 cr_assert(err_cqe.buf == 0, "Bad error buf");
5471 cr_assert(err_cqe.data == 0, "Bad error data");
5472 cr_assert(err_cqe.tag == 0, "Bad error tag");
5473 cr_assert(err_cqe.olen == 0, "Bad error olen");
5474 cr_assert(err_cqe.err == FI_ECANCELED, "Bad error errno");
5475 cr_assert(err_cqe.prov_errno == gnixu_to_fi_errno(GNI_RC_TRANSACTION_ERROR),
5476 "Bad prov errno");
5477 cr_assert(err_cqe.err_data == NULL, "Bad error provider data");
5478
5479 r_e[0] = 1;
5480 rdm_atomic_check_cntrs(w, r, w_e, r_e);
5481 }
5482
5483 TestSuite(rdm_atomic_rcntr_basic, .init = rdm_atomic_rcntr_basic_setup,
5484 .fini = rdm_atomic_teardown, .disabled = false);
5485
5486 TestSuite(rdm_atomic_rcntr_scalable, .init = rdm_atomic_rcntr_scalable_setup,
5487 .fini = rdm_atomic_teardown, .disabled = false);
5488
Test(rdm_atomic_rcntr_basic,amo_rcntr)5489 Test(rdm_atomic_rcntr_basic, amo_rcntr)
5490 {
5491 rdm_atomic_xfer_for_each_size(do_min, 1, 1);
5492 }
5493
Test(rdm_atomic_rcntr_basic,famo_rcntr)5494 Test(rdm_atomic_rcntr_basic, famo_rcntr)
5495 {
5496 rdm_atomic_xfer_for_each_size(do_fetch_min, 1, 1);
5497 }
5498
Test(rdm_atomic_rcntr_basic,camo_rcntr)5499 Test(rdm_atomic_rcntr_basic, camo_rcntr)
5500 {
5501 rdm_atomic_xfer_for_each_size(do_cswap, 1, 1);
5502 }
5503
Test(rdm_atomic_rcntr_scalable,amo_rcntr)5504 Test(rdm_atomic_rcntr_scalable, amo_rcntr)
5505 {
5506 rdm_atomic_xfer_for_each_size(do_min, 1, 1);
5507 }
5508
Test(rdm_atomic_rcntr_scalable,famo_rcntr)5509 Test(rdm_atomic_rcntr_scalable, famo_rcntr)
5510 {
5511 rdm_atomic_xfer_for_each_size(do_fetch_min, 1, 1);
5512 }
5513
Test(rdm_atomic_rcntr_scalable,camo_rcntr)5514 Test(rdm_atomic_rcntr_scalable, camo_rcntr)
5515 {
5516 rdm_atomic_xfer_for_each_size(do_cswap, 1, 1);
5517 }
5518
5519