1 /*
2 * Copyright (c) 2013-2018 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #ifndef _FI_PSM2_TRIGGER_H
34 #define _FI_PSM2_TRIGGER_H
35
36 #ifdef __cplusplus
37 extern "C" {
38 #endif
39
40 enum psmx2_triggered_op {
41 PSMX2_TRIGGERED_SEND,
42 PSMX2_TRIGGERED_SENDV,
43 PSMX2_TRIGGERED_RECV,
44 PSMX2_TRIGGERED_TSEND,
45 PSMX2_TRIGGERED_TSENDV,
46 PSMX2_TRIGGERED_TRECV,
47 PSMX2_TRIGGERED_WRITE,
48 PSMX2_TRIGGERED_WRITEV,
49 PSMX2_TRIGGERED_READ,
50 PSMX2_TRIGGERED_READV,
51 PSMX2_TRIGGERED_ATOMIC_WRITE,
52 PSMX2_TRIGGERED_ATOMIC_WRITEV,
53 PSMX2_TRIGGERED_ATOMIC_READWRITE,
54 PSMX2_TRIGGERED_ATOMIC_READWRITEV,
55 PSMX2_TRIGGERED_ATOMIC_COMPWRITE,
56 PSMX2_TRIGGERED_ATOMIC_COMPWRITEV,
57 };
58
59 struct psmx2_trigger {
60 enum psmx2_triggered_op op;
61 struct psmx2_fid_cntr *cntr;
62 size_t threshold;
63 union {
64 struct {
65 struct fid_ep *ep;
66 const void *buf;
67 size_t len;
68 void *desc;
69 fi_addr_t dest_addr;
70 void *context;
71 uint64_t flags;
72 uint64_t data;
73 } send;
74 struct {
75 struct fid_ep *ep;
76 const struct iovec *iov;
77 size_t count;
78 void **desc;
79 fi_addr_t dest_addr;
80 void *context;
81 uint64_t flags;
82 uint64_t data;
83 } sendv;
84 struct {
85 struct fid_ep *ep;
86 void *buf;
87 size_t len;
88 void *desc;
89 fi_addr_t src_addr;
90 void *context;
91 uint64_t flags;
92 } recv;
93 struct {
94 struct fid_ep *ep;
95 const void *buf;
96 size_t len;
97 void *desc;
98 fi_addr_t dest_addr;
99 uint64_t tag;
100 void *context;
101 uint64_t flags;
102 uint64_t data;
103 } tsend;
104 struct {
105 struct fid_ep *ep;
106 const struct iovec *iov;
107 size_t count;
108 void **desc;
109 fi_addr_t dest_addr;
110 uint64_t tag;
111 void *context;
112 uint64_t flags;
113 uint64_t data;
114 } tsendv;
115 struct {
116 struct fid_ep *ep;
117 void *buf;
118 size_t len;
119 void *desc;
120 fi_addr_t src_addr;
121 uint64_t tag;
122 uint64_t ignore;
123 void *context;
124 uint64_t flags;
125 } trecv;
126 struct {
127 struct fid_ep *ep;
128 const void *buf;
129 size_t len;
130 void *desc;
131 fi_addr_t dest_addr;
132 uint64_t addr;
133 uint64_t key;
134 void *context;
135 uint64_t flags;
136 uint64_t data;
137 } write;
138 struct {
139 struct fid_ep *ep;
140 const struct iovec *iov;
141 size_t count;
142 void *desc;
143 fi_addr_t dest_addr;
144 uint64_t addr;
145 uint64_t key;
146 void *context;
147 uint64_t flags;
148 uint64_t data;
149 } writev;
150 struct {
151 struct fid_ep *ep;
152 void *buf;
153 size_t len;
154 void *desc;
155 fi_addr_t src_addr;
156 uint64_t addr;
157 uint64_t key;
158 void *context;
159 uint64_t flags;
160 } read;
161 struct {
162 struct fid_ep *ep;
163 const struct iovec *iov;
164 size_t count;
165 void *desc;
166 fi_addr_t src_addr;
167 uint64_t addr;
168 uint64_t key;
169 void *context;
170 uint64_t flags;
171 } readv;
172 struct {
173 struct fid_ep *ep;
174 const void *buf;
175 size_t count;
176 void *desc;
177 fi_addr_t dest_addr;
178 uint64_t addr;
179 uint64_t key;
180 enum fi_datatype datatype;
181 enum fi_op atomic_op;
182 void *context;
183 uint64_t flags;
184 } atomic_write;
185 struct {
186 struct fid_ep *ep;
187 const struct fi_ioc *iov;
188 size_t count;
189 void *desc;
190 fi_addr_t dest_addr;
191 uint64_t addr;
192 uint64_t key;
193 enum fi_datatype datatype;
194 enum fi_op atomic_op;
195 void *context;
196 uint64_t flags;
197 } atomic_writev;
198 struct {
199 struct fid_ep *ep;
200 const void *buf;
201 size_t count;
202 void *desc;
203 void *result;
204 void *result_desc;
205 fi_addr_t dest_addr;
206 uint64_t addr;
207 uint64_t key;
208 enum fi_datatype datatype;
209 enum fi_op atomic_op;
210 void *context;
211 uint64_t flags;
212 } atomic_readwrite;
213 struct {
214 struct fid_ep *ep;
215 const struct fi_ioc *iov;
216 size_t count;
217 void **desc;
218 struct fi_ioc *resultv;
219 void **result_desc;
220 size_t result_count;
221 fi_addr_t dest_addr;
222 uint64_t addr;
223 uint64_t key;
224 enum fi_datatype datatype;
225 enum fi_op atomic_op;
226 void *context;
227 uint64_t flags;
228 } atomic_readwritev;
229 struct {
230 struct fid_ep *ep;
231 const void *buf;
232 size_t count;
233 void *desc;
234 const void *compare;
235 void *compare_desc;
236 void *result;
237 void *result_desc;
238 fi_addr_t dest_addr;
239 uint64_t addr;
240 uint64_t key;
241 enum fi_datatype datatype;
242 enum fi_op atomic_op;
243 void *context;
244 uint64_t flags;
245 } atomic_compwrite;
246 struct {
247 struct fid_ep *ep;
248 const struct fi_ioc *iov;
249 size_t count;
250 void **desc;
251 const struct fi_ioc *comparev;
252 void **compare_desc;
253 size_t compare_count;
254 struct fi_ioc *resultv;
255 void **result_desc;
256 size_t result_count;
257 fi_addr_t dest_addr;
258 uint64_t addr;
259 uint64_t key;
260 enum fi_datatype datatype;
261 enum fi_op atomic_op;
262 void *context;
263 uint64_t flags;
264 } atomic_compwritev;
265 };
266 struct psmx2_trigger *next; /* used for randomly accessed trigger list */
267 struct slist_entry list_entry; /* used for ready-to-fire trigger queue */
268 };
269
270 ssize_t psmx2_send_generic(
271 struct fid_ep *ep,
272 const void *buf, size_t len,
273 void *desc, fi_addr_t dest_addr,
274 void *context, uint64_t flags,
275 uint64_t data);
276
277 ssize_t psmx2_sendv_generic(
278 struct fid_ep *ep,
279 const struct iovec *iov, void **desc,
280 size_t count, fi_addr_t dest_addr,
281 void *context, uint64_t flags,
282 uint64_t data);
283
284 ssize_t psmx2_recv_generic(
285 struct fid_ep *ep,
286 void *buf, size_t len, void *desc,
287 fi_addr_t src_addr, void *context,
288 uint64_t flags);
289
290 ssize_t psmx2_tagged_send_generic(
291 struct fid_ep *ep,
292 const void *buf, size_t len,
293 void *desc, fi_addr_t dest_addr,
294 uint64_t tag, void *context,
295 uint64_t flags, uint64_t data);
296
297 ssize_t psmx2_tagged_sendv_generic(
298 struct fid_ep *ep,
299 const struct iovec *iov, void **desc,
300 size_t count, fi_addr_t dest_addr,
301 uint64_t tag, void *context,
302 uint64_t flags, uint64_t data);
303
304 ssize_t psmx2_tagged_recv_generic(
305 struct fid_ep *ep,
306 void *buf, size_t len,
307 void *desc, fi_addr_t src_addr,
308 uint64_t tag, uint64_t ignore,
309 void *context, uint64_t flags);
310
311 ssize_t psmx2_write_generic(
312 struct fid_ep *ep,
313 const void *buf, size_t len,
314 void *desc, fi_addr_t dest_addr,
315 uint64_t addr, uint64_t key,
316 void *context, uint64_t flags,
317 uint64_t data);
318
319 ssize_t psmx2_writev_generic(
320 struct fid_ep *ep,
321 const struct iovec *iov, void **desc,
322 size_t count, fi_addr_t dest_addr,
323 uint64_t addr, uint64_t key,
324 void *context, uint64_t flags,
325 uint64_t data);
326
327 ssize_t psmx2_read_generic(
328 struct fid_ep *ep,
329 void *buf, size_t len,
330 void *desc, fi_addr_t src_addr,
331 uint64_t addr, uint64_t key,
332 void *context, uint64_t flags);
333
334 ssize_t psmx2_readv_generic(
335 struct fid_ep *ep,
336 const struct iovec *iov, void *desc,
337 size_t count, fi_addr_t src_addr,
338 uint64_t addr, uint64_t key,
339 void *context, uint64_t flags);
340
341 ssize_t psmx2_atomic_write_generic(
342 struct fid_ep *ep,
343 const void *buf,
344 size_t count, void *desc,
345 fi_addr_t dest_addr,
346 uint64_t addr, uint64_t key,
347 enum fi_datatype datatype,
348 enum fi_op op, void *context,
349 uint64_t flags);
350
351 ssize_t psmx2_atomic_readwrite_generic(
352 struct fid_ep *ep,
353 const void *buf,
354 size_t count, void *desc,
355 void *result, void *result_desc,
356 fi_addr_t dest_addr,
357 uint64_t addr, uint64_t key,
358 enum fi_datatype datatype,
359 enum fi_op op, void *context,
360 uint64_t flags);
361
362 ssize_t psmx2_atomic_compwrite_generic(
363 struct fid_ep *ep,
364 const void *buf,
365 size_t count, void *desc,
366 const void *compare, void *compare_desc,
367 void *result, void *result_desc,
368 fi_addr_t dest_addr,
369 uint64_t addr, uint64_t key,
370 enum fi_datatype datatype,
371 enum fi_op op, void *context,
372 uint64_t flags);
373
374 static inline
psmx2_process_trigger(struct psmx2_trx_ctxt * trx_ctxt,struct psmx2_trigger * trigger)375 int psmx2_process_trigger(struct psmx2_trx_ctxt *trx_ctxt,
376 struct psmx2_trigger *trigger)
377 {
378 switch (trigger->op) {
379 case PSMX2_TRIGGERED_SEND:
380 psmx2_send_generic(trigger->send.ep,
381 trigger->send.buf,
382 trigger->send.len,
383 trigger->send.desc,
384 trigger->send.dest_addr,
385 trigger->send.context,
386 trigger->send.flags,
387 trigger->send.data);
388 break;
389 case PSMX2_TRIGGERED_SENDV:
390 psmx2_sendv_generic(trigger->sendv.ep,
391 trigger->sendv.iov,
392 trigger->sendv.desc,
393 trigger->sendv.count,
394 trigger->sendv.dest_addr,
395 trigger->sendv.context,
396 trigger->sendv.flags,
397 trigger->sendv.data);
398 break;
399 case PSMX2_TRIGGERED_RECV:
400 psmx2_recv_generic(trigger->recv.ep,
401 trigger->recv.buf,
402 trigger->recv.len,
403 trigger->recv.desc,
404 trigger->recv.src_addr,
405 trigger->recv.context,
406 trigger->recv.flags);
407 break;
408 case PSMX2_TRIGGERED_TSEND:
409 psmx2_tagged_send_generic(trigger->tsend.ep,
410 trigger->tsend.buf,
411 trigger->tsend.len,
412 trigger->tsend.desc,
413 trigger->tsend.dest_addr,
414 trigger->tsend.tag,
415 trigger->tsend.context,
416 trigger->tsend.flags,
417 trigger->tsend.data);
418 break;
419 case PSMX2_TRIGGERED_TSENDV:
420 psmx2_tagged_sendv_generic(trigger->tsendv.ep,
421 trigger->tsendv.iov,
422 trigger->tsendv.desc,
423 trigger->tsendv.count,
424 trigger->tsendv.dest_addr,
425 trigger->tsendv.tag,
426 trigger->tsendv.context,
427 trigger->tsendv.flags,
428 trigger->tsendv.data);
429 break;
430 case PSMX2_TRIGGERED_TRECV:
431 psmx2_tagged_recv_generic(trigger->trecv.ep,
432 trigger->trecv.buf,
433 trigger->trecv.len,
434 trigger->trecv.desc,
435 trigger->trecv.src_addr,
436 trigger->trecv.tag,
437 trigger->trecv.ignore,
438 trigger->trecv.context,
439 trigger->trecv.flags);
440 break;
441 case PSMX2_TRIGGERED_WRITE:
442 psmx2_write_generic(trigger->write.ep,
443 trigger->write.buf,
444 trigger->write.len,
445 trigger->write.desc,
446 trigger->write.dest_addr,
447 trigger->write.addr,
448 trigger->write.key,
449 trigger->write.context,
450 trigger->write.flags,
451 trigger->write.data);
452 break;
453
454 case PSMX2_TRIGGERED_WRITEV:
455 psmx2_writev_generic(trigger->writev.ep,
456 trigger->writev.iov,
457 trigger->writev.desc,
458 trigger->writev.count,
459 trigger->writev.dest_addr,
460 trigger->writev.addr,
461 trigger->writev.key,
462 trigger->writev.context,
463 trigger->writev.flags,
464 trigger->writev.data);
465 break;
466
467 case PSMX2_TRIGGERED_READ:
468 psmx2_read_generic(trigger->read.ep,
469 trigger->read.buf,
470 trigger->read.len,
471 trigger->read.desc,
472 trigger->read.src_addr,
473 trigger->read.addr,
474 trigger->read.key,
475 trigger->read.context,
476 trigger->read.flags);
477 break;
478
479 case PSMX2_TRIGGERED_READV:
480 psmx2_readv_generic(trigger->readv.ep,
481 trigger->readv.iov,
482 trigger->readv.desc,
483 trigger->readv.count,
484 trigger->readv.src_addr,
485 trigger->readv.addr,
486 trigger->readv.key,
487 trigger->readv.context,
488 trigger->readv.flags);
489 break;
490
491 case PSMX2_TRIGGERED_ATOMIC_WRITE:
492 psmx2_atomic_write_generic(
493 trigger->atomic_write.ep,
494 trigger->atomic_write.buf,
495 trigger->atomic_write.count,
496 trigger->atomic_write.desc,
497 trigger->atomic_write.dest_addr,
498 trigger->atomic_write.addr,
499 trigger->atomic_write.key,
500 trigger->atomic_write.datatype,
501 trigger->atomic_write.atomic_op,
502 trigger->atomic_write.context,
503 trigger->atomic_write.flags);
504 break;
505
506 case PSMX2_TRIGGERED_ATOMIC_READWRITE:
507 psmx2_atomic_readwrite_generic(
508 trigger->atomic_readwrite.ep,
509 trigger->atomic_readwrite.buf,
510 trigger->atomic_readwrite.count,
511 trigger->atomic_readwrite.desc,
512 trigger->atomic_readwrite.result,
513 trigger->atomic_readwrite.result_desc,
514 trigger->atomic_readwrite.dest_addr,
515 trigger->atomic_readwrite.addr,
516 trigger->atomic_readwrite.key,
517 trigger->atomic_readwrite.datatype,
518 trigger->atomic_readwrite.atomic_op,
519 trigger->atomic_readwrite.context,
520 trigger->atomic_readwrite.flags);
521 break;
522
523 case PSMX2_TRIGGERED_ATOMIC_COMPWRITE:
524 psmx2_atomic_compwrite_generic(
525 trigger->atomic_compwrite.ep,
526 trigger->atomic_compwrite.buf,
527 trigger->atomic_compwrite.count,
528 trigger->atomic_compwrite.desc,
529 trigger->atomic_compwrite.compare,
530 trigger->atomic_compwrite.compare_desc,
531 trigger->atomic_compwrite.result,
532 trigger->atomic_compwrite.result_desc,
533 trigger->atomic_compwrite.dest_addr,
534 trigger->atomic_compwrite.addr,
535 trigger->atomic_compwrite.key,
536 trigger->atomic_compwrite.datatype,
537 trigger->atomic_compwrite.atomic_op,
538 trigger->atomic_compwrite.context,
539 trigger->atomic_compwrite.flags);
540 break;
541 default:
542 FI_INFO(&psmx2_prov, FI_LOG_CQ,
543 "%d unsupported op\n", trigger->op);
544 break;
545 }
546
547 free(trigger);
548 return 0;
549 }
550
551 static inline
psmx2_trigger_queue_trecv(struct fid_ep * ep,void * buf,size_t len,void * desc,fi_addr_t src_addr,uint64_t tag,uint64_t ignore,void * context,uint64_t flags)552 int psmx2_trigger_queue_trecv(struct fid_ep *ep, void *buf,
553 size_t len, void *desc,
554 fi_addr_t src_addr,
555 uint64_t tag, uint64_t ignore,
556 void *context, uint64_t flags)
557 {
558 struct psmx2_trigger *trigger;
559 struct fi_triggered_context *ctxt = context;
560
561 trigger = calloc(1, sizeof(*trigger));
562 if (!trigger)
563 return -FI_ENOMEM;
564
565 trigger->op = PSMX2_TRIGGERED_TRECV;
566 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
567 struct psmx2_fid_cntr, cntr);
568 trigger->threshold = ctxt->trigger.threshold.threshold;
569 trigger->trecv.ep = ep;
570 trigger->trecv.buf = buf;
571 trigger->trecv.len = len;
572 trigger->trecv.desc = desc;
573 trigger->trecv.src_addr = src_addr;
574 trigger->trecv.tag = tag;
575 trigger->trecv.ignore = ignore;
576 trigger->trecv.context = context;
577 trigger->trecv.flags = flags & ~FI_TRIGGER;
578
579 psmx2_cntr_add_trigger(trigger->cntr, trigger);
580 return 0;
581 }
582
583 static inline
psmx2_trigger_queue_tsend(struct fid_ep * ep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,uint64_t tag,void * context,uint64_t flags,uint64_t data)584 int psmx2_trigger_queue_tsend(struct fid_ep *ep,
585 const void *buf, size_t len,
586 void *desc, fi_addr_t dest_addr,
587 uint64_t tag, void *context,
588 uint64_t flags, uint64_t data)
589 {
590 struct psmx2_trigger *trigger;
591 struct fi_triggered_context *ctxt = context;
592
593 trigger = calloc(1, sizeof(*trigger));
594 if (!trigger)
595 return -FI_ENOMEM;
596
597 trigger->op = PSMX2_TRIGGERED_TSEND;
598 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
599 struct psmx2_fid_cntr, cntr);
600 trigger->threshold = ctxt->trigger.threshold.threshold;
601 trigger->tsend.ep = ep;
602 trigger->tsend.buf = buf;
603 trigger->tsend.len = len;
604 trigger->tsend.desc = desc;
605 trigger->tsend.dest_addr = dest_addr;
606 trigger->tsend.tag = tag;
607 trigger->tsend.context = context;
608 trigger->tsend.flags = flags & ~FI_TRIGGER;
609 trigger->tsend.data = data;
610
611 psmx2_cntr_add_trigger(trigger->cntr, trigger);
612 return 0;
613 }
614
615 static inline
psmx2_trigger_queue_tsendv(struct fid_ep * ep,const struct iovec * iov,void * desc,size_t count,fi_addr_t dest_addr,uint64_t tag,void * context,uint64_t flags,uint64_t data)616 int psmx2_trigger_queue_tsendv(struct fid_ep *ep,
617 const struct iovec *iov, void *desc,
618 size_t count, fi_addr_t dest_addr,
619 uint64_t tag, void *context,
620 uint64_t flags, uint64_t data)
621 {
622 struct psmx2_trigger *trigger;
623 struct fi_triggered_context *ctxt = context;
624
625 trigger = calloc(1, sizeof(*trigger));
626 if (!trigger)
627 return -FI_ENOMEM;
628
629 trigger->op = PSMX2_TRIGGERED_TSENDV;
630 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
631 struct psmx2_fid_cntr, cntr);
632 trigger->threshold = ctxt->trigger.threshold.threshold;
633 trigger->tsendv.ep = ep;
634 trigger->tsendv.iov = iov;
635 trigger->tsendv.desc = desc;
636 trigger->tsendv.count = count;
637 trigger->tsendv.dest_addr = dest_addr;
638 trigger->tsendv.tag = tag;
639 trigger->tsendv.context = context;
640 trigger->tsendv.flags = flags & ~FI_TRIGGER;
641 trigger->tsendv.data = data;
642
643 psmx2_cntr_add_trigger(trigger->cntr, trigger);
644 return 0;
645 }
646
647 static inline
psmx2_trigger_queue_recv(struct fid_ep * ep,void * buf,size_t len,void * desc,fi_addr_t src_addr,void * context,uint64_t flags)648 int psmx2_trigger_queue_recv(struct fid_ep *ep, void *buf, size_t len,
649 void *desc, fi_addr_t src_addr, void *context,
650 uint64_t flags)
651 {
652 struct psmx2_trigger *trigger;
653 struct fi_triggered_context *ctxt = context;
654
655 trigger = calloc(1, sizeof(*trigger));
656 if (!trigger)
657 return -FI_ENOMEM;
658
659 trigger->op = PSMX2_TRIGGERED_RECV;
660 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
661 struct psmx2_fid_cntr, cntr);
662 trigger->threshold = ctxt->trigger.threshold.threshold;
663 trigger->recv.ep = ep;
664 trigger->recv.buf = buf;
665 trigger->recv.len = len;
666 trigger->recv.desc = desc;
667 trigger->recv.src_addr = src_addr;
668 trigger->recv.context = context;
669 trigger->recv.flags = flags & ~FI_TRIGGER;
670
671 psmx2_cntr_add_trigger(trigger->cntr, trigger);
672 return 0;
673 }
674
675 static inline
psmx2_trigger_queue_send(struct fid_ep * ep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,void * context,uint64_t flags,uint64_t data)676 int psmx2_trigger_queue_send(struct fid_ep *ep, const void *buf, size_t len,
677 void *desc, fi_addr_t dest_addr, void *context,
678 uint64_t flags, uint64_t data)
679 {
680 struct psmx2_trigger *trigger;
681 struct fi_triggered_context *ctxt = context;
682
683 trigger = calloc(1, sizeof(*trigger));
684 if (!trigger)
685 return -FI_ENOMEM;
686
687 trigger->op = PSMX2_TRIGGERED_SEND;
688 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
689 struct psmx2_fid_cntr, cntr);
690 trigger->threshold = ctxt->trigger.threshold.threshold;
691 trigger->send.ep = ep;
692 trigger->send.buf = buf;
693 trigger->send.len = len;
694 trigger->send.desc = desc;
695 trigger->send.dest_addr = dest_addr;
696 trigger->send.context = context;
697 trigger->send.flags = flags & ~FI_TRIGGER;
698 trigger->send.data = data;
699
700 psmx2_cntr_add_trigger(trigger->cntr, trigger);
701 return 0;
702 }
703
704 static inline
psmx2_trigger_queue_sendv(struct fid_ep * ep,const struct iovec * iov,void * desc,size_t count,fi_addr_t dest_addr,void * context,uint64_t flags,uint64_t data)705 int psmx2_trigger_queue_sendv(struct fid_ep *ep, const struct iovec *iov,
706 void *desc, size_t count, fi_addr_t dest_addr,
707 void *context, uint64_t flags, uint64_t data)
708 {
709 struct psmx2_trigger *trigger;
710 struct fi_triggered_context *ctxt = context;
711
712 trigger = calloc(1, sizeof(*trigger));
713 if (!trigger)
714 return -FI_ENOMEM;
715
716 trigger->op = PSMX2_TRIGGERED_SENDV;
717 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
718 struct psmx2_fid_cntr, cntr);
719 trigger->threshold = ctxt->trigger.threshold.threshold;
720 trigger->sendv.ep = ep;
721 trigger->sendv.iov = iov;
722 trigger->sendv.desc = desc;
723 trigger->sendv.count = count;
724 trigger->sendv.dest_addr = dest_addr;
725 trigger->sendv.context = context;
726 trigger->sendv.flags = flags & ~FI_TRIGGER;
727 trigger->sendv.data = data;
728
729 psmx2_cntr_add_trigger(trigger->cntr, trigger);
730 return 0;
731 }
732
733 static inline
psmx2_trigger_queue_read(struct fid_ep * ep,void * buf,size_t len,void * desc,fi_addr_t src_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags)734 int psmx2_trigger_queue_read(struct fid_ep *ep, void *buf, size_t len,
735 void *desc, fi_addr_t src_addr,
736 uint64_t addr, uint64_t key, void *context,
737 uint64_t flags)
738 {
739 struct psmx2_trigger *trigger;
740 struct fi_triggered_context *ctxt = context;
741
742 trigger = calloc(1, sizeof(*trigger));
743 if (!trigger)
744 return -FI_ENOMEM;
745
746 trigger->op = PSMX2_TRIGGERED_READ;
747 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
748 struct psmx2_fid_cntr, cntr);
749 trigger->threshold = ctxt->trigger.threshold.threshold;
750 trigger->read.ep = ep;
751 trigger->read.buf = buf;
752 trigger->read.len = len;
753 trigger->read.desc = desc;
754 trigger->read.src_addr = src_addr;
755 trigger->read.addr = addr;
756 trigger->read.key = key;
757 trigger->read.context = context;
758 trigger->read.flags = flags & ~FI_TRIGGER;
759
760 psmx2_cntr_add_trigger(trigger->cntr, trigger);
761 return 0;
762 }
763
764 static inline
psmx2_trigger_queue_readv(struct fid_ep * ep,const struct iovec * iov,void * desc,size_t count,fi_addr_t src_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags)765 int psmx2_trigger_queue_readv(struct fid_ep *ep, const struct iovec *iov,
766 void *desc, size_t count, fi_addr_t src_addr,
767 uint64_t addr, uint64_t key, void *context,
768 uint64_t flags)
769 {
770 struct psmx2_trigger *trigger;
771 struct fi_triggered_context *ctxt = context;
772
773 trigger = calloc(1, sizeof(*trigger));
774 if (!trigger)
775 return -FI_ENOMEM;
776
777 trigger->op = PSMX2_TRIGGERED_READV;
778 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
779 struct psmx2_fid_cntr, cntr);
780 trigger->threshold = ctxt->trigger.threshold.threshold;
781 trigger->readv.ep = ep;
782 trigger->readv.iov = iov;
783 trigger->readv.count = count;
784 trigger->readv.desc = desc;
785 trigger->readv.src_addr = src_addr;
786 trigger->readv.addr = addr;
787 trigger->readv.key = key;
788 trigger->readv.context = context;
789 trigger->readv.flags = flags & ~FI_TRIGGER;
790
791 psmx2_cntr_add_trigger(trigger->cntr, trigger);
792 return 0;
793 }
794
795 static inline
psmx2_trigger_queue_write(struct fid_ep * ep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags,uint64_t data)796 int psmx2_trigger_queue_write(struct fid_ep *ep, const void *buf, size_t len,
797 void *desc, fi_addr_t dest_addr,
798 uint64_t addr, uint64_t key, void *context,
799 uint64_t flags, uint64_t data)
800 {
801 struct psmx2_trigger *trigger;
802 struct fi_triggered_context *ctxt = context;
803
804 trigger = calloc(1, sizeof(*trigger));
805 if (!trigger)
806 return -FI_ENOMEM;
807
808 trigger->op = PSMX2_TRIGGERED_WRITE;
809 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
810 struct psmx2_fid_cntr, cntr);
811 trigger->threshold = ctxt->trigger.threshold.threshold;
812 trigger->write.ep = ep;
813 trigger->write.buf = buf;
814 trigger->write.len = len;
815 trigger->write.desc = desc;
816 trigger->write.dest_addr = dest_addr;
817 trigger->write.addr = addr;
818 trigger->write.key = key;
819 trigger->write.context = context;
820 trigger->write.flags = flags & ~FI_TRIGGER;
821 trigger->write.data = data;
822
823 psmx2_cntr_add_trigger(trigger->cntr, trigger);
824 return 0;
825 }
826
827 static inline
psmx2_trigger_queue_writev(struct fid_ep * ep,const struct iovec * iov,void ** desc,size_t count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags,uint64_t data)828 int psmx2_trigger_queue_writev(struct fid_ep *ep, const struct iovec *iov,
829 void **desc, size_t count, fi_addr_t dest_addr,
830 uint64_t addr, uint64_t key, void *context,
831 uint64_t flags, uint64_t data)
832 {
833 struct psmx2_trigger *trigger;
834 struct fi_triggered_context *ctxt = context;
835
836 trigger = calloc(1, sizeof(*trigger));
837 if (!trigger)
838 return -FI_ENOMEM;
839
840 trigger->op = PSMX2_TRIGGERED_WRITEV;
841 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
842 struct psmx2_fid_cntr, cntr);
843 trigger->threshold = ctxt->trigger.threshold.threshold;
844 trigger->writev.ep = ep;
845 trigger->writev.iov = iov;
846 trigger->writev.count = count;
847 trigger->writev.desc = desc;
848 trigger->writev.dest_addr = dest_addr;
849 trigger->writev.addr = addr;
850 trigger->writev.key = key;
851 trigger->writev.context = context;
852 trigger->writev.flags = flags & ~FI_TRIGGER;
853 trigger->writev.data = data;
854
855 psmx2_cntr_add_trigger(trigger->cntr, trigger);
856 return 0;
857 }
858
859 static inline
psmx2_trigger_queue_atomic_write(struct fid_ep * ep,const void * buf,size_t count,void * desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)860 int psmx2_trigger_queue_atomic_write(struct fid_ep *ep,
861 const void *buf,
862 size_t count, void *desc,
863 fi_addr_t dest_addr,
864 uint64_t addr, uint64_t key,
865 enum fi_datatype datatype,
866 enum fi_op op, void *context,
867 uint64_t flags)
868 {
869 struct psmx2_trigger *trigger;
870 struct fi_triggered_context *ctxt = context;
871
872 trigger = calloc(1, sizeof(*trigger));
873 if (!trigger)
874 return -FI_ENOMEM;
875
876 trigger->op = PSMX2_TRIGGERED_ATOMIC_WRITE;
877 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
878 struct psmx2_fid_cntr, cntr);
879 trigger->threshold = ctxt->trigger.threshold.threshold;
880 trigger->atomic_write.ep = ep;
881 trigger->atomic_write.buf = buf;
882 trigger->atomic_write.count = count;
883 trigger->atomic_write.desc = desc;
884 trigger->atomic_write.dest_addr = dest_addr;
885 trigger->atomic_write.addr = addr;
886 trigger->atomic_write.key = key;
887 trigger->atomic_write.datatype = datatype;
888 trigger->atomic_write.atomic_op = op;
889 trigger->atomic_write.context = context;
890 trigger->atomic_write.flags = flags & ~FI_TRIGGER;
891
892 psmx2_cntr_add_trigger(trigger->cntr, trigger);
893 return 0;
894 }
895
896 static inline
psmx2_trigger_queue_atomic_writev(struct fid_ep * ep,const struct fi_ioc * iov,void ** desc,size_t count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)897 int psmx2_trigger_queue_atomic_writev(struct fid_ep *ep,
898 const struct fi_ioc *iov,
899 void **desc, size_t count,
900 fi_addr_t dest_addr,
901 uint64_t addr, uint64_t key,
902 enum fi_datatype datatype,
903 enum fi_op op, void *context,
904 uint64_t flags)
905 {
906 struct psmx2_trigger *trigger;
907 struct fi_triggered_context *ctxt = context;
908
909 trigger = calloc(1, sizeof(*trigger));
910 if (!trigger)
911 return -FI_ENOMEM;
912
913 trigger->op = PSMX2_TRIGGERED_ATOMIC_WRITEV;
914 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
915 struct psmx2_fid_cntr, cntr);
916 trigger->threshold = ctxt->trigger.threshold.threshold;
917 trigger->atomic_writev.ep = ep;
918 trigger->atomic_writev.iov = iov;
919 trigger->atomic_writev.count = count;
920 trigger->atomic_writev.desc = desc;
921 trigger->atomic_writev.dest_addr = dest_addr;
922 trigger->atomic_writev.addr = addr;
923 trigger->atomic_writev.key = key;
924 trigger->atomic_writev.datatype = datatype;
925 trigger->atomic_writev.atomic_op = op;
926 trigger->atomic_writev.context = context;
927 trigger->atomic_writev.flags = flags & ~FI_TRIGGER;
928
929 psmx2_cntr_add_trigger(trigger->cntr, trigger);
930 return 0;
931 }
932
933 static inline
psmx2_trigger_queue_atomic_readwrite(struct fid_ep * ep,const void * buf,size_t count,void * desc,void * result,void * result_desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)934 int psmx2_trigger_queue_atomic_readwrite(struct fid_ep *ep,
935 const void *buf,
936 size_t count, void *desc,
937 void *result, void *result_desc,
938 fi_addr_t dest_addr,
939 uint64_t addr, uint64_t key,
940 enum fi_datatype datatype,
941 enum fi_op op, void *context,
942 uint64_t flags)
943 {
944 struct psmx2_trigger *trigger;
945 struct fi_triggered_context *ctxt = context;
946
947 trigger = calloc(1, sizeof(*trigger));
948 if (!trigger)
949 return -FI_ENOMEM;
950
951 trigger->op = PSMX2_TRIGGERED_ATOMIC_READWRITE;
952 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
953 struct psmx2_fid_cntr, cntr);
954 trigger->threshold = ctxt->trigger.threshold.threshold;
955 trigger->atomic_readwrite.ep = ep;
956 trigger->atomic_readwrite.buf = buf;
957 trigger->atomic_readwrite.count = count;
958 trigger->atomic_readwrite.desc = desc;
959 trigger->atomic_readwrite.result = result;
960 trigger->atomic_readwrite.result_desc = result_desc;
961 trigger->atomic_readwrite.dest_addr = dest_addr;
962 trigger->atomic_readwrite.addr = addr;
963 trigger->atomic_readwrite.key = key;
964 trigger->atomic_readwrite.datatype = datatype;
965 trigger->atomic_readwrite.atomic_op = op;
966 trigger->atomic_readwrite.context = context;
967 trigger->atomic_readwrite.flags = flags & ~FI_TRIGGER;
968
969 psmx2_cntr_add_trigger(trigger->cntr, trigger);
970 return 0;
971 }
972
973 static inline
psmx2_trigger_queue_atomic_readwritev(struct fid_ep * ep,const struct fi_ioc * iov,void ** desc,size_t count,struct fi_ioc * resultv,void ** result_desc,size_t result_count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)974 int psmx2_trigger_queue_atomic_readwritev(struct fid_ep *ep,
975 const struct fi_ioc *iov,
976 void **desc, size_t count,
977 struct fi_ioc *resultv,
978 void **result_desc,
979 size_t result_count,
980 fi_addr_t dest_addr,
981 uint64_t addr, uint64_t key,
982 enum fi_datatype datatype,
983 enum fi_op op, void *context,
984 uint64_t flags)
985 {
986 struct psmx2_trigger *trigger;
987 struct fi_triggered_context *ctxt = context;
988
989 trigger = calloc(1, sizeof(*trigger));
990 if (!trigger)
991 return -FI_ENOMEM;
992
993 trigger->op = PSMX2_TRIGGERED_ATOMIC_READWRITEV;
994 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
995 struct psmx2_fid_cntr, cntr);
996 trigger->threshold = ctxt->trigger.threshold.threshold;
997 trigger->atomic_readwritev.ep = ep;
998 trigger->atomic_readwritev.iov = iov;
999 trigger->atomic_readwritev.count = count;
1000 trigger->atomic_readwritev.desc = desc;
1001 trigger->atomic_readwritev.resultv = resultv;
1002 trigger->atomic_readwritev.result_desc = result_desc;
1003 trigger->atomic_readwritev.result_count = result_count;
1004 trigger->atomic_readwritev.dest_addr = dest_addr;
1005 trigger->atomic_readwritev.addr = addr;
1006 trigger->atomic_readwritev.key = key;
1007 trigger->atomic_readwritev.datatype = datatype;
1008 trigger->atomic_readwritev.atomic_op = op;
1009 trigger->atomic_readwritev.context = context;
1010 trigger->atomic_readwritev.flags = flags & ~FI_TRIGGER;
1011
1012 psmx2_cntr_add_trigger(trigger->cntr, trigger);
1013 return 0;
1014 }
1015
1016 static inline
psmx2_trigger_queue_atomic_compwrite(struct fid_ep * ep,const void * buf,size_t count,void * desc,const void * compare,void * compare_desc,void * result,void * result_desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)1017 int psmx2_trigger_queue_atomic_compwrite(struct fid_ep *ep,
1018 const void *buf,
1019 size_t count, void *desc,
1020 const void *compare, void *compare_desc,
1021 void *result, void *result_desc,
1022 fi_addr_t dest_addr,
1023 uint64_t addr, uint64_t key,
1024 enum fi_datatype datatype,
1025 enum fi_op op, void *context,
1026 uint64_t flags)
1027 {
1028 struct psmx2_trigger *trigger;
1029 struct fi_triggered_context *ctxt = context;
1030
1031 trigger = calloc(1, sizeof(*trigger));
1032 if (!trigger)
1033 return -FI_ENOMEM;
1034
1035 trigger->op = PSMX2_TRIGGERED_ATOMIC_COMPWRITE;
1036 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
1037 struct psmx2_fid_cntr, cntr);
1038 trigger->threshold = ctxt->trigger.threshold.threshold;
1039 trigger->atomic_compwrite.ep = ep;
1040 trigger->atomic_compwrite.buf = buf;
1041 trigger->atomic_compwrite.count = count;
1042 trigger->atomic_compwrite.desc = desc;
1043 trigger->atomic_compwrite.compare = compare;
1044 trigger->atomic_compwrite.compare_desc = compare_desc;
1045 trigger->atomic_compwrite.result = result;
1046 trigger->atomic_compwrite.result_desc = result_desc;
1047 trigger->atomic_compwrite.dest_addr = dest_addr;
1048 trigger->atomic_compwrite.addr = addr;
1049 trigger->atomic_compwrite.key = key;
1050 trigger->atomic_compwrite.datatype = datatype;
1051 trigger->atomic_compwrite.atomic_op = op;
1052 trigger->atomic_compwrite.context = context;
1053 trigger->atomic_compwrite.flags = flags & ~FI_TRIGGER;
1054
1055 psmx2_cntr_add_trigger(trigger->cntr, trigger);
1056 return 0;
1057 }
1058
1059 static inline
psmx2_trigger_queue_atomic_compwritev(struct fid_ep * ep,const struct fi_ioc * iov,void ** desc,size_t count,const struct fi_ioc * comparev,void ** compare_desc,size_t compare_count,struct fi_ioc * resultv,void ** result_desc,size_t result_count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)1060 int psmx2_trigger_queue_atomic_compwritev(struct fid_ep *ep,
1061 const struct fi_ioc *iov,
1062 void **desc, size_t count,
1063 const struct fi_ioc *comparev,
1064 void **compare_desc,
1065 size_t compare_count,
1066 struct fi_ioc *resultv,
1067 void **result_desc,
1068 size_t result_count,
1069 fi_addr_t dest_addr,
1070 uint64_t addr, uint64_t key,
1071 enum fi_datatype datatype,
1072 enum fi_op op, void *context,
1073 uint64_t flags)
1074 {
1075 struct psmx2_trigger *trigger;
1076 struct fi_triggered_context *ctxt = context;
1077
1078 trigger = calloc(1, sizeof(*trigger));
1079 if (!trigger)
1080 return -FI_ENOMEM;
1081
1082 trigger->op = PSMX2_TRIGGERED_ATOMIC_COMPWRITEV;
1083 trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
1084 struct psmx2_fid_cntr, cntr);
1085 trigger->threshold = ctxt->trigger.threshold.threshold;
1086 trigger->atomic_compwritev.ep = ep;
1087 trigger->atomic_compwritev.iov = iov;
1088 trigger->atomic_compwritev.desc = desc;
1089 trigger->atomic_compwritev.count = count;
1090 trigger->atomic_compwritev.comparev = comparev;
1091 trigger->atomic_compwritev.compare_desc = compare_desc;
1092 trigger->atomic_compwritev.compare_count = compare_count;
1093 trigger->atomic_compwritev.resultv = resultv;
1094 trigger->atomic_compwritev.result_desc = result_desc;
1095 trigger->atomic_compwritev.result_count = result_count;
1096 trigger->atomic_compwritev.dest_addr = dest_addr;
1097 trigger->atomic_compwritev.addr = addr;
1098 trigger->atomic_compwritev.key = key;
1099 trigger->atomic_compwritev.datatype = datatype;
1100 trigger->atomic_compwritev.atomic_op = op;
1101 trigger->atomic_compwritev.context = context;
1102 trigger->atomic_compwritev.flags = flags & ~FI_TRIGGER;
1103
1104 psmx2_cntr_add_trigger(trigger->cntr, trigger);
1105 return 0;
1106 }
1107
1108 #ifdef __cplusplus
1109 }
1110 #endif
1111
1112 #endif
1113
1114