1 /*
2  * Copyright (c) 2013-2018 Intel Corporation. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #ifndef _FI_PSM2_TRIGGER_H
34 #define _FI_PSM2_TRIGGER_H
35 
36 #ifdef __cplusplus
37 extern "C" {
38 #endif
39 
40 enum psmx2_triggered_op {
41 	PSMX2_TRIGGERED_SEND,
42 	PSMX2_TRIGGERED_SENDV,
43 	PSMX2_TRIGGERED_RECV,
44 	PSMX2_TRIGGERED_TSEND,
45 	PSMX2_TRIGGERED_TSENDV,
46 	PSMX2_TRIGGERED_TRECV,
47 	PSMX2_TRIGGERED_WRITE,
48 	PSMX2_TRIGGERED_WRITEV,
49 	PSMX2_TRIGGERED_READ,
50 	PSMX2_TRIGGERED_READV,
51 	PSMX2_TRIGGERED_ATOMIC_WRITE,
52 	PSMX2_TRIGGERED_ATOMIC_WRITEV,
53 	PSMX2_TRIGGERED_ATOMIC_READWRITE,
54 	PSMX2_TRIGGERED_ATOMIC_READWRITEV,
55 	PSMX2_TRIGGERED_ATOMIC_COMPWRITE,
56 	PSMX2_TRIGGERED_ATOMIC_COMPWRITEV,
57 };
58 
59 struct psmx2_trigger {
60 	enum psmx2_triggered_op	op;
61 	struct psmx2_fid_cntr	*cntr;
62 	size_t			threshold;
63 	union {
64 		struct {
65 			struct fid_ep	*ep;
66 			const void	*buf;
67 			size_t		len;
68 			void		*desc;
69 			fi_addr_t	dest_addr;
70 			void		*context;
71 			uint64_t	flags;
72 			uint64_t	data;
73 		} send;
74 		struct {
75 			struct fid_ep	*ep;
76 			const struct iovec *iov;
77 			size_t		count;
78 			void		**desc;
79 			fi_addr_t	dest_addr;
80 			void		*context;
81 			uint64_t	flags;
82 			uint64_t	data;
83 		} sendv;
84 		struct {
85 			struct fid_ep	*ep;
86 			void		*buf;
87 			size_t		len;
88 			void		*desc;
89 			fi_addr_t	src_addr;
90 			void		*context;
91 			uint64_t	flags;
92 		} recv;
93 		struct {
94 			struct fid_ep	*ep;
95 			const void	*buf;
96 			size_t		len;
97 			void		*desc;
98 			fi_addr_t	dest_addr;
99 			uint64_t	tag;
100 			void		*context;
101 			uint64_t	flags;
102 			uint64_t	data;
103 		} tsend;
104 		struct {
105 			struct fid_ep	*ep;
106 			const struct iovec *iov;
107 			size_t		count;
108 			void		**desc;
109 			fi_addr_t	dest_addr;
110 			uint64_t	tag;
111 			void		*context;
112 			uint64_t	flags;
113 			uint64_t	data;
114 		} tsendv;
115 		struct {
116 			struct fid_ep	*ep;
117 			void		*buf;
118 			size_t		len;
119 			void		*desc;
120 			fi_addr_t	src_addr;
121 			uint64_t	tag;
122 			uint64_t	ignore;
123 			void		*context;
124 			uint64_t	flags;
125 		} trecv;
126 		struct {
127 			struct fid_ep	*ep;
128 			const void	*buf;
129 			size_t		len;
130 			void		*desc;
131 			fi_addr_t	dest_addr;
132 			uint64_t	addr;
133 			uint64_t	key;
134 			void		*context;
135 			uint64_t	flags;
136 			uint64_t	data;
137 		} write;
138 		struct {
139 			struct fid_ep	*ep;
140 			const struct iovec *iov;
141 			size_t		count;
142 			void		*desc;
143 			fi_addr_t	dest_addr;
144 			uint64_t	addr;
145 			uint64_t	key;
146 			void		*context;
147 			uint64_t	flags;
148 			uint64_t	data;
149 		} writev;
150 		struct {
151 			struct fid_ep	*ep;
152 			void		*buf;
153 			size_t		len;
154 			void		*desc;
155 			fi_addr_t	src_addr;
156 			uint64_t	addr;
157 			uint64_t	key;
158 			void		*context;
159 			uint64_t	flags;
160 		} read;
161 		struct {
162 			struct fid_ep	*ep;
163 			const struct iovec *iov;
164 			size_t		count;
165 			void		*desc;
166 			fi_addr_t	src_addr;
167 			uint64_t	addr;
168 			uint64_t	key;
169 			void		*context;
170 			uint64_t	flags;
171 		} readv;
172 		struct {
173 			struct fid_ep	*ep;
174 			const void	*buf;
175 			size_t		count;
176 			void		*desc;
177 			fi_addr_t	dest_addr;
178 			uint64_t	addr;
179 			uint64_t	key;
180 			enum fi_datatype datatype;
181 			enum fi_op	atomic_op;
182 			void		*context;
183 			uint64_t	flags;
184 		} atomic_write;
185 		struct {
186 			struct fid_ep	*ep;
187 			const struct fi_ioc *iov;
188 			size_t		count;
189 			void		*desc;
190 			fi_addr_t	dest_addr;
191 			uint64_t	addr;
192 			uint64_t	key;
193 			enum fi_datatype datatype;
194 			enum fi_op	atomic_op;
195 			void		*context;
196 			uint64_t	flags;
197 		} atomic_writev;
198 		struct {
199 			struct fid_ep	*ep;
200 			const void	*buf;
201 			size_t		count;
202 			void		*desc;
203 			void		*result;
204 			void		*result_desc;
205 			fi_addr_t	dest_addr;
206 			uint64_t	addr;
207 			uint64_t	key;
208 			enum fi_datatype datatype;
209 			enum fi_op	atomic_op;
210 			void		*context;
211 			uint64_t	flags;
212 		} atomic_readwrite;
213 		struct {
214 			struct fid_ep	*ep;
215 			const struct fi_ioc *iov;
216 			size_t		count;
217 			void		**desc;
218 			struct fi_ioc	*resultv;
219 			void		**result_desc;
220 			size_t		result_count;
221 			fi_addr_t	dest_addr;
222 			uint64_t	addr;
223 			uint64_t	key;
224 			enum fi_datatype datatype;
225 			enum fi_op	atomic_op;
226 			void		*context;
227 			uint64_t	flags;
228 		} atomic_readwritev;
229 		struct {
230 			struct fid_ep	*ep;
231 			const void	*buf;
232 			size_t		count;
233 			void		*desc;
234 			const void	*compare;
235 			void		*compare_desc;
236 			void		*result;
237 			void		*result_desc;
238 			fi_addr_t	dest_addr;
239 			uint64_t	addr;
240 			uint64_t	key;
241 			enum fi_datatype datatype;
242 			enum fi_op	atomic_op;
243 			void		*context;
244 			uint64_t	flags;
245 		} atomic_compwrite;
246 		struct {
247 			struct fid_ep	*ep;
248 			const struct fi_ioc *iov;
249 			size_t		count;
250 			void		**desc;
251 			const struct fi_ioc *comparev;
252 			void		**compare_desc;
253 			size_t		compare_count;
254 			struct fi_ioc	*resultv;
255 			void		**result_desc;
256 			size_t		result_count;
257 			fi_addr_t	dest_addr;
258 			uint64_t	addr;
259 			uint64_t	key;
260 			enum fi_datatype datatype;
261 			enum fi_op	atomic_op;
262 			void		*context;
263 			uint64_t	flags;
264 		} atomic_compwritev;
265 	};
266 	struct psmx2_trigger *next;	/* used for randomly accessed trigger list */
267 	struct slist_entry list_entry;	/* used for ready-to-fire trigger queue */
268 };
269 
270 ssize_t psmx2_send_generic(
271 			struct fid_ep *ep,
272 			const void *buf, size_t len,
273 			void *desc, fi_addr_t dest_addr,
274 			void *context, uint64_t flags,
275 			uint64_t data);
276 
277 ssize_t psmx2_sendv_generic(
278 			struct fid_ep *ep,
279 			const struct iovec *iov, void **desc,
280 			size_t count, fi_addr_t dest_addr,
281 			void *context, uint64_t flags,
282 			uint64_t data);
283 
284 ssize_t psmx2_recv_generic(
285 			struct fid_ep *ep,
286 			void *buf, size_t len, void *desc,
287 			fi_addr_t src_addr, void *context,
288 			uint64_t flags);
289 
290 ssize_t psmx2_tagged_send_generic(
291 			struct fid_ep *ep,
292 			const void *buf, size_t len,
293 			void *desc, fi_addr_t dest_addr,
294 			uint64_t tag, void *context,
295 			uint64_t flags, uint64_t data);
296 
297 ssize_t psmx2_tagged_sendv_generic(
298 			struct fid_ep *ep,
299 			const struct iovec *iov, void **desc,
300 			size_t count, fi_addr_t dest_addr,
301 			uint64_t tag, void *context,
302 			uint64_t flags, uint64_t data);
303 
304 ssize_t psmx2_tagged_recv_generic(
305 			struct fid_ep *ep,
306 			void *buf, size_t len,
307 			void *desc, fi_addr_t src_addr,
308 			uint64_t tag, uint64_t ignore,
309 			void *context, uint64_t flags);
310 
311 ssize_t psmx2_write_generic(
312 			struct fid_ep *ep,
313 			const void *buf, size_t len,
314 			void *desc, fi_addr_t dest_addr,
315 			uint64_t addr, uint64_t key,
316 			void *context, uint64_t flags,
317 			uint64_t data);
318 
319 ssize_t psmx2_writev_generic(
320 			struct fid_ep *ep,
321 			const struct iovec *iov, void **desc,
322 			size_t count, fi_addr_t dest_addr,
323 			uint64_t addr, uint64_t key,
324 			void *context, uint64_t flags,
325 			uint64_t data);
326 
327 ssize_t psmx2_read_generic(
328 			struct fid_ep *ep,
329 			void *buf, size_t len,
330 			void *desc, fi_addr_t src_addr,
331 			uint64_t addr, uint64_t key,
332 			void *context, uint64_t flags);
333 
334 ssize_t psmx2_readv_generic(
335 			struct fid_ep *ep,
336 			const struct iovec *iov, void *desc,
337 			size_t count, fi_addr_t src_addr,
338 			uint64_t addr, uint64_t key,
339 			void *context, uint64_t flags);
340 
341 ssize_t psmx2_atomic_write_generic(
342 			struct fid_ep *ep,
343 			const void *buf,
344 			size_t count, void *desc,
345 			fi_addr_t dest_addr,
346 			uint64_t addr, uint64_t key,
347 			enum fi_datatype datatype,
348 			enum fi_op op, void *context,
349 			uint64_t flags);
350 
351 ssize_t psmx2_atomic_readwrite_generic(
352 			struct fid_ep *ep,
353 			const void *buf,
354 			size_t count, void *desc,
355 			void *result, void *result_desc,
356 			fi_addr_t dest_addr,
357 			uint64_t addr, uint64_t key,
358 			enum fi_datatype datatype,
359 			enum fi_op op, void *context,
360 			uint64_t flags);
361 
362 ssize_t psmx2_atomic_compwrite_generic(
363 			struct fid_ep *ep,
364 			const void *buf,
365 			size_t count, void *desc,
366 			const void *compare, void *compare_desc,
367 			void *result, void *result_desc,
368 			fi_addr_t dest_addr,
369 			uint64_t addr, uint64_t key,
370 			enum fi_datatype datatype,
371 			enum fi_op op, void *context,
372 			uint64_t flags);
373 
374 static inline
psmx2_process_trigger(struct psmx2_trx_ctxt * trx_ctxt,struct psmx2_trigger * trigger)375 int psmx2_process_trigger(struct psmx2_trx_ctxt *trx_ctxt,
376 			  struct psmx2_trigger *trigger)
377 {
378 	switch (trigger->op) {
379 	case PSMX2_TRIGGERED_SEND:
380 		psmx2_send_generic(trigger->send.ep,
381 				   trigger->send.buf,
382 				   trigger->send.len,
383 				   trigger->send.desc,
384 				   trigger->send.dest_addr,
385 				   trigger->send.context,
386 				   trigger->send.flags,
387 				   trigger->send.data);
388 		break;
389 	case PSMX2_TRIGGERED_SENDV:
390 		psmx2_sendv_generic(trigger->sendv.ep,
391 				    trigger->sendv.iov,
392 				    trigger->sendv.desc,
393 				    trigger->sendv.count,
394 				    trigger->sendv.dest_addr,
395 				    trigger->sendv.context,
396 				    trigger->sendv.flags,
397 				    trigger->sendv.data);
398 		break;
399 	case PSMX2_TRIGGERED_RECV:
400 		psmx2_recv_generic(trigger->recv.ep,
401 				   trigger->recv.buf,
402 				   trigger->recv.len,
403 				   trigger->recv.desc,
404 				   trigger->recv.src_addr,
405 				   trigger->recv.context,
406 				   trigger->recv.flags);
407 		break;
408 	case PSMX2_TRIGGERED_TSEND:
409 		psmx2_tagged_send_generic(trigger->tsend.ep,
410 					  trigger->tsend.buf,
411 					  trigger->tsend.len,
412 					  trigger->tsend.desc,
413 					  trigger->tsend.dest_addr,
414 					  trigger->tsend.tag,
415 					  trigger->tsend.context,
416 					  trigger->tsend.flags,
417 					  trigger->tsend.data);
418 		break;
419 	case PSMX2_TRIGGERED_TSENDV:
420 		psmx2_tagged_sendv_generic(trigger->tsendv.ep,
421 					   trigger->tsendv.iov,
422 					   trigger->tsendv.desc,
423 					   trigger->tsendv.count,
424 					   trigger->tsendv.dest_addr,
425 					   trigger->tsendv.tag,
426 					   trigger->tsendv.context,
427 					   trigger->tsendv.flags,
428 					   trigger->tsendv.data);
429 		break;
430 	case PSMX2_TRIGGERED_TRECV:
431 		psmx2_tagged_recv_generic(trigger->trecv.ep,
432 					  trigger->trecv.buf,
433 					  trigger->trecv.len,
434 					  trigger->trecv.desc,
435 					  trigger->trecv.src_addr,
436 					  trigger->trecv.tag,
437 					  trigger->trecv.ignore,
438 					  trigger->trecv.context,
439 					  trigger->trecv.flags);
440 		break;
441 	case PSMX2_TRIGGERED_WRITE:
442 		psmx2_write_generic(trigger->write.ep,
443 				    trigger->write.buf,
444 				    trigger->write.len,
445 				    trigger->write.desc,
446 				    trigger->write.dest_addr,
447 				    trigger->write.addr,
448 				    trigger->write.key,
449 				    trigger->write.context,
450 				    trigger->write.flags,
451 				    trigger->write.data);
452 		break;
453 
454 	case PSMX2_TRIGGERED_WRITEV:
455 		psmx2_writev_generic(trigger->writev.ep,
456 				     trigger->writev.iov,
457 				     trigger->writev.desc,
458 				     trigger->writev.count,
459 				     trigger->writev.dest_addr,
460 				     trigger->writev.addr,
461 				     trigger->writev.key,
462 				     trigger->writev.context,
463 				     trigger->writev.flags,
464 				     trigger->writev.data);
465 		break;
466 
467 	case PSMX2_TRIGGERED_READ:
468 		psmx2_read_generic(trigger->read.ep,
469 				   trigger->read.buf,
470 				   trigger->read.len,
471 				   trigger->read.desc,
472 				   trigger->read.src_addr,
473 				   trigger->read.addr,
474 				   trigger->read.key,
475 				   trigger->read.context,
476 				   trigger->read.flags);
477 		break;
478 
479 	case PSMX2_TRIGGERED_READV:
480 		psmx2_readv_generic(trigger->readv.ep,
481 				    trigger->readv.iov,
482 				    trigger->readv.desc,
483 				    trigger->readv.count,
484 				    trigger->readv.src_addr,
485 				    trigger->readv.addr,
486 				    trigger->readv.key,
487 				    trigger->readv.context,
488 				    trigger->readv.flags);
489 		break;
490 
491 	case PSMX2_TRIGGERED_ATOMIC_WRITE:
492 		psmx2_atomic_write_generic(
493 				trigger->atomic_write.ep,
494 				trigger->atomic_write.buf,
495 				trigger->atomic_write.count,
496 				trigger->atomic_write.desc,
497 				trigger->atomic_write.dest_addr,
498 				trigger->atomic_write.addr,
499 				trigger->atomic_write.key,
500 				trigger->atomic_write.datatype,
501 				trigger->atomic_write.atomic_op,
502 				trigger->atomic_write.context,
503 				trigger->atomic_write.flags);
504 		break;
505 
506 	case PSMX2_TRIGGERED_ATOMIC_READWRITE:
507 		psmx2_atomic_readwrite_generic(
508 				trigger->atomic_readwrite.ep,
509 				trigger->atomic_readwrite.buf,
510 				trigger->atomic_readwrite.count,
511 				trigger->atomic_readwrite.desc,
512 				trigger->atomic_readwrite.result,
513 				trigger->atomic_readwrite.result_desc,
514 				trigger->atomic_readwrite.dest_addr,
515 				trigger->atomic_readwrite.addr,
516 				trigger->atomic_readwrite.key,
517 				trigger->atomic_readwrite.datatype,
518 				trigger->atomic_readwrite.atomic_op,
519 				trigger->atomic_readwrite.context,
520 				trigger->atomic_readwrite.flags);
521 		break;
522 
523 	case PSMX2_TRIGGERED_ATOMIC_COMPWRITE:
524 		psmx2_atomic_compwrite_generic(
525 				trigger->atomic_compwrite.ep,
526 				trigger->atomic_compwrite.buf,
527 				trigger->atomic_compwrite.count,
528 				trigger->atomic_compwrite.desc,
529 				trigger->atomic_compwrite.compare,
530 				trigger->atomic_compwrite.compare_desc,
531 				trigger->atomic_compwrite.result,
532 				trigger->atomic_compwrite.result_desc,
533 				trigger->atomic_compwrite.dest_addr,
534 				trigger->atomic_compwrite.addr,
535 				trigger->atomic_compwrite.key,
536 				trigger->atomic_compwrite.datatype,
537 				trigger->atomic_compwrite.atomic_op,
538 				trigger->atomic_compwrite.context,
539 				trigger->atomic_compwrite.flags);
540 		break;
541 	default:
542 		FI_INFO(&psmx2_prov, FI_LOG_CQ,
543 			"%d unsupported op\n", trigger->op);
544 		break;
545 	}
546 
547 	free(trigger);
548 	return 0;
549 }
550 
551 static inline
psmx2_trigger_queue_trecv(struct fid_ep * ep,void * buf,size_t len,void * desc,fi_addr_t src_addr,uint64_t tag,uint64_t ignore,void * context,uint64_t flags)552 int psmx2_trigger_queue_trecv(struct fid_ep *ep, void *buf,
553 			      size_t len, void *desc,
554 			      fi_addr_t src_addr,
555 			      uint64_t tag, uint64_t ignore,
556 			      void *context, uint64_t flags)
557 {
558 	struct psmx2_trigger *trigger;
559 	struct fi_triggered_context *ctxt = context;
560 
561 	trigger = calloc(1, sizeof(*trigger));
562 	if (!trigger)
563 		return -FI_ENOMEM;
564 
565 	trigger->op = PSMX2_TRIGGERED_TRECV;
566 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
567 				     struct psmx2_fid_cntr, cntr);
568 	trigger->threshold = ctxt->trigger.threshold.threshold;
569 	trigger->trecv.ep = ep;
570 	trigger->trecv.buf = buf;
571 	trigger->trecv.len = len;
572 	trigger->trecv.desc = desc;
573 	trigger->trecv.src_addr = src_addr;
574 	trigger->trecv.tag = tag;
575 	trigger->trecv.ignore = ignore;
576 	trigger->trecv.context = context;
577 	trigger->trecv.flags = flags & ~FI_TRIGGER;
578 
579 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
580 	return 0;
581 }
582 
583 static inline
psmx2_trigger_queue_tsend(struct fid_ep * ep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,uint64_t tag,void * context,uint64_t flags,uint64_t data)584 int psmx2_trigger_queue_tsend(struct fid_ep *ep,
585 			      const void *buf, size_t len,
586 			      void *desc, fi_addr_t dest_addr,
587 			      uint64_t tag, void *context,
588 			      uint64_t flags, uint64_t data)
589 {
590 	struct psmx2_trigger *trigger;
591 	struct fi_triggered_context *ctxt = context;
592 
593 	trigger = calloc(1, sizeof(*trigger));
594 	if (!trigger)
595 		return -FI_ENOMEM;
596 
597 	trigger->op = PSMX2_TRIGGERED_TSEND;
598 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
599 				     struct psmx2_fid_cntr, cntr);
600 	trigger->threshold = ctxt->trigger.threshold.threshold;
601 	trigger->tsend.ep = ep;
602 	trigger->tsend.buf = buf;
603 	trigger->tsend.len = len;
604 	trigger->tsend.desc = desc;
605 	trigger->tsend.dest_addr = dest_addr;
606 	trigger->tsend.tag = tag;
607 	trigger->tsend.context = context;
608 	trigger->tsend.flags = flags & ~FI_TRIGGER;
609 	trigger->tsend.data = data;
610 
611 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
612 	return 0;
613 }
614 
615 static inline
psmx2_trigger_queue_tsendv(struct fid_ep * ep,const struct iovec * iov,void * desc,size_t count,fi_addr_t dest_addr,uint64_t tag,void * context,uint64_t flags,uint64_t data)616 int psmx2_trigger_queue_tsendv(struct fid_ep *ep,
617 			       const struct iovec *iov, void *desc,
618 			       size_t count, fi_addr_t dest_addr,
619 			       uint64_t tag, void *context,
620 			       uint64_t flags, uint64_t data)
621 {
622 	struct psmx2_trigger *trigger;
623 	struct fi_triggered_context *ctxt = context;
624 
625 	trigger = calloc(1, sizeof(*trigger));
626 	if (!trigger)
627 		return -FI_ENOMEM;
628 
629 	trigger->op = PSMX2_TRIGGERED_TSENDV;
630 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
631 				     struct psmx2_fid_cntr, cntr);
632 	trigger->threshold = ctxt->trigger.threshold.threshold;
633 	trigger->tsendv.ep = ep;
634 	trigger->tsendv.iov = iov;
635 	trigger->tsendv.desc = desc;
636 	trigger->tsendv.count = count;
637 	trigger->tsendv.dest_addr = dest_addr;
638 	trigger->tsendv.tag = tag;
639 	trigger->tsendv.context = context;
640 	trigger->tsendv.flags = flags & ~FI_TRIGGER;
641 	trigger->tsendv.data = data;
642 
643 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
644 	return 0;
645 }
646 
647 static inline
psmx2_trigger_queue_recv(struct fid_ep * ep,void * buf,size_t len,void * desc,fi_addr_t src_addr,void * context,uint64_t flags)648 int psmx2_trigger_queue_recv(struct fid_ep *ep, void *buf, size_t len,
649 			     void *desc, fi_addr_t src_addr, void *context,
650 			     uint64_t flags)
651 {
652 	struct psmx2_trigger *trigger;
653 	struct fi_triggered_context *ctxt = context;
654 
655 	trigger = calloc(1, sizeof(*trigger));
656 	if (!trigger)
657 		return -FI_ENOMEM;
658 
659 	trigger->op = PSMX2_TRIGGERED_RECV;
660 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
661 				     struct psmx2_fid_cntr, cntr);
662 	trigger->threshold = ctxt->trigger.threshold.threshold;
663 	trigger->recv.ep = ep;
664 	trigger->recv.buf = buf;
665 	trigger->recv.len = len;
666 	trigger->recv.desc = desc;
667 	trigger->recv.src_addr = src_addr;
668 	trigger->recv.context = context;
669 	trigger->recv.flags = flags & ~FI_TRIGGER;
670 
671 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
672 	return 0;
673 }
674 
675 static inline
psmx2_trigger_queue_send(struct fid_ep * ep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,void * context,uint64_t flags,uint64_t data)676 int psmx2_trigger_queue_send(struct fid_ep *ep, const void *buf, size_t len,
677 			     void *desc, fi_addr_t dest_addr, void *context,
678 			     uint64_t flags, uint64_t data)
679 {
680 	struct psmx2_trigger *trigger;
681 	struct fi_triggered_context *ctxt = context;
682 
683 	trigger = calloc(1, sizeof(*trigger));
684 	if (!trigger)
685 		return -FI_ENOMEM;
686 
687 	trigger->op = PSMX2_TRIGGERED_SEND;
688 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
689 				     struct psmx2_fid_cntr, cntr);
690 	trigger->threshold = ctxt->trigger.threshold.threshold;
691 	trigger->send.ep = ep;
692 	trigger->send.buf = buf;
693 	trigger->send.len = len;
694 	trigger->send.desc = desc;
695 	trigger->send.dest_addr = dest_addr;
696 	trigger->send.context = context;
697 	trigger->send.flags = flags & ~FI_TRIGGER;
698 	trigger->send.data = data;
699 
700 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
701 	return 0;
702 }
703 
704 static inline
psmx2_trigger_queue_sendv(struct fid_ep * ep,const struct iovec * iov,void * desc,size_t count,fi_addr_t dest_addr,void * context,uint64_t flags,uint64_t data)705 int psmx2_trigger_queue_sendv(struct fid_ep *ep, const struct iovec *iov,
706 			      void *desc, size_t count, fi_addr_t dest_addr,
707 			      void *context, uint64_t flags, uint64_t data)
708 {
709 	struct psmx2_trigger *trigger;
710 	struct fi_triggered_context *ctxt = context;
711 
712 	trigger = calloc(1, sizeof(*trigger));
713 	if (!trigger)
714 		return -FI_ENOMEM;
715 
716 	trigger->op = PSMX2_TRIGGERED_SENDV;
717 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
718 				     struct psmx2_fid_cntr, cntr);
719 	trigger->threshold = ctxt->trigger.threshold.threshold;
720 	trigger->sendv.ep = ep;
721 	trigger->sendv.iov = iov;
722 	trigger->sendv.desc = desc;
723 	trigger->sendv.count = count;
724 	trigger->sendv.dest_addr = dest_addr;
725 	trigger->sendv.context = context;
726 	trigger->sendv.flags = flags & ~FI_TRIGGER;
727 	trigger->sendv.data = data;
728 
729 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
730 	return 0;
731 }
732 
733 static inline
psmx2_trigger_queue_read(struct fid_ep * ep,void * buf,size_t len,void * desc,fi_addr_t src_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags)734 int psmx2_trigger_queue_read(struct fid_ep *ep, void *buf, size_t len,
735 			     void *desc, fi_addr_t src_addr,
736 			     uint64_t addr, uint64_t key, void *context,
737 			     uint64_t flags)
738 {
739 	struct psmx2_trigger *trigger;
740 	struct fi_triggered_context *ctxt = context;
741 
742 	trigger = calloc(1, sizeof(*trigger));
743 	if (!trigger)
744 		return -FI_ENOMEM;
745 
746 	trigger->op = PSMX2_TRIGGERED_READ;
747 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
748 				     struct psmx2_fid_cntr, cntr);
749 	trigger->threshold = ctxt->trigger.threshold.threshold;
750 	trigger->read.ep = ep;
751 	trigger->read.buf = buf;
752 	trigger->read.len = len;
753 	trigger->read.desc = desc;
754 	trigger->read.src_addr = src_addr;
755 	trigger->read.addr = addr;
756 	trigger->read.key = key;
757 	trigger->read.context = context;
758 	trigger->read.flags = flags & ~FI_TRIGGER;
759 
760 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
761 	return 0;
762 }
763 
764 static inline
psmx2_trigger_queue_readv(struct fid_ep * ep,const struct iovec * iov,void * desc,size_t count,fi_addr_t src_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags)765 int psmx2_trigger_queue_readv(struct fid_ep *ep, const struct iovec *iov,
766 			      void *desc, size_t count, fi_addr_t src_addr,
767 			      uint64_t addr, uint64_t key, void *context,
768 			      uint64_t flags)
769 {
770 	struct psmx2_trigger *trigger;
771 	struct fi_triggered_context *ctxt = context;
772 
773 	trigger = calloc(1, sizeof(*trigger));
774 	if (!trigger)
775 		return -FI_ENOMEM;
776 
777 	trigger->op = PSMX2_TRIGGERED_READV;
778 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
779 				     struct psmx2_fid_cntr, cntr);
780 	trigger->threshold = ctxt->trigger.threshold.threshold;
781 	trigger->readv.ep = ep;
782 	trigger->readv.iov = iov;
783 	trigger->readv.count = count;
784 	trigger->readv.desc = desc;
785 	trigger->readv.src_addr = src_addr;
786 	trigger->readv.addr = addr;
787 	trigger->readv.key = key;
788 	trigger->readv.context = context;
789 	trigger->readv.flags = flags & ~FI_TRIGGER;
790 
791 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
792 	return 0;
793 }
794 
795 static inline
psmx2_trigger_queue_write(struct fid_ep * ep,const void * buf,size_t len,void * desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags,uint64_t data)796 int psmx2_trigger_queue_write(struct fid_ep *ep, const void *buf, size_t len,
797 			      void *desc, fi_addr_t dest_addr,
798 			      uint64_t addr, uint64_t key, void *context,
799 			      uint64_t flags, uint64_t data)
800 {
801 	struct psmx2_trigger *trigger;
802 	struct fi_triggered_context *ctxt = context;
803 
804 	trigger = calloc(1, sizeof(*trigger));
805 	if (!trigger)
806 		return -FI_ENOMEM;
807 
808 	trigger->op = PSMX2_TRIGGERED_WRITE;
809 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
810 				     struct psmx2_fid_cntr, cntr);
811 	trigger->threshold = ctxt->trigger.threshold.threshold;
812 	trigger->write.ep = ep;
813 	trigger->write.buf = buf;
814 	trigger->write.len = len;
815 	trigger->write.desc = desc;
816 	trigger->write.dest_addr = dest_addr;
817 	trigger->write.addr = addr;
818 	trigger->write.key = key;
819 	trigger->write.context = context;
820 	trigger->write.flags = flags & ~FI_TRIGGER;
821 	trigger->write.data = data;
822 
823 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
824 	return 0;
825 }
826 
827 static inline
psmx2_trigger_queue_writev(struct fid_ep * ep,const struct iovec * iov,void ** desc,size_t count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,void * context,uint64_t flags,uint64_t data)828 int psmx2_trigger_queue_writev(struct fid_ep *ep, const struct iovec *iov,
829 			       void **desc, size_t count, fi_addr_t dest_addr,
830 			       uint64_t addr, uint64_t key, void *context,
831 			       uint64_t flags, uint64_t data)
832 {
833 	struct psmx2_trigger *trigger;
834 	struct fi_triggered_context *ctxt = context;
835 
836 	trigger = calloc(1, sizeof(*trigger));
837 	if (!trigger)
838 		return -FI_ENOMEM;
839 
840 	trigger->op = PSMX2_TRIGGERED_WRITEV;
841 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
842 				     struct psmx2_fid_cntr, cntr);
843 	trigger->threshold = ctxt->trigger.threshold.threshold;
844 	trigger->writev.ep = ep;
845 	trigger->writev.iov = iov;
846 	trigger->writev.count = count;
847 	trigger->writev.desc = desc;
848 	trigger->writev.dest_addr = dest_addr;
849 	trigger->writev.addr = addr;
850 	trigger->writev.key = key;
851 	trigger->writev.context = context;
852 	trigger->writev.flags = flags & ~FI_TRIGGER;
853 	trigger->writev.data = data;
854 
855 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
856 	return 0;
857 }
858 
859 static inline
psmx2_trigger_queue_atomic_write(struct fid_ep * ep,const void * buf,size_t count,void * desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)860 int psmx2_trigger_queue_atomic_write(struct fid_ep *ep,
861 				     const void *buf,
862 				     size_t count, void *desc,
863 				     fi_addr_t dest_addr,
864 				     uint64_t addr, uint64_t key,
865 				     enum fi_datatype datatype,
866 				     enum fi_op op, void *context,
867 				     uint64_t flags)
868 {
869 	struct psmx2_trigger *trigger;
870 	struct fi_triggered_context *ctxt = context;
871 
872 	trigger = calloc(1, sizeof(*trigger));
873 	if (!trigger)
874 		return -FI_ENOMEM;
875 
876 	trigger->op = PSMX2_TRIGGERED_ATOMIC_WRITE;
877 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
878 				     struct psmx2_fid_cntr, cntr);
879 	trigger->threshold = ctxt->trigger.threshold.threshold;
880 	trigger->atomic_write.ep = ep;
881 	trigger->atomic_write.buf = buf;
882 	trigger->atomic_write.count = count;
883 	trigger->atomic_write.desc = desc;
884 	trigger->atomic_write.dest_addr = dest_addr;
885 	trigger->atomic_write.addr = addr;
886 	trigger->atomic_write.key = key;
887 	trigger->atomic_write.datatype = datatype;
888 	trigger->atomic_write.atomic_op = op;
889 	trigger->atomic_write.context = context;
890 	trigger->atomic_write.flags = flags & ~FI_TRIGGER;
891 
892 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
893 	return 0;
894 }
895 
896 static inline
psmx2_trigger_queue_atomic_writev(struct fid_ep * ep,const struct fi_ioc * iov,void ** desc,size_t count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)897 int psmx2_trigger_queue_atomic_writev(struct fid_ep *ep,
898 				      const struct fi_ioc *iov,
899 				      void **desc, size_t count,
900 				      fi_addr_t dest_addr,
901 				      uint64_t addr, uint64_t key,
902 				      enum fi_datatype datatype,
903 				      enum fi_op op, void *context,
904 				      uint64_t flags)
905 {
906 	struct psmx2_trigger *trigger;
907 	struct fi_triggered_context *ctxt = context;
908 
909 	trigger = calloc(1, sizeof(*trigger));
910 	if (!trigger)
911 		return -FI_ENOMEM;
912 
913 	trigger->op = PSMX2_TRIGGERED_ATOMIC_WRITEV;
914 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
915 				     struct psmx2_fid_cntr, cntr);
916 	trigger->threshold = ctxt->trigger.threshold.threshold;
917 	trigger->atomic_writev.ep = ep;
918 	trigger->atomic_writev.iov = iov;
919 	trigger->atomic_writev.count = count;
920 	trigger->atomic_writev.desc = desc;
921 	trigger->atomic_writev.dest_addr = dest_addr;
922 	trigger->atomic_writev.addr = addr;
923 	trigger->atomic_writev.key = key;
924 	trigger->atomic_writev.datatype = datatype;
925 	trigger->atomic_writev.atomic_op = op;
926 	trigger->atomic_writev.context = context;
927 	trigger->atomic_writev.flags = flags & ~FI_TRIGGER;
928 
929 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
930 	return 0;
931 }
932 
933 static inline
psmx2_trigger_queue_atomic_readwrite(struct fid_ep * ep,const void * buf,size_t count,void * desc,void * result,void * result_desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)934 int psmx2_trigger_queue_atomic_readwrite(struct fid_ep *ep,
935 				         const void *buf,
936 				         size_t count, void *desc,
937 				         void *result, void *result_desc,
938 				         fi_addr_t dest_addr,
939 				         uint64_t addr, uint64_t key,
940 				         enum fi_datatype datatype,
941 				         enum fi_op op, void *context,
942 				         uint64_t flags)
943 {
944 	struct psmx2_trigger *trigger;
945 	struct fi_triggered_context *ctxt = context;
946 
947 	trigger = calloc(1, sizeof(*trigger));
948 	if (!trigger)
949 		return -FI_ENOMEM;
950 
951 	trigger->op = PSMX2_TRIGGERED_ATOMIC_READWRITE;
952 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
953 				     struct psmx2_fid_cntr, cntr);
954 	trigger->threshold = ctxt->trigger.threshold.threshold;
955 	trigger->atomic_readwrite.ep = ep;
956 	trigger->atomic_readwrite.buf = buf;
957 	trigger->atomic_readwrite.count = count;
958 	trigger->atomic_readwrite.desc = desc;
959 	trigger->atomic_readwrite.result = result;
960 	trigger->atomic_readwrite.result_desc = result_desc;
961 	trigger->atomic_readwrite.dest_addr = dest_addr;
962 	trigger->atomic_readwrite.addr = addr;
963 	trigger->atomic_readwrite.key = key;
964 	trigger->atomic_readwrite.datatype = datatype;
965 	trigger->atomic_readwrite.atomic_op = op;
966 	trigger->atomic_readwrite.context = context;
967 	trigger->atomic_readwrite.flags = flags & ~FI_TRIGGER;
968 
969 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
970 	return 0;
971 }
972 
973 static inline
psmx2_trigger_queue_atomic_readwritev(struct fid_ep * ep,const struct fi_ioc * iov,void ** desc,size_t count,struct fi_ioc * resultv,void ** result_desc,size_t result_count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)974 int psmx2_trigger_queue_atomic_readwritev(struct fid_ep *ep,
975 					  const struct fi_ioc *iov,
976 					  void **desc, size_t count,
977 					  struct fi_ioc *resultv,
978 					  void **result_desc,
979 					  size_t result_count,
980 					  fi_addr_t dest_addr,
981 					  uint64_t addr, uint64_t key,
982 					  enum fi_datatype datatype,
983 					  enum fi_op op, void *context,
984 					  uint64_t flags)
985 {
986 	struct psmx2_trigger *trigger;
987 	struct fi_triggered_context *ctxt = context;
988 
989 	trigger = calloc(1, sizeof(*trigger));
990 	if (!trigger)
991 		return -FI_ENOMEM;
992 
993 	trigger->op = PSMX2_TRIGGERED_ATOMIC_READWRITEV;
994 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
995 				     struct psmx2_fid_cntr, cntr);
996 	trigger->threshold = ctxt->trigger.threshold.threshold;
997 	trigger->atomic_readwritev.ep = ep;
998 	trigger->atomic_readwritev.iov = iov;
999 	trigger->atomic_readwritev.count = count;
1000 	trigger->atomic_readwritev.desc = desc;
1001 	trigger->atomic_readwritev.resultv = resultv;
1002 	trigger->atomic_readwritev.result_desc = result_desc;
1003 	trigger->atomic_readwritev.result_count = result_count;
1004 	trigger->atomic_readwritev.dest_addr = dest_addr;
1005 	trigger->atomic_readwritev.addr = addr;
1006 	trigger->atomic_readwritev.key = key;
1007 	trigger->atomic_readwritev.datatype = datatype;
1008 	trigger->atomic_readwritev.atomic_op = op;
1009 	trigger->atomic_readwritev.context = context;
1010 	trigger->atomic_readwritev.flags = flags & ~FI_TRIGGER;
1011 
1012 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
1013 	return 0;
1014 }
1015 
1016 static inline
psmx2_trigger_queue_atomic_compwrite(struct fid_ep * ep,const void * buf,size_t count,void * desc,const void * compare,void * compare_desc,void * result,void * result_desc,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)1017 int psmx2_trigger_queue_atomic_compwrite(struct fid_ep *ep,
1018 				         const void *buf,
1019 				         size_t count, void *desc,
1020 				         const void *compare, void *compare_desc,
1021 				         void *result, void *result_desc,
1022 				         fi_addr_t dest_addr,
1023 				         uint64_t addr, uint64_t key,
1024 				         enum fi_datatype datatype,
1025 				         enum fi_op op, void *context,
1026 				         uint64_t flags)
1027 {
1028 	struct psmx2_trigger *trigger;
1029 	struct fi_triggered_context *ctxt = context;
1030 
1031 	trigger = calloc(1, sizeof(*trigger));
1032 	if (!trigger)
1033 		return -FI_ENOMEM;
1034 
1035 	trigger->op = PSMX2_TRIGGERED_ATOMIC_COMPWRITE;
1036 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
1037 				     struct psmx2_fid_cntr, cntr);
1038 	trigger->threshold = ctxt->trigger.threshold.threshold;
1039 	trigger->atomic_compwrite.ep = ep;
1040 	trigger->atomic_compwrite.buf = buf;
1041 	trigger->atomic_compwrite.count = count;
1042 	trigger->atomic_compwrite.desc = desc;
1043 	trigger->atomic_compwrite.compare = compare;
1044 	trigger->atomic_compwrite.compare_desc = compare_desc;
1045 	trigger->atomic_compwrite.result = result;
1046 	trigger->atomic_compwrite.result_desc = result_desc;
1047 	trigger->atomic_compwrite.dest_addr = dest_addr;
1048 	trigger->atomic_compwrite.addr = addr;
1049 	trigger->atomic_compwrite.key = key;
1050 	trigger->atomic_compwrite.datatype = datatype;
1051 	trigger->atomic_compwrite.atomic_op = op;
1052 	trigger->atomic_compwrite.context = context;
1053 	trigger->atomic_compwrite.flags = flags & ~FI_TRIGGER;
1054 
1055 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
1056 	return 0;
1057 }
1058 
1059 static inline
psmx2_trigger_queue_atomic_compwritev(struct fid_ep * ep,const struct fi_ioc * iov,void ** desc,size_t count,const struct fi_ioc * comparev,void ** compare_desc,size_t compare_count,struct fi_ioc * resultv,void ** result_desc,size_t result_count,fi_addr_t dest_addr,uint64_t addr,uint64_t key,enum fi_datatype datatype,enum fi_op op,void * context,uint64_t flags)1060 int psmx2_trigger_queue_atomic_compwritev(struct fid_ep *ep,
1061 					  const struct fi_ioc *iov,
1062 					  void **desc, size_t count,
1063 					  const struct fi_ioc *comparev,
1064 					  void **compare_desc,
1065 					  size_t compare_count,
1066 					  struct fi_ioc *resultv,
1067 					  void **result_desc,
1068 					  size_t result_count,
1069 					  fi_addr_t dest_addr,
1070 					  uint64_t addr, uint64_t key,
1071 					  enum fi_datatype datatype,
1072 					  enum fi_op op, void *context,
1073 					  uint64_t flags)
1074 {
1075 	struct psmx2_trigger *trigger;
1076 	struct fi_triggered_context *ctxt = context;
1077 
1078 	trigger = calloc(1, sizeof(*trigger));
1079 	if (!trigger)
1080 		return -FI_ENOMEM;
1081 
1082 	trigger->op = PSMX2_TRIGGERED_ATOMIC_COMPWRITEV;
1083 	trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
1084 				     struct psmx2_fid_cntr, cntr);
1085 	trigger->threshold = ctxt->trigger.threshold.threshold;
1086 	trigger->atomic_compwritev.ep = ep;
1087 	trigger->atomic_compwritev.iov = iov;
1088 	trigger->atomic_compwritev.desc = desc;
1089 	trigger->atomic_compwritev.count = count;
1090 	trigger->atomic_compwritev.comparev = comparev;
1091 	trigger->atomic_compwritev.compare_desc = compare_desc;
1092 	trigger->atomic_compwritev.compare_count = compare_count;
1093 	trigger->atomic_compwritev.resultv = resultv;
1094 	trigger->atomic_compwritev.result_desc = result_desc;
1095 	trigger->atomic_compwritev.result_count = result_count;
1096 	trigger->atomic_compwritev.dest_addr = dest_addr;
1097 	trigger->atomic_compwritev.addr = addr;
1098 	trigger->atomic_compwritev.key = key;
1099 	trigger->atomic_compwritev.datatype = datatype;
1100 	trigger->atomic_compwritev.atomic_op = op;
1101 	trigger->atomic_compwritev.context = context;
1102 	trigger->atomic_compwritev.flags = flags & ~FI_TRIGGER;
1103 
1104 	psmx2_cntr_add_trigger(trigger->cntr, trigger);
1105 	return 0;
1106 }
1107 
1108 #ifdef __cplusplus
1109 }
1110 #endif
1111 
1112 #endif
1113 
1114