1 /*
2 * Copyright (c) 2013-2017 Intel Corporation. All rights reserved.
3 * Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #ifndef _FI_PSM_H
35 #define _FI_PSM_H
36
37 #ifdef __cplusplus
38 extern "C" {
39 #endif
40
41 #include "config.h"
42
43 #include <assert.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <stdarg.h>
47 #include <string.h>
48 #include <errno.h>
49 #include <unistd.h>
50 #include <fcntl.h>
51 #include <inttypes.h>
52 #include <pthread.h>
53 #include <sys/types.h>
54 #include <sys/socket.h>
55 #include <netdb.h>
56 #include <complex.h>
57 #include <rdma/fabric.h>
58 #include <rdma/fi_domain.h>
59 #include <rdma/fi_endpoint.h>
60 #include <rdma/fi_tagged.h>
61 #include <rdma/fi_rma.h>
62 #include <rdma/fi_atomic.h>
63 #include <rdma/fi_trigger.h>
64 #include <rdma/fi_cm.h>
65 #include <rdma/fi_errno.h>
66 #include "ofi.h"
67 #include "ofi_atomic.h"
68 #include "ofi_enosys.h"
69 #include "ofi_list.h"
70 #include "ofi_util.h"
71 #include "ofi_file.h"
72 #include "rbtree.h"
73 #include "version.h"
74
75 extern struct fi_provider psmx_prov;
76
77 extern int psmx_am_compat_mode;
78
79
80 #define PSMX_OP_FLAGS (FI_INJECT | FI_MULTI_RECV | FI_COMPLETION | \
81 FI_TRIGGER | FI_INJECT_COMPLETE | \
82 FI_TRANSMIT_COMPLETE | FI_DELIVERY_COMPLETE)
83
84 #define PSMX_CAPS (FI_TAGGED | FI_MSG | FI_ATOMICS | \
85 FI_RMA | FI_MULTI_RECV | \
86 FI_READ | FI_WRITE | FI_SEND | FI_RECV | \
87 FI_REMOTE_READ | FI_REMOTE_WRITE | \
88 FI_TRIGGER | FI_RMA_EVENT)
89
90 #define PSMX_CAPS2 ((PSMX_CAPS | FI_DIRECTED_RECV) & ~FI_TAGGED)
91
92 #define PSMX_SUB_CAPS (FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE | \
93 FI_SEND | FI_RECV)
94
95 #define PSMX_DOM_CAPS (FI_LOCAL_COMM | FI_REMOTE_COMM)
96
97 #define PSMX_MAX_MSG_SIZE ((0x1ULL << 32) - 1)
98 #define PSMX_INJECT_SIZE (64)
99 #define PSMX_MSG_ORDER (FI_ORDER_SAS | OFI_ORDER_RAR_SET | OFI_ORDER_RAW_SET | \
100 OFI_ORDER_WAR_SET | OFI_ORDER_WAW_SET)
101 #define PSMX_COMP_ORDER FI_ORDER_NONE
102
103 #define PSMX_MSG_BIT (0x1ULL << 63)
104 #define PSMX_RMA_BIT (0x1ULL << 62)
105
106 /* Bits 60 .. 63 of the flag are provider specific */
107 #define PSMX_NO_COMPLETION (1ULL << 60)
108
109 enum psmx_context_type {
110 PSMX_NOCOMP_SEND_CONTEXT = 1,
111 PSMX_NOCOMP_RECV_CONTEXT,
112 PSMX_NOCOMP_WRITE_CONTEXT,
113 PSMX_NOCOMP_READ_CONTEXT,
114 PSMX_SEND_CONTEXT,
115 PSMX_RECV_CONTEXT,
116 PSMX_MULTI_RECV_CONTEXT,
117 PSMX_TSEND_CONTEXT,
118 PSMX_TRECV_CONTEXT,
119 PSMX_WRITE_CONTEXT,
120 PSMX_READ_CONTEXT,
121 PSMX_REMOTE_WRITE_CONTEXT,
122 PSMX_REMOTE_READ_CONTEXT,
123 };
124
125 union psmx_pi {
126 void *p;
127 int i;
128 };
129
130 #define PSMX_CTXT_REQ(fi_context) ((fi_context)->internal[0])
131 #define PSMX_CTXT_TYPE(fi_context) (((union psmx_pi *)&(fi_context)->internal[1])->i)
132 #define PSMX_CTXT_USER(fi_context) ((fi_context)->internal[2])
133 #define PSMX_CTXT_EP(fi_context) ((fi_context)->internal[3])
134
135 #define PSMX_AM_RMA_HANDLER 0
136 #define PSMX_AM_MSG_HANDLER 1
137 #define PSMX_AM_ATOMIC_HANDLER 2
138 #define PSMX_AM_CHUNK_SIZE 2032 /* The maximum that's actually working:
139 * 2032 for inter-node, 2072 for intra-node.
140 */
141 #define PSMX_RMA_ORDER_SIZE PSMX_AM_CHUNK_SIZE
142
143 #define PSMX_AM_OP_MASK 0x0000FFFF
144 #define PSMX_AM_FLAG_MASK 0xFFFF0000
145 #define PSMX_AM_EOM 0x40000000
146 #define PSMX_AM_DATA 0x20000000
147 #define PSMX_AM_FORCE_ACK 0x10000000
148
149 enum {
150 PSMX_AM_REQ_WRITE = 1,
151 PSMX_AM_REQ_WRITE_LONG,
152 PSMX_AM_REP_WRITE,
153 PSMX_AM_REQ_READ,
154 PSMX_AM_REQ_READ_LONG,
155 PSMX_AM_REP_READ,
156 PSMX_AM_REQ_SEND,
157 PSMX_AM_REP_SEND,
158 PSMX_AM_REQ_ATOMIC_WRITE,
159 PSMX_AM_REP_ATOMIC_WRITE,
160 PSMX_AM_REQ_ATOMIC_READWRITE,
161 PSMX_AM_REP_ATOMIC_READWRITE,
162 PSMX_AM_REQ_ATOMIC_COMPWRITE,
163 PSMX_AM_REP_ATOMIC_COMPWRITE,
164 };
165
166 struct psmx_am_request {
167 int op;
168 union {
169 struct {
170 uint8_t *buf;
171 size_t len;
172 uint64_t addr;
173 uint64_t key;
174 void *context;
175 void *peer_context;
176 void *peer_addr;
177 uint64_t data;
178 } write;
179 struct {
180 uint8_t *buf;
181 size_t len;
182 uint64_t addr;
183 uint64_t key;
184 void *context;
185 void *peer_addr;
186 size_t len_read;
187 } read;
188 struct {
189 uint8_t *buf;
190 size_t len;
191 void *context;
192 void *peer_context;
193 volatile int peer_ready;
194 void *dest_addr;
195 size_t len_sent;
196 } send;
197 struct {
198 uint8_t *buf;
199 size_t len;
200 void *context;
201 void *src_addr;
202 size_t len_received;
203 } recv;
204 struct {
205 uint8_t *buf;
206 size_t len;
207 uint64_t addr;
208 uint64_t key;
209 void *context;
210 uint8_t *result;
211 } atomic;
212 };
213 uint64_t cq_flags;
214 struct fi_context fi_context;
215 struct psmx_fid_ep *ep;
216 int no_event;
217 int error;
218 struct slist_entry list_entry;
219 };
220
221 struct psmx_unexp {
222 psm_epaddr_t sender_addr;
223 uint64_t sender_context;
224 uint32_t len_received;
225 uint32_t done;
226 struct slist_entry list_entry;
227 char buf[0];
228 };
229
230 struct psmx_req_queue {
231 fastlock_t lock;
232 struct slist list;
233 };
234
235 struct psmx_multi_recv {
236 uint64_t tag;
237 uint64_t tagsel;
238 uint8_t *buf;
239 size_t len;
240 size_t offset;
241 int min_buf_size;
242 int flag;
243 void *context;
244 };
245
246 struct psmx_fid_fabric {
247 struct util_fabric util_fabric;
248 struct psmx_fid_domain *active_domain;
249 psm_uuid_t uuid;
250 struct util_ns name_server;
251 };
252
253 struct psmx_fid_domain {
254 struct util_domain util_domain;
255 struct psmx_fid_fabric *fabric;
256 psm_ep_t psm_ep;
257 psm_epid_t psm_epid;
258 psm_mq_t psm_mq;
259 struct psmx_fid_ep *tagged_ep;
260 struct psmx_fid_ep *msg_ep;
261 struct psmx_fid_ep *rma_ep;
262 struct psmx_fid_ep *atomics_ep;
263 uint64_t mode;
264 uint64_t caps;
265
266 enum fi_mr_mode mr_mode;
267 fastlock_t mr_lock;
268 uint64_t mr_reserved_key;
269 RbtHandle mr_map;
270
271 int am_initialized;
272
273 /* incoming req queue for AM based RMA request. */
274 struct psmx_req_queue rma_queue;
275
276 /* send queue for AM based messages. */
277 struct psmx_req_queue send_queue;
278
279 /* recv queue for AM based messages. */
280 struct psmx_req_queue recv_queue;
281 struct psmx_req_queue unexp_queue;
282
283 /* triggered operations that are ready to be processed */
284 struct psmx_req_queue trigger_queue;
285
286 /* certain bits in the tag space can be reserved for non tag-matching
287 * purpose. The tag-matching functions automatically treat these bits
288 * as 0. This field is a bit mask, with reserved bits valued as "1".
289 */
290 uint64_t reserved_tag_bits;
291
292 /* lock to prevent the sequence of psm_mq_ipeek and psm_mq_test be
293 * interleaved in a multithreaded environment.
294 */
295 fastlock_t poll_lock;
296
297 int progress_thread_enabled;
298 pthread_t progress_thread;
299 };
300
301 #define PSMX_DEFAULT_UNIT (-1)
302 #define PSMX_DEFAULT_PORT 0
303 #define PSMX_ANY_SERVICE 0
304
305 struct psmx_src_name {
306 uint16_t signature; /* 0xFFFF, different from any valid epid */
307 int8_t unit; /* start from 0. -1 means any */
308 uint8_t port; /* start from 1. 0 means any */
309 uint32_t service; /* 0 means any */
310 };
311
312 struct psmx_cq_event {
313 union {
314 struct fi_cq_entry context;
315 struct fi_cq_msg_entry msg;
316 struct fi_cq_data_entry data;
317 struct fi_cq_tagged_entry tagged;
318 struct fi_cq_err_entry err;
319 } cqe;
320 int error;
321 uint64_t source;
322 struct slist_entry list_entry;
323 };
324
325 struct psmx_fid_cq {
326 struct fid_cq cq;
327 struct psmx_fid_domain *domain;
328 int format;
329 int entry_size;
330 size_t event_count;
331 struct slist event_queue;
332 struct slist free_list;
333 fastlock_t lock;
334 struct psmx_cq_event *pending_error;
335 struct util_wait *wait;
336 int wait_cond;
337 int wait_is_local;
338 };
339
340 enum psmx_triggered_op {
341 PSMX_TRIGGERED_SEND,
342 PSMX_TRIGGERED_RECV,
343 PSMX_TRIGGERED_TSEND,
344 PSMX_TRIGGERED_TRECV,
345 PSMX_TRIGGERED_WRITE,
346 PSMX_TRIGGERED_READ,
347 PSMX_TRIGGERED_ATOMIC_WRITE,
348 PSMX_TRIGGERED_ATOMIC_READWRITE,
349 PSMX_TRIGGERED_ATOMIC_COMPWRITE,
350 };
351
352 struct psmx_trigger {
353 enum psmx_triggered_op op;
354 struct psmx_fid_cntr *cntr;
355 size_t threshold;
356 union {
357 struct {
358 struct fid_ep *ep;
359 const void *buf;
360 size_t len;
361 void *desc;
362 fi_addr_t dest_addr;
363 void *context;
364 uint64_t flags;
365 } send;
366 struct {
367 struct fid_ep *ep;
368 void *buf;
369 size_t len;
370 void *desc;
371 fi_addr_t src_addr;
372 void *context;
373 uint64_t flags;
374 } recv;
375 struct {
376 struct fid_ep *ep;
377 const void *buf;
378 size_t len;
379 void *desc;
380 fi_addr_t dest_addr;
381 uint64_t tag;
382 void *context;
383 uint64_t flags;
384 } tsend;
385 struct {
386 struct fid_ep *ep;
387 void *buf;
388 size_t len;
389 void *desc;
390 fi_addr_t src_addr;
391 uint64_t tag;
392 uint64_t ignore;
393 void *context;
394 uint64_t flags;
395 } trecv;
396 struct {
397 struct fid_ep *ep;
398 const void *buf;
399 size_t len;
400 void *desc;
401 fi_addr_t dest_addr;
402 uint64_t addr;
403 uint64_t key;
404 void *context;
405 uint64_t flags;
406 uint64_t data;
407 } write;
408 struct {
409 struct fid_ep *ep;
410 void *buf;
411 size_t len;
412 void *desc;
413 fi_addr_t src_addr;
414 uint64_t addr;
415 uint64_t key;
416 void *context;
417 uint64_t flags;
418 } read;
419 struct {
420 struct fid_ep *ep;
421 const void *buf;
422 size_t count;
423 void *desc;
424 fi_addr_t dest_addr;
425 uint64_t addr;
426 uint64_t key;
427 enum fi_datatype datatype;
428 enum fi_op atomic_op;
429 void *context;
430 uint64_t flags;
431 } atomic_write;
432 struct {
433 struct fid_ep *ep;
434 const void *buf;
435 size_t count;
436 void *desc;
437 void *result;
438 void *result_desc;
439 fi_addr_t dest_addr;
440 uint64_t addr;
441 uint64_t key;
442 enum fi_datatype datatype;
443 enum fi_op atomic_op;
444 void *context;
445 uint64_t flags;
446 } atomic_readwrite;
447 struct {
448 struct fid_ep *ep;
449 const void *buf;
450 size_t count;
451 void *desc;
452 const void *compare;
453 void *compare_desc;
454 void *result;
455 void *result_desc;
456 fi_addr_t dest_addr;
457 uint64_t addr;
458 uint64_t key;
459 enum fi_datatype datatype;
460 enum fi_op atomic_op;
461 void *context;
462 uint64_t flags;
463 } atomic_compwrite;
464 };
465 struct psmx_trigger *next; /* used for randomly accessed trigger list */
466 struct slist_entry list_entry; /* used for ready-to-fire trigger queue */
467 };
468
469 struct psmx_fid_cntr {
470 union {
471 struct fid_cntr cntr;
472 struct util_cntr util_cntr; /* for util_poll_run */
473 };
474 struct psmx_fid_domain *domain;
475 int events;
476 uint64_t flags;
477 ofi_atomic64_t counter;
478 ofi_atomic64_t error_counter;
479 struct util_wait *wait;
480 int wait_is_local;
481 struct psmx_trigger *trigger;
482 pthread_mutex_t trigger_lock;
483 };
484
485 struct psmx_fid_av {
486 struct fid_av av;
487 struct psmx_fid_domain *domain;
488 struct fid_eq *eq;
489 int type;
490 uint64_t flags;
491 size_t addrlen;
492 size_t count;
493 size_t last;
494 psm_epid_t *psm_epids;
495 psm_epaddr_t *psm_epaddrs;
496 };
497
498 struct psmx_fid_ep {
499 struct fid_ep ep;
500 struct psmx_fid_ep *base_ep;
501 struct psmx_fid_domain *domain;
502 struct psmx_fid_av *av;
503 struct psmx_fid_cq *send_cq;
504 struct psmx_fid_cq *recv_cq;
505 struct psmx_fid_cntr *send_cntr;
506 struct psmx_fid_cntr *recv_cntr;
507 struct psmx_fid_cntr *write_cntr;
508 struct psmx_fid_cntr *read_cntr;
509 struct psmx_fid_cntr *remote_write_cntr;
510 struct psmx_fid_cntr *remote_read_cntr;
511 unsigned send_selective_completion:1;
512 unsigned recv_selective_completion:1;
513 unsigned enabled:1;
514 uint64_t tx_flags;
515 uint64_t rx_flags;
516 uint64_t caps;
517 ofi_atomic32_t ref;
518 struct fi_context nocomp_send_context;
519 struct fi_context nocomp_recv_context;
520 size_t min_multi_recv;
521 int service;
522 };
523
524 struct psmx_fid_stx {
525 struct fid_stx stx;
526 struct psmx_fid_domain *domain;
527 };
528
529 struct psmx_fid_mr {
530 struct fid_mr mr;
531 struct psmx_fid_domain *domain;
532 struct psmx_fid_cntr *cntr;
533 uint64_t access;
534 uint64_t flags;
535 uint64_t offset;
536 size_t iov_count;
537 struct iovec iov[0]; /* must be the last field */
538 };
539
540 struct psmx_epaddr_context {
541 struct psmx_fid_domain *domain;
542 psm_epid_t epid;
543 };
544
545 struct psmx_env {
546 int name_server;
547 int am_msg;
548 int tagged_rma;
549 char *uuid;
550 int delay;
551 int timeout;
552 int prog_thread;
553 int prog_interval;
554 char *prog_affinity;
555 };
556
557 extern struct fi_ops_mr psmx_mr_ops;
558 extern struct fi_ops_cm psmx_cm_ops;
559 extern struct fi_ops_tagged psmx_tagged_ops;
560 extern struct fi_ops_tagged psmx_tagged_ops_no_flag_av_map;
561 extern struct fi_ops_tagged psmx_tagged_ops_no_flag_av_table;
562 extern struct fi_ops_tagged psmx_tagged_ops_no_event_av_map;
563 extern struct fi_ops_tagged psmx_tagged_ops_no_event_av_table;
564 extern struct fi_ops_tagged psmx_tagged_ops_no_send_event_av_map;
565 extern struct fi_ops_tagged psmx_tagged_ops_no_send_event_av_table;
566 extern struct fi_ops_tagged psmx_tagged_ops_no_recv_event_av_map;
567 extern struct fi_ops_tagged psmx_tagged_ops_no_recv_event_av_table;
568 extern struct fi_ops_msg psmx_msg_ops;
569 extern struct fi_ops_msg psmx_msg2_ops;
570 extern struct fi_ops_rma psmx_rma_ops;
571 extern struct fi_ops_atomic psmx_atomic_ops;
572 extern struct psm_am_parameters psmx_am_param;
573 extern struct psmx_env psmx_env;
574 extern struct psmx_fid_fabric *psmx_active_fabric;
575
576 int psmx_fabric(struct fi_fabric_attr *attr,
577 struct fid_fabric **fabric, void *context);
578 int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
579 struct fid_domain **domain, void *context);
580 int psmx_wait_open(struct fid_fabric *fabric, struct fi_wait_attr *attr,
581 struct fid_wait **waitset);
582 int psmx_wait_trywait(struct fid_fabric *fabric, struct fid **fids,
583 int count);
584 int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
585 struct fid_ep **ep, void *context);
586 int psmx_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr,
587 struct fid_stx **stx, void *context);
588 int psmx_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
589 struct fid_cq **cq, void *context);
590 int psmx_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
591 struct fid_av **av, void *context);
592 int psmx_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
593 struct fid_cntr **cntr, void *context);
594 int psmx_query_atomic(struct fid_domain *doamin, enum fi_datatype datatype,
595 enum fi_op op, struct fi_atomic_attr *attr,
596 uint64_t flags);
597
psmx_fabric_acquire(struct psmx_fid_fabric * fabric)598 static inline void psmx_fabric_acquire(struct psmx_fid_fabric *fabric)
599 {
600 ofi_atomic_inc32(&fabric->util_fabric.ref);
601 }
602
psmx_fabric_release(struct psmx_fid_fabric * fabric)603 static inline void psmx_fabric_release(struct psmx_fid_fabric *fabric)
604 {
605 ofi_atomic_dec32(&fabric->util_fabric.ref);
606 }
607
psmx_domain_acquire(struct psmx_fid_domain * domain)608 static inline void psmx_domain_acquire(struct psmx_fid_domain *domain)
609 {
610 ofi_atomic_inc32(&domain->util_domain.ref);
611 }
612
psmx_domain_release(struct psmx_fid_domain * domain)613 static inline void psmx_domain_release(struct psmx_fid_domain *domain)
614 {
615 ofi_atomic_dec32(&domain->util_domain.ref);
616 }
617
618 int psmx_domain_check_features(struct psmx_fid_domain *domain, uint64_t ep_caps);
619 int psmx_domain_enable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep);
620 void psmx_domain_disable_ep(struct psmx_fid_domain *domain, struct psmx_fid_ep *ep);
621
622 static inline
psmx_ns_service_cmp(void * svc1,void * svc2)623 int psmx_ns_service_cmp(void *svc1, void *svc2)
624 {
625 int service1 = *(int *)svc1, service2 = *(int *)svc2;
626 if (service1 == PSMX_ANY_SERVICE ||
627 service2 == PSMX_ANY_SERVICE)
628 return 0;
629 return (service1 < service2) ? -1 : (service1 > service2);
630 }
631
632 static inline
psmx_ns_is_service_wildcard(void * svc)633 int psmx_ns_is_service_wildcard(void *svc)
634 {
635 return (*(int *)svc == PSMX_ANY_SERVICE);
636 }
637
638 void psmx_get_uuid(psm_uuid_t uuid);
639 int psmx_uuid_to_port(psm_uuid_t uuid);
640 char *psmx_uuid_to_string(psm_uuid_t uuid);
641 int psmx_errno(int err);
642 int psmx_epid_to_epaddr(struct psmx_fid_domain *domain,
643 psm_epid_t epid, psm_epaddr_t *epaddr);
644 void psmx_query_mpi(void);
645
646 void psmx_cq_enqueue_event(struct psmx_fid_cq *cq, struct psmx_cq_event *event);
647 struct psmx_cq_event *psmx_cq_create_event(struct psmx_fid_cq *cq,
648 void *op_context, void *buf,
649 uint64_t flags, size_t len,
650 uint64_t data, uint64_t tag,
651 size_t olen, int err);
652 int psmx_cq_poll_mq(struct psmx_fid_cq *cq, struct psmx_fid_domain *domain,
653 struct psmx_cq_event *event, int count, fi_addr_t *src_addr);
654
655 int psmx_am_init(struct psmx_fid_domain *domain);
656 int psmx_am_fini(struct psmx_fid_domain *domain);
657 int psmx_am_progress(struct psmx_fid_domain *domain);
658 int psmx_am_process_send(struct psmx_fid_domain *domain,
659 struct psmx_am_request *req);
660 int psmx_am_process_rma(struct psmx_fid_domain *domain,
661 struct psmx_am_request *req);
662 int psmx_process_trigger(struct psmx_fid_domain *domain,
663 struct psmx_trigger *trigger);
664 int psmx_am_msg_handler(psm_am_token_t token, psm_epaddr_t epaddr,
665 psm_amarg_t *args, int nargs, void *src, uint32_t len);
666 int psmx_am_rma_handler(psm_am_token_t token, psm_epaddr_t epaddr,
667 psm_amarg_t *args, int nargs, void *src, uint32_t len);
668 int psmx_am_atomic_handler(psm_am_token_t token, psm_epaddr_t epaddr,
669 psm_amarg_t *args, int nargs, void *src, uint32_t len);
670 void psmx_atomic_init(void);
671 void psmx_atomic_fini(void);
672
673 void psmx_am_ack_rma(struct psmx_am_request *req);
674
675 struct psmx_fid_mr *psmx_mr_get(struct psmx_fid_domain *domain, uint64_t key);
676 int psmx_mr_validate(struct psmx_fid_mr *mr, uint64_t addr, size_t len, uint64_t access);
677 void psmx_cntr_check_trigger(struct psmx_fid_cntr *cntr);
678 void psmx_cntr_add_trigger(struct psmx_fid_cntr *cntr, struct psmx_trigger *trigger);
679
psmx_cntr_inc(struct psmx_fid_cntr * cntr)680 static inline void psmx_cntr_inc(struct psmx_fid_cntr *cntr)
681 {
682 ofi_atomic_inc64(&cntr->counter);
683 psmx_cntr_check_trigger(cntr);
684 if (cntr->wait)
685 cntr->wait->signal(cntr->wait);
686 }
687
psmx_progress(struct psmx_fid_domain * domain)688 static inline void psmx_progress(struct psmx_fid_domain *domain)
689 {
690 if (domain) {
691 psmx_cq_poll_mq(NULL, domain, NULL, 0, NULL);
692 if (domain->am_initialized)
693 psmx_am_progress(domain);
694 }
695 }
696
697 ssize_t _psmx_send(struct fid_ep *ep, const void *buf, size_t len,
698 void *desc, fi_addr_t dest_addr, void *context,
699 uint64_t flags);
700 ssize_t _psmx_recv(struct fid_ep *ep, void *buf, size_t len,
701 void *desc, fi_addr_t src_addr, void *context,
702 uint64_t flags);
703 ssize_t _psmx_tagged_send(struct fid_ep *ep, const void *buf, size_t len,
704 void *desc, fi_addr_t dest_addr, uint64_t tag,
705 void *context, uint64_t flags);
706 ssize_t _psmx_tagged_recv(struct fid_ep *ep, void *buf, size_t len,
707 void *desc, fi_addr_t src_addr, uint64_t tag,
708 uint64_t ignore, void *context, uint64_t flags);
709 ssize_t _psmx_write(struct fid_ep *ep, const void *buf, size_t len,
710 void *desc, fi_addr_t dest_addr,
711 uint64_t addr, uint64_t key, void *context,
712 uint64_t flags, uint64_t data);
713 ssize_t _psmx_read(struct fid_ep *ep, void *buf, size_t len,
714 void *desc, fi_addr_t src_addr,
715 uint64_t addr, uint64_t key, void *context,
716 uint64_t flags);
717 ssize_t _psmx_atomic_write(struct fid_ep *ep,
718 const void *buf,
719 size_t count, void *desc,
720 fi_addr_t dest_addr,
721 uint64_t addr, uint64_t key,
722 enum fi_datatype datatype,
723 enum fi_op op, void *context,
724 uint64_t flags);
725 ssize_t _psmx_atomic_readwrite(struct fid_ep *ep,
726 const void *buf,
727 size_t count, void *desc,
728 void *result, void *result_desc,
729 fi_addr_t dest_addr,
730 uint64_t addr, uint64_t key,
731 enum fi_datatype datatype,
732 enum fi_op op, void *context,
733 uint64_t flags);
734 ssize_t _psmx_atomic_compwrite(struct fid_ep *ep,
735 const void *buf,
736 size_t count, void *desc,
737 const void *compare, void *compare_desc,
738 void *result, void *result_desc,
739 fi_addr_t dest_addr,
740 uint64_t addr, uint64_t key,
741 enum fi_datatype datatype,
742 enum fi_op op, void *context,
743 uint64_t flags);
744
745 #ifdef __cplusplus
746 }
747 #endif
748
749 #endif
750
751