1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
4 * reserved.
5 * Copyright (c) 2015 Research Organization for Information Science
6 * and Technology (RIST). All rights reserved.
7 *
8 * $COPYRIGHT$
9 *
10 * Additional copyrights may follow
11 *
12 * $HEADER$
13 */
14
15 #include "btl_openib.h"
16 #include "btl_openib_endpoint.h"
17 #include "btl_openib_proc.h"
18 #include "btl_openib_xrc.h"
19
20 #if HAVE_DECL_IBV_ATOMIC_HCA
21
mca_btl_openib_atomic_internal(struct mca_btl_base_module_t * btl,struct mca_btl_base_endpoint_t * endpoint,void * local_address,uint64_t remote_address,mca_btl_base_registration_handle_t * local_handle,mca_btl_base_registration_handle_t * remote_handle,enum ibv_wr_opcode opcode,int64_t operand,int64_t operand2,int flags,int order,mca_btl_base_rdma_completion_fn_t cbfunc,void * cbcontext,void * cbdata)22 static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
23 void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
24 mca_btl_base_registration_handle_t *remote_handle, enum ibv_wr_opcode opcode,
25 int64_t operand, int64_t operand2, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
26 void *cbcontext, void *cbdata)
27 {
28 mca_btl_openib_get_frag_t* frag = NULL;
29 int qp = order;
30 int32_t rkey;
31 int rc;
32
33 frag = to_get_frag(alloc_recv_user_frag());
34 if (OPAL_UNLIKELY(NULL == frag)) {
35 return OPAL_ERR_OUT_OF_RESOURCE;
36 }
37
38 if (MCA_BTL_NO_ORDER == qp) {
39 qp = mca_btl_openib_component.rdma_qp;
40 }
41
42 /* set base descriptor flags */
43 to_base_frag(frag)->base.order = qp;
44 /* free this descriptor when the operation is complete */
45 to_base_frag(frag)->base.des_flags = MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
46
47 /* set up scatter-gather entry */
48 to_com_frag(frag)->sg_entry.length = 8;
49 to_com_frag(frag)->sg_entry.lkey = local_handle->lkey;
50 to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t) local_address;
51 to_com_frag(frag)->endpoint = endpoint;
52
53 /* set up rdma callback */
54 frag->cb.func = cbfunc;
55 frag->cb.context = cbcontext;
56 frag->cb.data = cbdata;
57 frag->cb.local_handle = local_handle;
58
59 /* set up descriptor */
60 frag->sr_desc.wr.atomic.remote_addr = remote_address;
61 frag->sr_desc.opcode = opcode;
62 frag->sr_desc.wr.atomic.compare_add = operand;
63 frag->sr_desc.wr.atomic.swap = operand2;
64
65 rkey = remote_handle->rkey;
66
67 #if OPAL_ENABLE_HETEROGENEOUS_SUPPORT
68 if((endpoint->endpoint_proc->proc_opal->proc_arch & OPAL_ARCH_ISBIGENDIAN)
69 != (opal_proc_local_get()->proc_arch & OPAL_ARCH_ISBIGENDIAN)) {
70 rkey = opal_swap_bytes4 (rkey);
71 }
72 #endif
73
74 frag->sr_desc.wr.atomic.rkey = rkey;
75
76 /* NTH: the SRQ# is set in mca_btl_get_internal */
77
78 if (endpoint->endpoint_state != MCA_BTL_IB_CONNECTED) {
79 OPAL_THREAD_LOCK(&endpoint->endpoint_lock);
80 rc = check_endpoint_state(endpoint, &to_base_frag(frag)->base, &endpoint->pending_get_frags);
81 OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock);
82 if (OPAL_ERR_RESOURCE_BUSY == rc) {
83 return OPAL_SUCCESS;
84 }
85
86 if (OPAL_SUCCESS != rc) {
87 MCA_BTL_IB_FRAG_RETURN (frag);
88 return rc;
89 }
90 }
91
92 rc = mca_btl_openib_get_internal (btl, endpoint, frag);
93 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
94 if (OPAL_LIKELY(OPAL_ERR_OUT_OF_RESOURCE == rc)) {
95 rc = OPAL_SUCCESS;
96
97 OPAL_THREAD_SCOPED_LOCK(&endpoint->endpoint_lock,
98 opal_list_append(&endpoint->pending_get_frags, (opal_list_item_t*)frag));
99 } else {
100 MCA_BTL_IB_FRAG_RETURN (frag);
101 }
102 }
103
104 return rc;
105 }
106
mca_btl_openib_atomic_fop(struct mca_btl_base_module_t * btl,struct mca_btl_base_endpoint_t * endpoint,void * local_address,uint64_t remote_address,struct mca_btl_base_registration_handle_t * local_handle,struct mca_btl_base_registration_handle_t * remote_handle,mca_btl_base_atomic_op_t op,uint64_t operand,int flags,int order,mca_btl_base_rdma_completion_fn_t cbfunc,void * cbcontext,void * cbdata)107 int mca_btl_openib_atomic_fop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
108 void *local_address, uint64_t remote_address,
109 struct mca_btl_base_registration_handle_t *local_handle,
110 struct mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
111 uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
112 void *cbcontext, void *cbdata)
113 {
114
115 if (OPAL_UNLIKELY(MCA_BTL_ATOMIC_ADD != op || (MCA_BTL_ATOMIC_FLAG_32BIT & flags))) {
116 return OPAL_ERR_NOT_SUPPORTED;
117 }
118
119 return mca_btl_openib_atomic_internal (btl, endpoint, local_address, remote_address, local_handle,
120 remote_handle, IBV_WR_ATOMIC_FETCH_AND_ADD, operand, 0,
121 flags, order, cbfunc, cbcontext, cbdata);
122 }
123
mca_btl_openib_atomic_cswap(struct mca_btl_base_module_t * btl,struct mca_btl_base_endpoint_t * endpoint,void * local_address,uint64_t remote_address,struct mca_btl_base_registration_handle_t * local_handle,struct mca_btl_base_registration_handle_t * remote_handle,uint64_t compare,uint64_t value,int flags,int order,mca_btl_base_rdma_completion_fn_t cbfunc,void * cbcontext,void * cbdata)124 int mca_btl_openib_atomic_cswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
125 void *local_address, uint64_t remote_address,
126 struct mca_btl_base_registration_handle_t *local_handle,
127 struct mca_btl_base_registration_handle_t *remote_handle, uint64_t compare,
128 uint64_t value, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
129 void *cbcontext, void *cbdata)
130 {
131 if (OPAL_UNLIKELY(MCA_BTL_ATOMIC_FLAG_32BIT & flags)) {
132 return OPAL_ERR_NOT_SUPPORTED;
133 }
134
135 return mca_btl_openib_atomic_internal (btl, endpoint, local_address, remote_address, local_handle,
136 remote_handle, IBV_WR_ATOMIC_CMP_AND_SWP, compare, value,
137 flags, order, cbfunc, cbcontext, cbdata);
138 }
139
140 #endif
141