1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
4 * reserved.
5 * $COPYRIGHT$
6 *
7 * Additional copyrights may follow
8 *
9 * $HEADER$
10 */
11
12 #include "btl_ugni_rdma.h"
13
14 static gni_fma_cmd_type_t amo_cmds[][MCA_BTL_ATOMIC_LAST] = {
15 [OPAL_INT32] = {
16 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_IADD_S,
17 [MCA_BTL_ATOMIC_LAND] = GNI_FMA_ATOMIC2_AND_S,
18 [MCA_BTL_ATOMIC_LOR] = GNI_FMA_ATOMIC2_OR_S,
19 [MCA_BTL_ATOMIC_LXOR] = GNI_FMA_ATOMIC2_XOR_S,
20 [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_SWAP_S,
21 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_IMIN_S,
22 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_IMAX_S,
23 },
24 [OPAL_INT64] = {
25 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC_ADD,
26 [MCA_BTL_ATOMIC_AND] = GNI_FMA_ATOMIC_AND,
27 [MCA_BTL_ATOMIC_OR] = GNI_FMA_ATOMIC_OR,
28 [MCA_BTL_ATOMIC_XOR] = GNI_FMA_ATOMIC_XOR,
29 [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_SWAP,
30 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_IMIN,
31 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_IMAX,
32 },
33 [OPAL_FLOAT] = {
34 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FPADD_S,
35 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FPMIN_S,
36 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FPMAX_S,
37 },
38 [OPAL_DOUBLE] = {
39 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FPADD,
40 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FPMIN,
41 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FPMAX,
42 },
43 };
44
45 static gni_fma_cmd_type_t famo_cmds[][MCA_BTL_ATOMIC_LAST] = {
46 [OPAL_INT32] = {
47 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FIADD_S,
48 [MCA_BTL_ATOMIC_LAND] = GNI_FMA_ATOMIC2_FAND_S,
49 [MCA_BTL_ATOMIC_LOR] = GNI_FMA_ATOMIC2_FOR_S,
50 [MCA_BTL_ATOMIC_LXOR] = GNI_FMA_ATOMIC2_FXOR_S,
51 [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_FSWAP_S,
52 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FIMIN_S,
53 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FIMAX_S,
54 },
55 [OPAL_INT64] = {
56 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC_FADD,
57 [MCA_BTL_ATOMIC_AND] = GNI_FMA_ATOMIC_FAND,
58 [MCA_BTL_ATOMIC_OR] = GNI_FMA_ATOMIC_FOR,
59 [MCA_BTL_ATOMIC_XOR] = GNI_FMA_ATOMIC_FXOR,
60 [MCA_BTL_ATOMIC_SWAP] = GNI_FMA_ATOMIC2_FSWAP,
61 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FIMIN,
62 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FIMAX,
63 },
64 [OPAL_FLOAT] = {
65 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FFPADD_S,
66 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FFPMIN_S,
67 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FFPMAX_S,
68 },
69 [OPAL_DOUBLE] = {
70 [MCA_BTL_ATOMIC_ADD] = GNI_FMA_ATOMIC2_FFPADD,
71 [MCA_BTL_ATOMIC_MIN] = GNI_FMA_ATOMIC2_FFPMIN,
72 [MCA_BTL_ATOMIC_MAX] = GNI_FMA_ATOMIC2_FFPMAX,
73 },
74 };
75
mca_btl_ugni_aop(struct mca_btl_base_module_t * btl,struct mca_btl_base_endpoint_t * endpoint,uint64_t remote_address,mca_btl_base_registration_handle_t * remote_handle,mca_btl_base_atomic_op_t op,uint64_t operand,int flags,int order,mca_btl_base_rdma_completion_fn_t cbfunc,void * cbcontext,void * cbdata)76 int mca_btl_ugni_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
77 uint64_t remote_address, mca_btl_base_registration_handle_t *remote_handle,
78 mca_btl_base_atomic_op_t op, uint64_t operand, int flags, int order,
79 mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
80 {
81 gni_mem_handle_t dummy = {0, 0};
82 mca_btl_ugni_post_descriptor_t *post_desc;
83 int gni_op, rc, type;
84 size_t size;
85
86 size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
87 if (MCA_BTL_ATOMIC_FLAG_FLOAT & flags) {
88 type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? OPAL_FLOAT : OPAL_DOUBLE;
89 } else {
90 type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? OPAL_INT32 : OPAL_INT64;
91 }
92
93 gni_op = amo_cmds[type][op];
94 if (0 == gni_op) {
95 return OPAL_ERR_NOT_SUPPORTED;
96 }
97
98 post_desc = mca_btl_ugni_alloc_post_descriptor (endpoint, NULL, cbfunc, cbcontext, cbdata);
99 if (OPAL_UNLIKELY(NULL == post_desc)) {
100 return OPAL_ERR_OUT_OF_RESOURCE;
101 }
102
103 init_gni_post_desc (post_desc, order, GNI_POST_AMO, 0, dummy, remote_address,
104 remote_handle->gni_handle, size, 0);
105 post_desc->desc.amo_cmd = gni_op;
106
107 post_desc->desc.first_operand = operand;
108
109 rc = mca_btl_ugni_endpoint_post_fma (endpoint, post_desc);
110 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
111 mca_btl_ugni_return_post_descriptor (post_desc);
112 }
113
114 return rc;
115 }
116
mca_btl_ugni_afop(struct mca_btl_base_module_t * btl,struct mca_btl_base_endpoint_t * endpoint,void * local_address,uint64_t remote_address,mca_btl_base_registration_handle_t * local_handle,mca_btl_base_registration_handle_t * remote_handle,mca_btl_base_atomic_op_t op,uint64_t operand,int flags,int order,mca_btl_base_rdma_completion_fn_t cbfunc,void * cbcontext,void * cbdata)117 int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
118 void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
119 mca_btl_base_registration_handle_t *remote_handle, mca_btl_base_atomic_op_t op,
120 uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
121 void *cbcontext, void *cbdata)
122 {
123 mca_btl_ugni_post_descriptor_t *post_desc;
124 int gni_op, rc, type;
125 size_t size;
126
127 size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
128 if (MCA_BTL_ATOMIC_FLAG_FLOAT & flags) {
129 type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? OPAL_FLOAT : OPAL_DOUBLE;
130 } else {
131 type = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? OPAL_INT32 : OPAL_INT64;
132 }
133
134 gni_op = famo_cmds[type][op];
135 if (0 == gni_op) {
136 return OPAL_ERR_NOT_SUPPORTED;
137 }
138
139 post_desc = mca_btl_ugni_alloc_post_descriptor (endpoint, local_handle, cbfunc, cbcontext, cbdata);
140 if (OPAL_UNLIKELY(NULL == post_desc)) {
141 return OPAL_ERR_OUT_OF_RESOURCE;
142 }
143
144
145 init_gni_post_desc (post_desc, order, GNI_POST_AMO, (intptr_t) local_address, local_handle->gni_handle,
146 remote_address, remote_handle->gni_handle, size, 0);
147 post_desc->desc.amo_cmd = gni_op;
148
149 post_desc->desc.first_operand = operand;
150
151 rc = mca_btl_ugni_endpoint_post_fma (endpoint, post_desc);
152 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
153 mca_btl_ugni_return_post_descriptor (post_desc);
154 }
155
156 return rc;
157 }
158
mca_btl_ugni_acswap(struct mca_btl_base_module_t * btl,struct mca_btl_base_endpoint_t * endpoint,void * local_address,uint64_t remote_address,mca_btl_base_registration_handle_t * local_handle,mca_btl_base_registration_handle_t * remote_handle,uint64_t compare,uint64_t value,int flags,int order,mca_btl_base_rdma_completion_fn_t cbfunc,void * cbcontext,void * cbdata)159 int mca_btl_ugni_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
160 void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
161 mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
162 int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
163 {
164 mca_btl_ugni_post_descriptor_t *post_desc;
165 int gni_op, rc;
166 size_t size;
167
168 gni_op = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? GNI_FMA_ATOMIC2_CSWAP_S : GNI_FMA_ATOMIC_CSWAP;
169 size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
170
171 post_desc = mca_btl_ugni_alloc_post_descriptor (endpoint, local_handle, cbfunc, cbcontext, cbdata);
172 if (OPAL_UNLIKELY(NULL == post_desc)) {
173 return OPAL_ERR_OUT_OF_RESOURCE;
174 }
175
176
177 init_gni_post_desc (post_desc, order, GNI_POST_AMO, (intptr_t) local_address, local_handle->gni_handle,
178 remote_address, remote_handle->gni_handle, size, 0);
179 post_desc->desc.amo_cmd = gni_op;
180
181 post_desc->desc.first_operand = compare;
182 post_desc->desc.second_operand = value;
183
184 rc = mca_btl_ugni_endpoint_post_fma (endpoint, post_desc);
185 if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
186 mca_btl_ugni_return_post_descriptor (post_desc);
187 }
188
189 return rc;
190 }
191