1 /*
2 * Copyright (c) 2013-2015 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under the BSD license
5 * below:
6 *
7 * Redistribution and use in source and binary forms, with or
8 * without modification, are permitted provided that the following
9 * conditions are met:
10 *
11 * - Redistributions of source code must retain the above
12 * copyright notice, this list of conditions and the following
13 * disclaimer.
14 *
15 * - Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials
18 * provided with the distribution.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
24 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
25 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
26 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 * SOFTWARE.
28 */
29
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <getopt.h>
34
35 #include <rdma/fi_errno.h>
36 #include <rdma/fi_endpoint.h>
37 #include <rdma/fi_cm.h>
38
39 #include <shared.h>
40
41
42 static int ctx_cnt = 2;
43 static int rx_ctx_bits = 0;
44 static struct fid_ep *sep;
45 static struct fid_ep **tx_ep, **rx_ep;
46 static struct fid_cq **txcq_array;
47 static struct fid_cq **rxcq_array;
48 static fi_addr_t *remote_rx_addr;
49
free_res(void)50 static void free_res(void)
51 {
52 if (rx_ep) {
53 FT_CLOSEV_FID(rx_ep, ctx_cnt);
54 free(rx_ep);
55 rx_ep = NULL;
56 }
57 if (tx_ep) {
58 FT_CLOSEV_FID(tx_ep, ctx_cnt);
59 free(tx_ep);
60 tx_ep = NULL;
61 }
62 if (rxcq_array) {
63 FT_CLOSEV_FID(rxcq_array, ctx_cnt);
64 free(rxcq_array);
65 rxcq_array = NULL;
66 }
67 if (txcq_array) {
68 FT_CLOSEV_FID(txcq_array, ctx_cnt);
69 free(txcq_array);
70 txcq_array = NULL;
71 }
72 }
73
alloc_ep_res(struct fid_ep * sep)74 static int alloc_ep_res(struct fid_ep *sep)
75 {
76 int i, ret;
77
78 /* Get number of bits needed to represent ctx_cnt */
79 while (ctx_cnt >> ++rx_ctx_bits);
80
81 av_attr.rx_ctx_bits = rx_ctx_bits;
82
83 ret = ft_alloc_ep_res(fi);
84 if (ret)
85 return ret;
86
87 txcq_array = calloc(ctx_cnt, sizeof *txcq_array);
88 rxcq_array = calloc(ctx_cnt, sizeof *rxcq_array);
89 tx_ep = calloc(ctx_cnt, sizeof *tx_ep);
90 rx_ep = calloc(ctx_cnt, sizeof *rx_ep);
91 remote_rx_addr = calloc(ctx_cnt, sizeof *remote_rx_addr);
92
93 if (!buf || !txcq_array || !rxcq_array || !tx_ep || !rx_ep || !remote_rx_addr) {
94 perror("malloc");
95 return -1;
96 }
97
98 for (i = 0; i < ctx_cnt; i++) {
99 ret = fi_tx_context(sep, i, NULL, &tx_ep[i], NULL);
100 if (ret) {
101 FT_PRINTERR("fi_tx_context", ret);
102 return ret;
103 }
104
105 ret = fi_cq_open(domain, &cq_attr, &txcq_array[i], NULL);
106 if (ret) {
107 FT_PRINTERR("fi_cq_open", ret);
108 return ret;
109 }
110
111 ret = fi_rx_context(sep, i, NULL, &rx_ep[i], NULL);
112 if (ret) {
113 FT_PRINTERR("fi_rx_context", ret);
114 return ret;
115 }
116
117 ret = fi_cq_open(domain, &cq_attr, &rxcq_array[i], NULL);
118 if (ret) {
119 FT_PRINTERR("fi_cq_open", ret);
120 return ret;
121 }
122 }
123
124 return 0;
125 }
126
bind_ep_res(void)127 static int bind_ep_res(void)
128 {
129 int i, ret;
130
131 ret = fi_scalable_ep_bind(sep, &av->fid, 0);
132 if (ret) {
133 FT_PRINTERR("fi_scalable_ep_bind", ret);
134 return ret;
135 }
136
137 for (i = 0; i < ctx_cnt; i++) {
138 ret = fi_ep_bind(tx_ep[i], &txcq_array[i]->fid, FI_SEND);
139 if (ret) {
140 FT_PRINTERR("fi_ep_bind", ret);
141 return ret;
142 }
143
144 ret = fi_enable(tx_ep[i]);
145 if (ret) {
146 FT_PRINTERR("fi_enable", ret);
147 return ret;
148 }
149 }
150
151 for (i = 0; i < ctx_cnt; i++) {
152 ret = fi_ep_bind(rx_ep[i], &rxcq_array[i]->fid, FI_RECV);
153 if (ret) {
154 FT_PRINTERR("fi_ep_bind", ret);
155 return ret;
156 }
157
158 ret = fi_enable(rx_ep[i]);
159 if (ret) {
160 FT_PRINTERR("fi_enable", ret);
161 return ret;
162 }
163
164 ret = fi_recv(rx_ep[i], rx_buf, MAX(rx_size, FT_MAX_CTRL_MSG),
165 mr_desc, 0, NULL);
166 if (ret) {
167 FT_PRINTERR("fi_recv", ret);
168 return ret;
169 }
170 }
171
172 ret = fi_enable(sep);
173 if (ret) {
174 FT_PRINTERR("fi_enable", ret);
175 return ret;
176 }
177
178 return 0;
179 }
180
wait_for_comp(struct fid_cq * cq)181 static int wait_for_comp(struct fid_cq *cq)
182 {
183 struct fi_cq_entry comp;
184 int ret;
185
186 do {
187 ret = fi_cq_read(cq, &comp, 1);
188 } while (ret < 0 && ret == -FI_EAGAIN);
189
190 if (ret != 1)
191 FT_PRINTERR("fi_cq_read", ret);
192 else
193 ret = 0;
194
195 return ret;
196 }
197
198 #define DATA 0x12345670
199
run_test()200 static int run_test()
201 {
202 int ret = 0, i;
203 uint32_t data;
204 uint32_t *tb = (uint32_t *)tx_buf;
205 uint32_t *rb = (uint32_t *)rx_buf;
206
207 if (opts.dst_addr) {
208 for (i = 0; i < ctx_cnt && !ret; i++) {
209 fprintf(stdout, "Posting send for ctx: %d\n", i);
210 tb[0] = DATA + i;
211 ret = fi_send(tx_ep[i], tx_buf, tx_size, mr_desc,
212 remote_rx_addr[i], NULL);
213 if (ret) {
214 FT_PRINTERR("fi_send", ret);
215 return ret;
216 }
217
218 ret = wait_for_comp(txcq_array[i]);
219 }
220 } else {
221 for (i = 0; i < ctx_cnt && !ret; i++) {
222 fprintf(stdout, "wait for recv completion for ctx: %d\n", i);
223 ret = wait_for_comp(rxcq_array[i]);
224
225 data = DATA + i;
226 if (memcmp(&data, rx_buf, 4) != 0) {
227 fprintf(stdout, "failed compare expected 0x%x,"
228 " read 0x%x\n", data, rb[0]);
229 }
230 }
231 }
232
233 return ret;
234 }
235
init_fabric(void)236 static int init_fabric(void)
237 {
238 int ret;
239
240 ret = ft_getinfo(hints, &fi);
241 if (ret)
242 return ret;
243
244 /* Check the optimal number of TX and RX contexts supported by the provider */
245 ctx_cnt = MIN(ctx_cnt, fi->domain_attr->tx_ctx_cnt);
246 ctx_cnt = MIN(ctx_cnt, fi->domain_attr->rx_ctx_cnt);
247 if (!ctx_cnt) {
248 fprintf(stderr, "Provider doesn't support contexts\n");
249 return 1;
250 }
251
252 fi->ep_attr->tx_ctx_cnt = ctx_cnt;
253 fi->ep_attr->rx_ctx_cnt = ctx_cnt;
254
255 ret = ft_open_fabric_res();
256 if (ret)
257 return ret;
258
259 ret = fi_scalable_ep(domain, fi, &sep, NULL);
260 if (ret) {
261 FT_PRINTERR("fi_scalable_ep", ret);
262 return ret;
263 }
264
265 ret = alloc_ep_res(sep);
266 if (ret)
267 return ret;
268
269 ret = bind_ep_res();
270 return ret;
271 }
272
init_av(void)273 static int init_av(void)
274 {
275 size_t addrlen;
276 int ret, i;
277
278 if (opts.dst_addr) {
279 ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL);
280 if (ret)
281 return ret;
282
283 addrlen = FT_MAX_CTRL_MSG;
284 ret = fi_getname(&sep->fid, tx_buf, &addrlen);
285 if (ret) {
286 FT_PRINTERR("fi_getname", ret);
287 return ret;
288 }
289
290 ret = fi_send(tx_ep[0], tx_buf, addrlen,
291 mr_desc, remote_fi_addr, NULL);
292 if (ret) {
293 FT_PRINTERR("fi_send", ret);
294 return ret;
295 }
296
297 ret = wait_for_comp(rxcq_array[0]);
298 if (ret)
299 return ret;
300 } else {
301 ret = wait_for_comp(rxcq_array[0]);
302 if (ret)
303 return ret;
304
305 ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL);
306 if (ret)
307 return ret;
308
309 ret = fi_send(tx_ep[0], tx_buf, 1,
310 mr_desc, remote_fi_addr, NULL);
311 if (ret) {
312 FT_PRINTERR("fi_send", ret);
313 return ret;
314 }
315 }
316
317 for (i = 0; i < ctx_cnt; i++)
318 remote_rx_addr[i] = fi_rx_addr(remote_fi_addr, i, rx_ctx_bits);
319
320 ret = fi_recv(rx_ep[0], rx_buf, rx_size, mr_desc, 0, NULL);
321 if (ret) {
322 FT_PRINTERR("fi_recv", ret);
323 return ret;
324 }
325
326 ret = wait_for_comp(txcq_array[0]);
327 return ret;
328 }
329
330
run(void)331 static int run(void)
332 {
333 int ret = 0;
334
335 ret = init_fabric();
336 if (ret)
337 return ret;
338
339 ret = init_av();
340 if (ret)
341 return ret;
342
343 ret = run_test();
344
345 /*TODO: Add a local finalize applicable for scalable ep */
346 //ft_finalize(fi, tx_ep[0], txcq_array[0], rxcq_array[0], remote_rx_addr[0]);
347
348 return ret;
349 }
350
main(int argc,char ** argv)351 int main(int argc, char **argv)
352 {
353 int ret, op;
354
355 opts = INIT_OPTS;
356 opts.options = FT_OPT_SIZE;
357
358 hints = fi_allocinfo();
359 if (!hints)
360 return EXIT_FAILURE;
361
362 while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
363 switch (op) {
364 default:
365 ft_parse_addr_opts(op, optarg, &opts);
366 ft_parseinfo(op, optarg, hints, &opts);
367 break;
368 case '?':
369 case 'h':
370 ft_usage(argv[0], "An RDM client-server example with scalable endpoints.\n");
371 return EXIT_FAILURE;
372 }
373 }
374
375 if (optind < argc)
376 opts.dst_addr = argv[optind];
377
378 hints->ep_attr->type = FI_EP_RDM;
379 hints->caps = FI_MSG | FI_NAMED_RX_CTX;
380 hints->domain_attr->mr_mode = opts.mr_mode;
381
382 ret = run();
383
384 free_res();
385 /* Closes the scalable ep that was allocated in the test */
386 FT_CLOSE_FID(sep);
387 ft_free_res();
388 return ft_exit_code(ret);
389 }
390