1 /*
2  * Copyright (c) 2013-2015 Intel Corporation.  All rights reserved.
3  *
4  * This software is available to you under the BSD license
5  * below:
6  *
7  *     Redistribution and use in source and binary forms, with or
8  *     without modification, are permitted provided that the following
9  *     conditions are met:
10  *
11  *      - Redistributions of source code must retain the above
12  *        copyright notice, this list of conditions and the following
13  *        disclaimer.
14  *
15  *      - Redistributions in binary form must reproduce the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer in the documentation and/or other materials
18  *        provided with the distribution.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
24  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
25  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
26  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27  * SOFTWARE.
28  */
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <getopt.h>
34 
35 #include <rdma/fi_errno.h>
36 #include <rdma/fi_endpoint.h>
37 #include <rdma/fi_cm.h>
38 
39 #include <shared.h>
40 
41 
42 static int ctx_cnt = 2;
43 static int rx_ctx_bits = 0;
44 static struct fid_ep *sep;
45 static struct fid_ep **tx_ep, **rx_ep;
46 static struct fid_cq **txcq_array;
47 static struct fid_cq **rxcq_array;
48 static fi_addr_t *remote_rx_addr;
49 
free_res(void)50 static void free_res(void)
51 {
52 	if (rx_ep) {
53 		FT_CLOSEV_FID(rx_ep, ctx_cnt);
54 		free(rx_ep);
55 		rx_ep = NULL;
56 	}
57 	if (tx_ep) {
58 		FT_CLOSEV_FID(tx_ep, ctx_cnt);
59 		free(tx_ep);
60 		tx_ep = NULL;
61 	}
62 	if (rxcq_array) {
63 		FT_CLOSEV_FID(rxcq_array, ctx_cnt);
64 		free(rxcq_array);
65 		rxcq_array = NULL;
66 	}
67 	if (txcq_array) {
68 		FT_CLOSEV_FID(txcq_array, ctx_cnt);
69 		free(txcq_array);
70 		txcq_array = NULL;
71 	}
72 }
73 
alloc_ep_res(struct fid_ep * sep)74 static int alloc_ep_res(struct fid_ep *sep)
75 {
76 	int i, ret;
77 
78 	/* Get number of bits needed to represent ctx_cnt */
79 	while (ctx_cnt >> ++rx_ctx_bits);
80 
81 	av_attr.rx_ctx_bits = rx_ctx_bits;
82 
83 	ret = ft_alloc_ep_res(fi);
84 	if (ret)
85 		return ret;
86 
87 	txcq_array = calloc(ctx_cnt, sizeof *txcq_array);
88 	rxcq_array = calloc(ctx_cnt, sizeof *rxcq_array);
89 	tx_ep = calloc(ctx_cnt, sizeof *tx_ep);
90 	rx_ep = calloc(ctx_cnt, sizeof *rx_ep);
91 	remote_rx_addr = calloc(ctx_cnt, sizeof *remote_rx_addr);
92 
93 	if (!buf || !txcq_array || !rxcq_array || !tx_ep || !rx_ep || !remote_rx_addr) {
94 		perror("malloc");
95 		return -1;
96 	}
97 
98 	for (i = 0; i < ctx_cnt; i++) {
99 		ret = fi_tx_context(sep, i, NULL, &tx_ep[i], NULL);
100 		if (ret) {
101 			FT_PRINTERR("fi_tx_context", ret);
102 			return ret;
103 		}
104 
105 		ret = fi_cq_open(domain, &cq_attr, &txcq_array[i], NULL);
106 		if (ret) {
107 			FT_PRINTERR("fi_cq_open", ret);
108 			return ret;
109 		}
110 
111 		ret = fi_rx_context(sep, i, NULL, &rx_ep[i], NULL);
112 		if (ret) {
113 			FT_PRINTERR("fi_rx_context", ret);
114 			return ret;
115 		}
116 
117 		ret = fi_cq_open(domain, &cq_attr, &rxcq_array[i], NULL);
118 		if (ret) {
119 			FT_PRINTERR("fi_cq_open", ret);
120 			return ret;
121 		}
122 	}
123 
124 	return 0;
125 }
126 
bind_ep_res(void)127 static int bind_ep_res(void)
128 {
129 	int i, ret;
130 
131 	ret = fi_scalable_ep_bind(sep, &av->fid, 0);
132 	if (ret) {
133 		FT_PRINTERR("fi_scalable_ep_bind", ret);
134 		return ret;
135 	}
136 
137 	for (i = 0; i < ctx_cnt; i++) {
138 		ret = fi_ep_bind(tx_ep[i], &txcq_array[i]->fid, FI_SEND);
139 		if (ret) {
140 			FT_PRINTERR("fi_ep_bind", ret);
141 			return ret;
142 		}
143 
144 		ret = fi_enable(tx_ep[i]);
145 		if (ret) {
146 			FT_PRINTERR("fi_enable", ret);
147 			return ret;
148 		}
149 	}
150 
151 	for (i = 0; i < ctx_cnt; i++) {
152 		ret = fi_ep_bind(rx_ep[i], &rxcq_array[i]->fid, FI_RECV);
153 		if (ret) {
154 			FT_PRINTERR("fi_ep_bind", ret);
155 			return ret;
156 		}
157 
158 		ret = fi_enable(rx_ep[i]);
159 		if (ret) {
160 			FT_PRINTERR("fi_enable", ret);
161 			return ret;
162 		}
163 
164 		ret = fi_recv(rx_ep[i], rx_buf, MAX(rx_size, FT_MAX_CTRL_MSG),
165 			      mr_desc, 0, NULL);
166 		if (ret) {
167 			FT_PRINTERR("fi_recv", ret);
168 			return ret;
169 		}
170 	}
171 
172 	ret = fi_enable(sep);
173 	if (ret) {
174 		FT_PRINTERR("fi_enable", ret);
175 		return ret;
176 	}
177 
178 	return 0;
179 }
180 
wait_for_comp(struct fid_cq * cq)181 static int wait_for_comp(struct fid_cq *cq)
182 {
183 	struct fi_cq_entry comp;
184 	int ret;
185 
186 	do {
187 		ret = fi_cq_read(cq, &comp, 1);
188 	} while (ret < 0 && ret == -FI_EAGAIN);
189 
190 	if (ret != 1)
191 		FT_PRINTERR("fi_cq_read", ret);
192 	else
193 		ret = 0;
194 
195 	return ret;
196 }
197 
198 #define DATA 0x12345670
199 
run_test()200 static int run_test()
201 {
202 	int ret = 0, i;
203 	uint32_t data;
204 	uint32_t *tb = (uint32_t *)tx_buf;
205 	uint32_t *rb = (uint32_t *)rx_buf;
206 
207 	if (opts.dst_addr) {
208 		for (i = 0; i < ctx_cnt && !ret; i++) {
209 			fprintf(stdout, "Posting send for ctx: %d\n", i);
210 			tb[0] = DATA + i;
211 			ret = fi_send(tx_ep[i], tx_buf, tx_size, mr_desc,
212 				      remote_rx_addr[i], NULL);
213 			if (ret) {
214 				FT_PRINTERR("fi_send", ret);
215 				return ret;
216 			}
217 
218 			ret = wait_for_comp(txcq_array[i]);
219 		}
220 	} else {
221 		for (i = 0; i < ctx_cnt && !ret; i++) {
222 			fprintf(stdout, "wait for recv completion for ctx: %d\n", i);
223 			ret = wait_for_comp(rxcq_array[i]);
224 
225 			data = DATA + i;
226 			if (memcmp(&data, rx_buf, 4) != 0) {
227 				fprintf(stdout, "failed compare expected 0x%x,"
228 					" read 0x%x\n", data, rb[0]);
229 			}
230 		}
231 	}
232 
233 	return ret;
234 }
235 
init_fabric(void)236 static int init_fabric(void)
237 {
238 	int ret;
239 
240 	ret = ft_getinfo(hints, &fi);
241 	if (ret)
242 		return ret;
243 
244 	/* Check the optimal number of TX and RX contexts supported by the provider */
245 	ctx_cnt = MIN(ctx_cnt, fi->domain_attr->tx_ctx_cnt);
246 	ctx_cnt = MIN(ctx_cnt, fi->domain_attr->rx_ctx_cnt);
247 	if (!ctx_cnt) {
248 		fprintf(stderr, "Provider doesn't support contexts\n");
249 		return 1;
250 	}
251 
252 	fi->ep_attr->tx_ctx_cnt = ctx_cnt;
253 	fi->ep_attr->rx_ctx_cnt = ctx_cnt;
254 
255 	ret = ft_open_fabric_res();
256 	if (ret)
257 		return ret;
258 
259 	ret = fi_scalable_ep(domain, fi, &sep, NULL);
260 	if (ret) {
261 		FT_PRINTERR("fi_scalable_ep", ret);
262 		return ret;
263 	}
264 
265 	ret = alloc_ep_res(sep);
266 	if (ret)
267 		return ret;
268 
269 	ret = bind_ep_res();
270 	return ret;
271 }
272 
init_av(void)273 static int init_av(void)
274 {
275 	size_t addrlen;
276 	int ret, i;
277 
278 	if (opts.dst_addr) {
279 		ret = ft_av_insert(av, fi->dest_addr, 1, &remote_fi_addr, 0, NULL);
280 		if (ret)
281 			return ret;
282 
283 		addrlen = FT_MAX_CTRL_MSG;
284 		ret = fi_getname(&sep->fid, tx_buf, &addrlen);
285 		if (ret) {
286 			FT_PRINTERR("fi_getname", ret);
287 			return ret;
288 		}
289 
290 		ret = fi_send(tx_ep[0], tx_buf, addrlen,
291 			      mr_desc, remote_fi_addr, NULL);
292 		if (ret) {
293 			FT_PRINTERR("fi_send", ret);
294 			return ret;
295 		}
296 
297 		ret = wait_for_comp(rxcq_array[0]);
298 		if (ret)
299 			return ret;
300 	} else {
301 		ret = wait_for_comp(rxcq_array[0]);
302 		if (ret)
303 			return ret;
304 
305 		ret = ft_av_insert(av, rx_buf, 1, &remote_fi_addr, 0, NULL);
306 		if (ret)
307 			return ret;
308 
309 		ret = fi_send(tx_ep[0], tx_buf, 1,
310 			      mr_desc, remote_fi_addr, NULL);
311 		if (ret) {
312 			FT_PRINTERR("fi_send", ret);
313 			return ret;
314 		}
315 	}
316 
317 	for (i = 0; i < ctx_cnt; i++)
318 		remote_rx_addr[i] = fi_rx_addr(remote_fi_addr, i, rx_ctx_bits);
319 
320 	ret = fi_recv(rx_ep[0], rx_buf, rx_size, mr_desc, 0, NULL);
321 	if (ret) {
322 		FT_PRINTERR("fi_recv", ret);
323 		return ret;
324 	}
325 
326 	ret = wait_for_comp(txcq_array[0]);
327 	return ret;
328 }
329 
330 
run(void)331 static int run(void)
332 {
333 	int ret = 0;
334 
335 	ret = init_fabric();
336 	if (ret)
337 		return ret;
338 
339 	ret = init_av();
340 	if (ret)
341 		return ret;
342 
343 	ret = run_test();
344 
345 	/*TODO: Add a local finalize applicable for scalable ep */
346 	//ft_finalize(fi, tx_ep[0], txcq_array[0], rxcq_array[0], remote_rx_addr[0]);
347 
348 	return ret;
349 }
350 
main(int argc,char ** argv)351 int main(int argc, char **argv)
352 {
353 	int ret, op;
354 
355 	opts = INIT_OPTS;
356 	opts.options = FT_OPT_SIZE;
357 
358 	hints = fi_allocinfo();
359 	if (!hints)
360 		return EXIT_FAILURE;
361 
362 	while ((op = getopt(argc, argv, "h" ADDR_OPTS INFO_OPTS)) != -1) {
363 		switch (op) {
364 		default:
365 			ft_parse_addr_opts(op, optarg, &opts);
366 			ft_parseinfo(op, optarg, hints, &opts);
367 			break;
368 		case '?':
369 		case 'h':
370 			ft_usage(argv[0], "An RDM client-server example with scalable endpoints.\n");
371 			return EXIT_FAILURE;
372 		}
373 	}
374 
375 	if (optind < argc)
376 		opts.dst_addr = argv[optind];
377 
378 	hints->ep_attr->type = FI_EP_RDM;
379 	hints->caps = FI_MSG | FI_NAMED_RX_CTX;
380 	hints->domain_attr->mr_mode = opts.mr_mode;
381 
382 	ret = run();
383 
384 	free_res();
385 	/* Closes the scalable ep that was allocated in the test */
386 	FT_CLOSE_FID(sep);
387 	ft_free_res();
388 	return ft_exit_code(ret);
389 }
390