xref: /freebsd/contrib/ofed/libibmad/rpc.c (revision 4f52dfbb) 
1 /*
2  * Copyright (c) 2004-2009 Voltaire Inc.  All rights reserved.
3  * Copyright (c) 2009 HNR Consulting.  All rights reserved.
4  * Copyright (c) 2011 Mellanox Technologies LTD.  All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35 
36 #if HAVE_CONFIG_H
37 #  include <config.h>
38 #endif				/* HAVE_CONFIG_H */
39 
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43 #include <errno.h>
44 
45 #include <infiniband/umad.h>
46 #include <infiniband/mad.h>
47 
48 #include "mad_internal.h"
49 
50 int ibdebug;
51 
52 static struct ibmad_port mad_port;
53 struct ibmad_port *ibmp = &mad_port;
54 
55 static int iberrs;
56 
57 int madrpc_retries = MAD_DEF_RETRIES;
58 int madrpc_timeout = MAD_DEF_TIMEOUT_MS;
59 
60 static void *save_mad;
61 static int save_mad_len = 256;
62 
63 #undef DEBUG
64 #define DEBUG	if (ibdebug)	IBWARN
65 #define ERRS(fmt, ...) do {	\
66 	if (iberrs || ibdebug)	\
67 		IBWARN(fmt, ## __VA_ARGS__); \
68 } while (0)
69 
70 #define MAD_TID(mad)	(*((uint64_t *)((char *)(mad) + 8)))
71 
72 void madrpc_show_errors(int set)
73 {
74 	iberrs = set;
75 }
76 
77 void madrpc_save_mad(void *madbuf, int len)
78 {
79 	save_mad = madbuf;
80 	save_mad_len = len;
81 }
82 
83 int madrpc_set_retries(int retries)
84 {
85 	if (retries > 0)
86 		madrpc_retries = retries;
87 	return madrpc_retries;
88 }
89 
90 int madrpc_set_timeout(int timeout)
91 {
92 	madrpc_timeout = timeout;
93 	return 0;
94 }
95 
96 void mad_rpc_set_retries(struct ibmad_port *port, int retries)
97 {
98 	port->retries = retries;
99 }
100 
101 void mad_rpc_set_timeout(struct ibmad_port *port, int timeout)
102 {
103 	port->timeout = timeout;
104 }
105 
106 int madrpc_def_timeout(void)
107 {
108 	return madrpc_timeout;
109 }
110 
111 int madrpc_portid(void)
112 {
113 	return ibmp->port_id;
114 }
115 
116 int mad_rpc_portid(struct ibmad_port *srcport)
117 {
118 	return srcport->port_id;
119 }
120 
121 int mad_rpc_class_agent(struct ibmad_port *port, int class)
122 {
123 	if (class < 1 || class >= MAX_CLASS)
124 		return -1;
125 	return port->class_agents[class];
126 }
127 
128 static int
129 _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len,
130 	   int timeout, int max_retries, int *p_error)
131 {
132 	uint32_t trid;		/* only low 32 bits - see mad_trid() */
133 	int retries;
134 	int length, status;
135 
136 	if (ibdebug > 1) {
137 		IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len);
138 		xdump(stderr, "send buf\n", sndbuf, umad_size() + len);
139 	}
140 
141 	if (save_mad) {
142 		memcpy(save_mad, umad_get_mad(sndbuf),
143 		       save_mad_len < len ? save_mad_len : len);
144 		save_mad = 0;
145 	}
146 
147 	if (max_retries <= 0) {
148 		errno = EINVAL;
149 		*p_error = EINVAL;
150 		ERRS("max_retries %d <= 0", max_retries);
151 		return -1;
152 	}
153 
154 	trid =
155 	    (uint32_t) mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F);
156 
157 	for (retries = 0; retries < max_retries; retries++) {
158 		if (retries)
159 			ERRS("retry %d (timeout %d ms)", retries, timeout);
160 
161 		length = len;
162 		if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) {
163 			IBWARN("send failed; %s", strerror(errno));
164 			return -1;
165 		}
166 
167 		/* Use same timeout on receive side just in case */
168 		/* send packet is lost somewhere. */
169 		do {
170 			length = len;
171 			if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) {
172 				IBWARN("recv failed: %s", strerror(errno));
173 				return -1;
174 			}
175 
176 			if (ibdebug > 2)
177 				umad_addr_dump(umad_get_mad_addr(rcvbuf));
178 			if (ibdebug > 1) {
179 				IBWARN("rcv buf:");
180 				xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf),
181 				      IB_MAD_SIZE);
182 			}
183 		} while ((uint32_t)
184 			 mad_get_field64(umad_get_mad(rcvbuf), 0,
185 					 IB_MAD_TRID_F) != trid);
186 
187 		status = umad_status(rcvbuf);
188 		if (!status)
189 			return length;	/* done */
190 		if (status == ENOMEM)
191 			return length;
192 	}
193 
194 	errno = status;
195 	*p_error = ETIMEDOUT;
196 	ERRS("timeout after %d retries, %d ms", retries, timeout * retries);
197 	return -1;
198 }
199 
200 static int redirect_port(ib_portid_t * port, uint8_t * mad)
201 {
202 	port->lid = mad_get_field(mad, 64, IB_CPI_REDIRECT_LID_F);
203 	if (!port->lid) {
204 		IBWARN("GID-based redirection is not supported");
205 		return -1;
206 	}
207 
208 	port->qp = mad_get_field(mad, 64, IB_CPI_REDIRECT_QP_F);
209 	port->qkey = mad_get_field(mad, 64, IB_CPI_REDIRECT_QKEY_F);
210 	port->sl = (uint8_t) mad_get_field(mad, 64, IB_CPI_REDIRECT_SL_F);
211 
212 	/* TODO: Reverse map redirection P_Key to P_Key index */
213 
214 	if (ibdebug)
215 		IBWARN("redirected to lid %d, qp 0x%x, qkey 0x%x, sl 0x%x",
216 		       port->lid, port->qp, port->qkey, port->sl);
217 
218 	return 0;
219 }
220 
221 void *mad_rpc(const struct ibmad_port *port, ib_rpc_t * rpc,
222 	      ib_portid_t * dport, void *payload, void *rcvdata)
223 {
224 	int status, len;
225 	uint8_t sndbuf[1024], rcvbuf[1024], *mad;
226 	ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc;
227 	int error = 0;
228 
229 	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
230 		rpcv1->error = 0;
231 	do {
232 		len = 0;
233 		memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
234 
235 		if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0)
236 			return NULL;
237 
238 		if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf,
239 				      port->class_agents[rpc->mgtclass & 0xff],
240 				      len, mad_get_timeout(port, rpc->timeout),
241 				      mad_get_retries(port), &error)) < 0) {
242 			if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) ==
243 			    IB_MAD_RPC_VERSION1)
244 				rpcv1->error = error;
245 			IBWARN("_do_madrpc failed; dport (%s)",
246 			       portid2str(dport));
247 			return NULL;
248 		}
249 
250 		mad = umad_get_mad(rcvbuf);
251 		status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F);
252 
253 		/* check for exact match instead of only the redirect bit;
254 		 * that way, weird statuses cause an error, too */
255 		if (status == IB_MAD_STS_REDIRECT) {
256 			/* update dport for next request and retry */
257 			/* bail if redirection fails */
258 			if (redirect_port(dport, mad))
259 				break;
260 		} else
261 			break;
262 	} while (1);
263 
264 	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
265 		rpcv1->error = error;
266 	rpc->rstatus = status;
267 
268 	if (status != 0) {
269 		ERRS("MAD completed with error status 0x%x; dport (%s)",
270 		     status, portid2str(dport));
271 		errno = EIO;
272 		return NULL;
273 	}
274 
275 	if (ibdebug) {
276 		IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
277 		xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz);
278 	}
279 
280 	if (rcvdata)
281 		memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz);
282 
283 	return rcvdata;
284 }
285 
286 void *mad_rpc_rmpp(const struct ibmad_port *port, ib_rpc_t * rpc,
287 		   ib_portid_t * dport, ib_rmpp_hdr_t * rmpp, void *data)
288 {
289 	int status, len;
290 	uint8_t sndbuf[1024], rcvbuf[1024], *mad;
291 	ib_rpc_v1_t *rpcv1 = (ib_rpc_v1_t *)rpc;
292 	int error = 0;
293 
294 	memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
295 
296 	DEBUG("rmpp %p data %p", rmpp, data);
297 
298 	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
299 		rpcv1->error = 0;
300 	if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0)
301 		return NULL;
302 
303 	if ((len = _do_madrpc(port->port_id, sndbuf, rcvbuf,
304 			      port->class_agents[rpc->mgtclass & 0xff],
305 			      len, mad_get_timeout(port, rpc->timeout),
306 			      mad_get_retries(port), &error)) < 0) {
307 		if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
308 			rpcv1->error = error;
309 		IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
310 		return NULL;
311 	}
312 
313 	if ((rpc->mgtclass & IB_MAD_RPC_VERSION_MASK) == IB_MAD_RPC_VERSION1)
314 		rpcv1->error = error;
315 
316 	mad = umad_get_mad(rcvbuf);
317 
318 	if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) {
319 		ERRS("MAD completed with error status 0x%x; dport (%s)",
320 		     status, portid2str(dport));
321 		errno = EIO;
322 		return NULL;
323 	}
324 
325 	if (ibdebug) {
326 		IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
327 		xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs,
328 		      rpc->datasz);
329 	}
330 
331 	if (rmpp) {
332 		rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F);
333 		if ((rmpp->flags & 0x3) &&
334 		    mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) {
335 			IBWARN("bad rmpp version");
336 			return NULL;
337 		}
338 		rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F);
339 		rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F);
340 		DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status);
341 		rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F);
342 		rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F);
343 	}
344 
345 	if (data)
346 		memcpy(data, mad + rpc->dataoffs, rpc->datasz);
347 
348 	rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F);
349 
350 	return data;
351 }
352 
353 void *madrpc(ib_rpc_t * rpc, ib_portid_t * dport, void *payload, void *rcvdata)
354 {
355 	return mad_rpc(ibmp, rpc, dport, payload, rcvdata);
356 }
357 
358 void *madrpc_rmpp(ib_rpc_t * rpc, ib_portid_t * dport, ib_rmpp_hdr_t * rmpp,
359 		  void *data)
360 {
361 	return mad_rpc_rmpp(ibmp, rpc, dport, rmpp, data);
362 }
363 
364 void
365 madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes)
366 {
367 	int fd;
368 
369 	if (umad_init() < 0)
370 		IBPANIC("can't init UMAD library");
371 
372 	if ((fd = umad_open_port(dev_name, dev_port)) < 0)
373 		IBPANIC("can't open UMAD port (%s:%d)",
374 		dev_name ? dev_name : "(nil)", dev_port);
375 
376 	if (num_classes >= MAX_CLASS)
377 		IBPANIC("too many classes %d requested", num_classes);
378 
379 	ibmp->port_id = fd;
380 	memset(ibmp->class_agents, 0xff, sizeof ibmp->class_agents);
381 	while (num_classes--) {
382 		uint8_t rmpp_version = 0;
383 		int mgmt = *mgmt_classes++;
384 
385 		if (mgmt == IB_SA_CLASS)
386 			rmpp_version = 1;
387 		if (mad_register_client_via(mgmt, rmpp_version, ibmp) < 0)
388 			IBPANIC("client_register for mgmt class %d failed",
389 				mgmt);
390 	}
391 }
392 
393 struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port,
394 				     int *mgmt_classes, int num_classes)
395 {
396 	struct ibmad_port *p;
397 	int port_id;
398 
399 	if (num_classes >= MAX_CLASS) {
400 		IBWARN("too many classes %d requested", num_classes);
401 		errno = EINVAL;
402 		return NULL;
403 	}
404 
405 	if (umad_init() < 0) {
406 		IBWARN("can't init UMAD library");
407 		errno = ENODEV;
408 		return NULL;
409 	}
410 
411 	p = malloc(sizeof(*p));
412 	if (!p) {
413 		errno = ENOMEM;
414 		return NULL;
415 	}
416 	memset(p, 0, sizeof(*p));
417 
418 	if ((port_id = umad_open_port(dev_name, dev_port)) < 0) {
419 		IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port);
420 		if (!errno)
421 			errno = EIO;
422 		free(p);
423 		return NULL;
424 	}
425 
426 	p->port_id = port_id;
427 	if (dev_name)
428 		strncpy(p->ca_name, dev_name, sizeof p->ca_name - 1);
429 	p->portnum = dev_port;
430 
431 	memset(p->class_agents, 0xff, sizeof p->class_agents);
432 	while (num_classes--) {
433 		uint8_t rmpp_version = 0;
434 		int mgmt = *mgmt_classes++;
435 
436 		if (mgmt == IB_SA_CLASS)
437 			rmpp_version = 1;
438 		if (mgmt < 0 || mgmt >= MAX_CLASS ||
439 		    mad_register_client_via(mgmt, rmpp_version, p) < 0) {
440 			IBWARN("client_register for mgmt %d failed", mgmt);
441 			if (!errno)
442 				errno = EINVAL;
443 			umad_close_port(port_id);
444 			free(p);
445 			return NULL;
446 		}
447 	}
448 
449 	return p;
450 }
451 
452 void mad_rpc_close_port(struct ibmad_port *port)
453 {
454 	umad_close_port(port->port_id);
455 	free(port);
456 }
457