1 /*
2  * Copyright (c) 2006-2009 Voltaire, Inc. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  *
32  */
33 
34 #if HAVE_CONFIG_H
35 #  include <config.h>
36 #endif				/* HAVE_CONFIG_H */
37 
38 #include <stdio.h>
39 #include <string.h>
40 #include <errno.h>
41 #include <inttypes.h>
42 
43 #include <infiniband/umad.h>
44 #include <infiniband/mad.h>
45 #include <infiniband/iba/ib_types.h>
46 
47 #include "ibdiag_common.h"
48 
49 #define info(fmt, ...) fprintf(stderr, "INFO: " fmt, ## __VA_ARGS__ )
50 #define err(fmt, ...) fprintf(stderr, "ERR: " fmt, ## __VA_ARGS__ )
51 #ifdef NOISY_DEBUG
52 #define dbg(fmt, ...) fprintf(stderr, "DBG: " fmt, ## __VA_ARGS__ )
53 #else
54 #define dbg(fmt, ...)
55 #endif
56 
57 #define TMO 100
58 
59 static ibmad_gid_t mgid_ipoib = {
60 	0xff, 0x12, 0x40, 0x1b, 0xff, 0xff, 0x00, 0x00,
61 	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
62 };
63 
64 struct ibmad_port *srcport;
65 
66 uint64_t build_mcm_rec(uint8_t * data, ibmad_gid_t mgid, ibmad_gid_t port_gid)
67 {
68 	memset(data, 0, IB_SA_DATA_SIZE);
69 	mad_set_array(data, 0, IB_SA_MCM_MGID_F, mgid);
70 	mad_set_array(data, 0, IB_SA_MCM_PORTGID_F, port_gid);
71 	mad_set_field(data, 0, IB_SA_MCM_JOIN_STATE_F, 1);
72 
73 	return IB_MCR_COMPMASK_MGID | IB_MCR_COMPMASK_PORT_GID |
74 	    IB_MCR_COMPMASK_JOIN_STATE;
75 }
76 
77 static void build_mcm_rec_umad(void *umad, ib_portid_t * dport, int method,
78 			       uint64_t comp_mask, uint8_t * data)
79 {
80 	ib_rpc_t rpc;
81 
82 	memset(&rpc, 0, sizeof(rpc));
83 	rpc.mgtclass = IB_SA_CLASS;
84 	rpc.method = method;
85 	rpc.attr.id = IB_SA_ATTR_MCRECORD;
86 	rpc.attr.mod = 0;	// ???
87 	rpc.mask = comp_mask;
88 	rpc.datasz = IB_SA_DATA_SIZE;
89 	rpc.dataoffs = IB_SA_DATA_OFFS;
90 
91 	mad_build_pkt(umad, &rpc, dport, NULL, data);
92 }
93 
94 static int rereg_send(int port, int agent, ib_portid_t * dport,
95 		      uint8_t * umad, int len, int method, ibmad_gid_t port_gid)
96 {
97 	uint8_t data[IB_SA_DATA_SIZE];
98 	uint64_t comp_mask;
99 
100 	comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
101 
102 	build_mcm_rec_umad(umad, dport, method, comp_mask, data);
103 	if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
104 		err("umad_send %s failed: %s\n",
105 		    (method == IB_MAD_METHOD_GET) ? "query" : "non query",
106 		    strerror(errno));
107 		return -1;
108 	}
109 	dbg("umad_send %d: tid = 0x%016" PRIx64 "\n", method,
110 	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
111 
112 	return 0;
113 }
114 
115 static int rereg_port_gid(int port, int agent, ib_portid_t * dport,
116 			  uint8_t * umad, int len, ibmad_gid_t port_gid)
117 {
118 	uint8_t data[IB_SA_DATA_SIZE];
119 	uint64_t comp_mask;
120 
121 	comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
122 
123 	build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_DELETE, comp_mask, data);
124 	if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
125 		err("umad_send leave failed: %s\n", strerror(errno));
126 		return -1;
127 	}
128 	dbg("umad_send leave: tid = 0x%016" PRIx64 "\n",
129 	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
130 
131 	build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_SET, comp_mask, data);
132 	if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
133 		err("umad_send join failed: %s\n", strerror(errno));
134 		return -1;
135 	}
136 	dbg("umad_send join: tid = 0x%016" PRIx64 "\n",
137 	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
138 
139 	return 0;
140 }
141 
142 struct guid_trid {
143 	ibmad_gid_t gid;
144 	uint64_t guid;
145 	uint64_t trid;
146 };
147 
148 static int rereg_send_all(int port, int agent, ib_portid_t * dport,
149 			  struct guid_trid *list, unsigned cnt)
150 {
151 	uint8_t *umad;
152 	int len = umad_size() + 256;
153 	unsigned i;
154 	int ret;
155 
156 	info("rereg_send_all... cnt = %u\n", cnt);
157 
158 	umad = calloc(1, len);
159 	if (!umad) {
160 		err("cannot alloc mem for umad: %s\n", strerror(errno));
161 		return -1;
162 	}
163 
164 	for (i = 0; i < cnt; i++) {
165 		ret =
166 		    rereg_port_gid(port, agent, dport, umad, len, list[i].gid);
167 		if (ret < 0) {
168 			err("rereg_send_all: rereg_port_gid 0x%016" PRIx64
169 			    " failed\n", ntohll(list[i].guid));
170 			continue;
171 		}
172 		list[i].trid = mad_get_field64(umad_get_mad(umad), 0,
173 					       IB_MAD_TRID_F);
174 	}
175 
176 	info("rereg_send_all: sent %u requests\n", cnt * 2);
177 
178 	free(umad);
179 
180 	return 0;
181 }
182 
183 static int rereg_recv(int port, int agent, ib_portid_t * dport,
184 		      uint8_t * umad, int length, int tmo)
185 {
186 	int ret, retry = 0;
187 	int len = length;
188 
189 	while ((ret = umad_recv(port, umad, &len, tmo)) < 0 &&
190 	       errno == ETIMEDOUT) {
191 		if (retry++ > 3)
192 			return 0;
193 	}
194 	if (ret < 0) {
195 		err("umad_recv %d failed: %s\n", ret, strerror(errno));
196 		return -1;
197 	}
198 	dbg("umad_recv (retries %d), tid = 0x%016" PRIx64
199 	    ": len = %d, status = %d\n", retry,
200 	    mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), len,
201 	    umad_status(umad));
202 
203 	return 1;
204 }
205 
206 static int rereg_recv_all(int port, int agent, ib_portid_t * dport,
207 			  struct guid_trid *list, unsigned cnt)
208 {
209 	uint8_t *umad, *mad;
210 	int len = umad_size() + 256;
211 	uint64_t trid;
212 	unsigned n, method, status;
213 	unsigned i;
214 
215 	info("rereg_recv_all...\n");
216 
217 	umad = calloc(1, len);
218 	if (!umad) {
219 		err("cannot alloc mem for umad: %s\n", strerror(errno));
220 		return -1;
221 	}
222 
223 	n = 0;
224 	while (rereg_recv(port, agent, dport, umad, len, TMO) > 0) {
225 		dbg("rereg_recv_all: done %d\n", n);
226 		n++;
227 		mad = umad_get_mad(umad);
228 
229 		method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
230 		status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
231 
232 		if (status)
233 			dbg("MAD status %x, method %x\n", status, method);
234 
235 		if (status &&
236 		    (method & 0x7f) == (IB_MAD_METHOD_GET_RESPONSE & 0x7f)) {
237 			trid = mad_get_field64(mad, 0, IB_MAD_TRID_F);
238 			for (i = 0; i < cnt; i++)
239 				if (trid == list[i].trid)
240 					break;
241 			if (i == cnt) {
242 				err("cannot find trid 0x%016" PRIx64 "\n",
243 				    trid);
244 				continue;
245 			}
246 			info("guid 0x%016" PRIx64
247 			     ": method = %x status = %x. Resending\n",
248 			     ntohll(list[i].guid), method, status);
249 			rereg_port_gid(port, agent, dport, umad, len,
250 				       list[i].gid);
251 			list[i].trid =
252 			    mad_get_field64(umad_get_mad(umad), 0,
253 					    IB_MAD_TRID_F);
254 		}
255 	}
256 
257 	info("rereg_recv_all: got %u responses\n", n);
258 
259 	free(umad);
260 	return 0;
261 }
262 
263 static int rereg_query_all(int port, int agent, ib_portid_t * dport,
264 			   struct guid_trid *list, unsigned cnt)
265 {
266 	uint8_t *umad, *mad;
267 	int len = umad_size() + 256;
268 	unsigned method, status;
269 	unsigned i;
270 	int ret;
271 
272 	info("rereg_query_all...\n");
273 
274 	umad = calloc(1, len);
275 	if (!umad) {
276 		err("cannot alloc mem for umad: %s\n", strerror(errno));
277 		return -1;
278 	}
279 
280 	for (i = 0; i < cnt; i++) {
281 		ret = rereg_send(port, agent, dport, umad, len,
282 				 IB_MAD_METHOD_GET, list[i].gid);
283 		if (ret < 0) {
284 			err("query_all: rereg_send failed.\n");
285 			continue;
286 		}
287 
288 		ret = rereg_recv(port, agent, dport, umad, len, TMO);
289 		if (ret < 0) {
290 			err("query_all: rereg_recv failed.\n");
291 			continue;
292 		}
293 
294 		mad = umad_get_mad(umad);
295 
296 		method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
297 		status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
298 
299 		if (status)
300 			info("guid 0x%016" PRIx64 ": status %x, method %x\n",
301 			     ntohll(list[i].guid), status, method);
302 	}
303 
304 	info("rereg_query_all: %u queried.\n", cnt);
305 
306 	free(umad);
307 	return 0;
308 }
309 
310 #define MAX_CLIENTS 50
311 
312 static int rereg_and_test_port(char *guid_file, int port, int agent,
313 			       ib_portid_t * dport, int timeout)
314 {
315 	char line[256];
316 	FILE *f;
317 	ibmad_gid_t port_gid;
318 	uint64_t prefix = htonll(0xfe80000000000000ull);
319 	uint64_t guid = htonll(0x0002c90200223825ull);
320 	struct guid_trid *list;
321 	int i = 0;
322 
323 	list = calloc(MAX_CLIENTS, sizeof(*list));
324 	if (!list) {
325 		err("cannot alloc mem for guid/trid list: %s\n",
326 		    strerror(errno));
327 		return -1;
328 	}
329 
330 	f = fopen(guid_file, "r");
331 	if (!f) {
332 		err("cannot open %s: %s\n", guid_file, strerror(errno));
333 		return -1;
334 	}
335 
336 	while (fgets(line, sizeof(line), f)) {
337 		guid = strtoull(line, NULL, 0);
338 		guid = htonll(guid);
339 		memcpy(&port_gid[0], &prefix, 8);
340 		memcpy(&port_gid[8], &guid, 8);
341 
342 		list[i].guid = guid;
343 		memcpy(list[i].gid, port_gid, sizeof(list[i].gid));
344 		list[i].trid = 0;
345 		if (++i >= MAX_CLIENTS)
346 			break;
347 	}
348 	fclose(f);
349 
350 	rereg_send_all(port, agent, dport, list, i);
351 	rereg_recv_all(port, agent, dport, list, i);
352 
353 	rereg_query_all(port, agent, dport, list, i);
354 
355 	free(list);
356 	return 0;
357 }
358 
359 int main(int argc, char **argv)
360 {
361 	char *guid_file = "port_guids.list";
362 	int mgmt_classes[2] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS };
363 	ib_portid_t dport_id;
364 	int port, agent;
365 	uint8_t *umad, *mad;
366 	int len;
367 
368 	if (argc > 1)
369 		guid_file = argv[1];
370 
371 	srcport = mad_rpc_open_port(NULL, 0, mgmt_classes, 2);
372 	if (!srcport)
373 		err("Failed to open port");
374 
375 	resolve_sm_portid(NULL, 0, &dport_id);
376 	dport_id.qp = 1;
377 	if (!dport_id.qkey)
378 		dport_id.qkey = IB_DEFAULT_QP1_QKEY;
379 
380 	len = umad_size() + 256;
381 	umad = calloc(1, len);
382 	if (!umad) {
383 		err("cannot alloc mem for umad: %s\n", strerror(errno));
384 		return -1;
385 	}
386 	port = mad_rpc_portid(srcport);
387 
388 	agent = umad_register(port, IB_SA_CLASS, 2, 0, NULL);
389 
390 	rereg_and_test_port(guid_file, port, agent, &dport_id, TMO);
391 	mad = umad_get_mad(umad);
392 
393 	free(umad);
394 	umad_unregister(port, agent);
395 	umad_close_port(port);
396 	umad_done();
397 
398 	return 0;
399 }
400