1 /*
2  * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4  * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  *
34  */
35 
36 /*  AUTHOR                 Edward Bortnikov
37  *
38  *  DESCRIPTION
39  *     The lower-level MAD transport interface implementation
40  *     that allows sending a single MAD/receiving a callback
41  *     when a single MAD is received.
42  */
43 
44 #if HAVE_CONFIG_H
45 #  include <config.h>
46 #endif				/* HAVE_CONFIG_H */
47 
48 #include <sys/types.h>
49 #include <sys/stat.h>
50 #include <sys/ioctl.h>
51 #include <fcntl.h>
52 #include <stddef.h>
53 #include <errno.h>
54 #include <stdlib.h>
55 #include <string.h>
56 
57 #include <vendor/osm_vendor_api.h>
58 #include <vendor/osm_vendor_mlx_transport.h>
59 #include <vendor/osm_vendor_mlx_transport_anafa.h>
60 #include <vendor/osm_vendor_mlx_dispatcher.h>
61 #include <vendor/osm_vendor_mlx_svc.h>
62 #include <vendor/osm_ts_useraccess.h>
63 
64 static void
65 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
66 					  IN struct ib_mad *p_mad,
67 					  IN uint8_t is_smi,
68 					  OUT osm_mad_addr_t * p_mad_addr);
69 
70 static void
71 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t *
72 					  p_mad_addr, IN uint8_t is_smi,
73 					  OUT struct ib_mad *p_mad);
74 
75 void __osmv_TOPSPIN_ANAFA_receiver_thr(void *p_ctx)
76 {
77 	int ts_ret_code;
78 	struct ib_mad mad;
79 	osm_mad_addr_t mad_addr;
80 	osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
81 	ib_api_status_t status = IB_SUCCESS;
82 
83 	OSM_LOG_ENTER(p_bo->p_vendor->p_log);
84 
85 	/* Make sure the p_bo object is still relevant */
86 	if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
87 		return;
88 
89 	/* we set the type of cancelation for this thread */
90 	/* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */
91 
92 	while (1) {
93 		/* Make sure the p_bo object is still relevant */
94 		if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
95 			return;
96 
97 		/* we read one mad at a time and pass it to the read callback function */
98 		ts_ret_code =
99 		    read(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
100 								  p_transp_mgr))->
101 			 device_fd, &mad, sizeof(mad));
102 
103 		/* Make sure the p_bo object is still relevant */
104 		if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
105 			return;
106 
107 		if (ts_ret_code != sizeof(mad)) {
108 			osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
109 				"__osmv_TOPSPIN_ANAFA_receiver_thr: ERR 6903: "
110 				"error with read, bytes = %d\n", ts_ret_code);
111 			break;
112 		} else {
113 			osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
114 				"__osmv_TOPSPIN_ANAFA_receiver_thr: "
115 				"MAD QPN:%d SLID:0x%04x class:0x%02x "
116 				"method:0x%02x attr:0x%04x status:0x%04x "
117 				"tid:0x%016" PRIx64 "\n",
118 				mad.dqpn,
119 				cl_ntoh16(mad.slid),
120 				mad.mgmt_class,
121 				mad.r_method,
122 				cl_ntoh16(mad.attribute_id),
123 				cl_ntoh16(mad.status),
124 				cl_ntoh64(mad.transaction_id));
125 
126 			/* first arrange an address */
127 			__osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr
128 			    (p_bo->p_vendor, &mad,
129 			     (((ib_mad_t *) & mad)->mgmt_class ==
130 			      IB_MCLASS_SUBN_LID)
131 			     || (((ib_mad_t *) & mad)->mgmt_class ==
132 				 IB_MCLASS_SUBN_DIR), &mad_addr);
133 
134 			/* call the receiver callback */
135 
136 			status =
137 			    osmv_dispatch_mad((osm_bind_handle_t) p_bo,
138 					      (void *)&mad, &mad_addr);
139 
140 			/* Make sure the p_bo object is still relevant */
141 			if (p_bo->magic_ptr != p_bo)
142 				return;
143 
144 			if (IB_INTERRUPTED == status) {
145 
146 				osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
147 					"__osmv_TOPSPIN_ANAFA_receiver_thr: "
148 					"The bind handle %p is being closed. "
149 					"Breaking the loop.\n", p_bo);
150 				break;
151 			}
152 		}
153 	}
154 
155 	OSM_LOG_EXIT(p_bo->p_vendor->p_log);
156 }
157 
158 /*
159  * NAME
160  *   osmv_transport_init
161  *
162  * DESCRIPTION
163  *   Setup the MAD transport infrastructure (filters, callbacks etc).
164  */
165 
166 ib_api_status_t
167 osmv_transport_init(IN osm_bind_info_t * p_info,
168 		    IN char hca_id[VENDOR_HCA_MAXNAMES],
169 		    IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
170 {
171 	cl_status_t cl_st;
172 
173 	int ts_ioctl_ret;
174 	int device_fd;
175 	char *device_file = "/dev/ts_ua0";
176 	osm_ts_user_mad_filter filter;
177 	osmv_TOPSPIN_ANAFA_transport_mgr_t *p_mgr;
178 	osmv_TOPSPIN_ANAFA_transport_info_t *p_tpot_info;
179 	p_tpot_info =
180 	    (osmv_TOPSPIN_ANAFA_transport_info_t *) p_bo->p_vendor->
181 	    p_transport_info;
182 
183 	p_mgr = malloc(sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
184 	if (!p_mgr) {
185 		return IB_INSUFFICIENT_MEMORY;
186 	}
187 
188 	memset(p_mgr, 0, sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t));
189 
190 	/* open TopSpin file device */
191 	device_fd = open(device_file, O_RDWR);
192 	if (device_fd < 0) {
193 		fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n",
194 			device_file, errno);
195 		return IB_ERROR;
196 	}
197 	p_mgr->device_fd = device_fd;
198 
199 	/*
200 	 * Create the MAD filter on this file handle.
201 	 */
202 
203 	filter.port = 0;	/* Victor */
204 	filter.direction = TS_IB_MAD_DIRECTION_IN;
205 	filter.mask =
206 	    TS_IB_MAD_FILTER_DIRECTION |
207 	    TS_IB_MAD_FILTER_PORT |
208 	    TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS;
209 
210 	switch (p_info->mad_class) {
211 	case IB_MCLASS_SUBN_LID:
212 	case IB_MCLASS_SUBN_DIR:
213 		filter.qpn = 0;
214 		filter.mgmt_class = IB_MCLASS_SUBN_LID;
215 		ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
216 		if (ts_ioctl_ret < 0) {
217 			return IB_ERROR;
218 		}
219 
220 		filter.mgmt_class = IB_MCLASS_SUBN_DIR;
221 		ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
222 		if (ts_ioctl_ret < 0) {
223 			return IB_ERROR;
224 		}
225 
226 		break;
227 
228 	case IB_MCLASS_SUBN_ADM:
229 	default:
230 		filter.qpn = 1;
231 		filter.mgmt_class = p_info->mad_class;
232 		ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter);
233 		if (ts_ioctl_ret < 0) {
234 			return IB_ERROR;
235 		}
236 		break;
237 	}
238 
239 	p_bo->p_transp_mgr = p_mgr;
240 
241 	/* Initialize the magic_ptr to the pointer of the p_bo info.
242 	   This will be used to signal when the object is being destroyed, so no
243 	   real action will be done then. */
244 	p_bo->magic_ptr = p_bo;
245 
246 	/* init receiver thread */
247 	cl_st =
248 	    cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_ANAFA_receiver_thr,
249 			   (void *)p_bo, "osmv TOPSPIN_ANAFA rcv thr");
250 
251 	return (ib_api_status_t) cl_st;
252 }
253 
254 /*
255  * NAME
256  *   osmv_transport_send_mad
257  *
258  * DESCRIPTION
259  *   Send a single MAD (256 byte)
260  */
261 
262 ib_api_status_t
263 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
264 			IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
265 {
266 
267 	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
268 	osm_vendor_t const *p_vend = p_bo->p_vendor;
269 	struct ib_mad ts_mad = { 0 };
270 	int ret;
271 	ib_api_status_t status;
272 
273 	const ib_mad_t *p_mad_hdr = p_mad;
274 
275 	OSM_LOG_ENTER(p_vend->p_log);
276 
277 	/* Make sure the p_bo object is still relevant */
278 	if (p_bo->magic_ptr != p_bo)
279 		return IB_INVALID_CALLBACK;
280 
281 	/*
282 	 * Copy the MAD over to the sent mad
283 	 */
284 	memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE);
285 
286 	/*
287 	 * For all sends other than directed route SM MADs,
288 	 * acquire an address vector for the destination.
289 	 */
290 	if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
291 
292 		__osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(p_mad_addr,
293 							  p_mad_hdr->
294 							  mgmt_class ==
295 							  IB_MCLASS_SUBN_LID,
296 							  &ts_mad);
297 	} else {
298 		/* is a directed route - we need to construct a permissive address */
299 		/* we do not need port number since it is part of the mad_hndl */
300 		ts_mad.dlid = IB_LID_PERMISSIVE;
301 		ts_mad.slid = IB_LID_PERMISSIVE;
302 	}
303 	if ((p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_DIR) ||
304 	    (p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_LID)) {
305 		ts_mad.sqpn = 0;
306 		ts_mad.dqpn = 0;
307 	} else {
308 		ts_mad.sqpn = 1;
309 		ts_mad.dqpn = 1;
310 	}
311 
312 	/* ts_mad.port = p_bo->port_num; */
313 	ts_mad.port = 0;	/* Victor */
314 
315 	/* send it */
316 	ret =
317 	    write(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->
318 							   p_transp_mgr))->
319 		  device_fd, &ts_mad, sizeof(ts_mad));
320 
321 	if (ret != sizeof(ts_mad)) {
322 		osm_log(p_vend->p_log, OSM_LOG_ERROR,
323 			"osmv_transport_mad_send: ERR 6904: "
324 			"Error sending mad (%d).\n", ret);
325 		status = IB_ERROR;
326 		goto Exit;
327 	}
328 
329 	status = IB_SUCCESS;
330 
331 Exit:
332 	OSM_LOG_EXIT(p_vend->p_log);
333 	return (status);
334 }
335 
336 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
337 {
338 	osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
339 	osmv_TOPSPIN_ANAFA_transport_mgr_t *p_tpot_mgr =
340 	    (osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->p_transp_mgr);
341 
342 	CL_ASSERT(p_bo);
343 
344 	/* First of all - zero out the magic_ptr, so if a callback is called -
345 	   it'll know that we are currently closing down, and will not handle the
346 	   mad. */
347 	p_bo->magic_ptr = 0;
348 
349 	/* usleep(3000000); */
350 
351 	/* pthread_cancel (p_tpot_mgr->receiver.osd.id); */
352 	cl_thread_destroy(&(p_tpot_mgr->receiver));
353 	free(p_tpot_mgr);
354 }
355 
356 static void
357 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr,
358 					  IN uint8_t is_smi,
359 					  OUT struct ib_mad *p_mad)
360 {
361 
362 	/* For global destination or Multicast address: */
363 	p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid);
364 	p_mad->sl = p_mad_addr->addr_type.gsi.service_level;
365 	if (is_smi) {
366 		p_mad->sqpn = 0;
367 		p_mad->dqpn = 0;
368 	} else {
369 		p_mad->sqpn = 1;
370 		p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp;
371 	}
372 	/*
373 	   HACK we limit to the first PKey Index assuming it will
374 	   always be the default PKey
375 	 */
376 	p_mad->pkey_index = 0;
377 }
378 
379 static void
380 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
381 					  IN struct ib_mad *p_mad,
382 					  IN uint8_t is_smi,
383 					  OUT osm_mad_addr_t * p_mad_addr)
384 {
385 	p_mad_addr->dest_lid = cl_hton16(p_mad->slid);
386 	p_mad_addr->static_rate = 0;
387 	p_mad_addr->path_bits = 0;
388 	if (is_smi) {
389 		/* SMI */
390 		p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid);
391 		p_mad_addr->addr_type.smi.port_num = p_mad->port;
392 	} else {
393 		/* GSI */
394 		p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn;
395 		p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
396 		p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index;
397 		p_mad_addr->addr_type.gsi.service_level = p_mad->sl;
398 
399 		p_mad_addr->addr_type.gsi.global_route = FALSE;
400 		/* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
401 		/*
402 		   if (p_mad_addr->addr_type.gsi.global_route)
403 		   {
404 		   p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
405 		   ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
406 		   p_rcv_desc->grh.traffic_class,
407 		   p_rcv_desc->grh.flow_label);
408 		   p_mad_addr->addr_type.gsi.grh_info.hop_limit =  p_rcv_desc->grh.hop_limit;
409 		   memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
410 		   &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
411 		   memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
412 		   p_rcv_desc->grh.dgid,  sizeof(ib_net64_t));
413 		   }
414 		 */
415 	}
416 }
417