1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 /* AUTHOR Eitan Zahavi
37 *
38 * DESCRIPTION
39 * The lower-level MAD transport interface implementation
40 * that allows sending a single MAD/receiving a callback
41 * when a single MAD is received.
42 */
43
44 #if HAVE_CONFIG_H
45 # include <config.h>
46 #endif /* HAVE_CONFIG_H */
47
48 #include <sys/types.h>
49 #include <sys/stat.h>
50 #include <sys/ioctl.h>
51 #include <fcntl.h>
52 #include <errno.h>
53 #include <stdlib.h>
54 #include <string.h>
55
56 #include <vendor/osm_vendor_api.h>
57 #include <vendor/osm_vendor_mlx_transport.h>
58 #include <vendor/osm_vendor_mlx_dispatcher.h>
59 #include <vendor/osm_vendor_mlx_svc.h>
60 #include <complib/cl_thread.h>
61
62 /* the simulator messages definition */
63 #include <ibmgtsim/ibms_client_api.h>
64
65 typedef struct _osmv_ibms_transport_mgr {
66 ibms_conn_handle_t conHdl; /* the connection handle we talk to */
67 ibms_bind_msg_t filter; /* the bind message defining the filtering */
68 cl_thread_t receiver; /* the thread waiting for incomming messages */
69 } osmv_ibms_transport_mgr_t;
70
71 static void
72 __osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
73 IN struct _ibms_mad_addr *p_ibms_addr,
74 IN uint8_t is_smi,
75 OUT osm_mad_addr_t * p_osm_addr);
76
77 static void
78 __osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,
79 IN uint8_t is_smi,
80 OUT struct _ibms_mad_addr *p_ibms_addr);
81
82 /* this is the callback function the "server" will call on incoming
83 messages */
__osmv_ibms_receiver_callback(void * p_ctx,ibms_mad_msg_t * p_mad)84 void __osmv_ibms_receiver_callback(void *p_ctx, ibms_mad_msg_t * p_mad)
85 {
86 osm_mad_addr_t mad_addr;
87 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx;
88 ib_api_status_t status = IB_SUCCESS;
89
90 /* Make sure the p_bo object is still relevant */
91 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
92 return;
93
94 {
95 OSM_LOG_ENTER(p_bo->p_vendor->p_log);
96
97 /* some logging */
98 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
99 "__osmv_ibms_receiver_callback: "
100 "MAD QPN:%d SLID:0x%04x class:0x%02x "
101 "method:0x%02x attr:0x%04x status:0x%04x "
102 "tid:0x%016" PRIx64 "\n",
103 p_mad->addr.dqpn,
104 cl_ntoh16(p_mad->addr.slid),
105 p_mad->header.mgmt_class,
106 p_mad->header.method,
107 cl_ntoh16(p_mad->header.attr_id),
108 cl_ntoh16(p_mad->header.status),
109 cl_ntoh64(p_mad->header.trans_id));
110
111 /* first arrange an address */
112 __osmv_ibms_mad_addr_to_osm_addr(p_bo->p_vendor,
113 &p_mad->addr,
114 (((ib_mad_t *) & p_mad->
115 header)->mgmt_class ==
116 IB_MCLASS_SUBN_LID)
117 ||
118 (((ib_mad_t *) & p_mad->
119 header)->mgmt_class ==
120 IB_MCLASS_SUBN_DIR),
121 &mad_addr);
122
123 /* call the receiver callback */
124
125 status =
126 osmv_dispatch_mad((osm_bind_handle_t) p_bo,
127 (void *)&p_mad->header, &mad_addr);
128
129 OSM_LOG_EXIT(p_bo->p_vendor->p_log);
130 }
131 }
132
133 ib_api_status_t
134 osm_vendor_get_guid_by_ca_and_port(IN osm_vendor_t * const p_vend,
135 IN char *hca_id,
136 IN uint32_t port_num,
137 OUT uint64_t * p_port_guid);
138
139 /*
140 * NAME
141 * osmv_transport_init
142 *
143 * DESCRIPTION
144 * Setup the MAD transport infrastructure (filters, callbacks etc).
145 */
146
147 ib_api_status_t
osmv_transport_init(IN osm_bind_info_t * p_info,IN char hca_id[VENDOR_HCA_MAXNAMES],IN uint8_t hca_idx,IN osmv_bind_obj_t * p_bo)148 osmv_transport_init(IN osm_bind_info_t * p_info,
149 IN char hca_id[VENDOR_HCA_MAXNAMES],
150 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
151 {
152 ibms_conn_handle_t conHdl; /* the connection we talk to the simulator through */
153 osmv_ibms_transport_mgr_t *p_mgr =
154 malloc(sizeof(osmv_ibms_transport_mgr_t));
155 int qpn;
156 int ibms_status;
157 uint64_t port_guid;
158
159 if (!p_mgr) {
160 return IB_INSUFFICIENT_MEMORY;
161 }
162
163 memset(p_mgr, 0, sizeof(osmv_ibms_transport_mgr_t));
164
165 /* create the client socket connected to the simulator */
166 /* also perform the "connect" message - such that we
167 validate the target guid */
168 if (osm_vendor_get_guid_by_ca_and_port
169 (p_bo->p_vendor, hca_id, p_bo->port_num, &port_guid)) {
170 return IB_INVALID_GUID;
171 }
172
173 conHdl =
174 ibms_connect(port_guid, __osmv_ibms_receiver_callback,
175 (void *)p_bo);
176 if (!conHdl) {
177 printf("fail to connect to the server.\n");
178 exit(1);
179 }
180
181 /*
182 * Create the MAD filter on this file handle.
183 */
184
185 p_mgr->filter.port = p_bo->port_num;
186 p_mgr->filter.only_input = 1;
187 p_mgr->filter.mask =
188 IBMS_BIND_MASK_PORT |
189 IBMS_BIND_MASK_INPUT | IBMS_BIND_MASK_QP | IBMS_BIND_MASK_CLASS;
190
191 switch (p_info->mad_class) {
192 case IB_MCLASS_SUBN_LID:
193 case IB_MCLASS_SUBN_DIR:
194 qpn = 0;
195 p_mgr->filter.qpn = qpn;
196 p_mgr->filter.mgt_class = IB_MCLASS_SUBN_LID;
197 ibms_status = ibms_bind(conHdl, &p_mgr->filter);
198 if (ibms_status) {
199 return IB_ERROR;
200 }
201
202 p_mgr->filter.mgt_class = IB_MCLASS_SUBN_DIR;
203 ibms_status = ibms_bind(conHdl, &p_mgr->filter);
204 if (ibms_status) {
205 return IB_ERROR;
206 }
207
208 break;
209
210 case IB_MCLASS_SUBN_ADM:
211 default:
212 qpn = 1;
213 p_mgr->filter.qpn = qpn;
214 p_mgr->filter.mgt_class = p_info->mad_class;
215 ibms_status = ibms_bind(conHdl, &p_mgr->filter);
216 if (ibms_status) {
217 return IB_ERROR;
218 }
219 break;
220 }
221
222 p_mgr->conHdl = conHdl;
223
224 p_bo->p_transp_mgr = p_mgr;
225
226 /* Initialize the magic_ptr to the pointer of the p_bo info.
227 This will be used to signal when the object is being destroyed, so no
228 real action will be done then. */
229 p_bo->magic_ptr = p_bo;
230
231 return IB_SUCCESS;
232 }
233
234 /*
235 * NAME
236 * osmv_transport_send_mad
237 *
238 * DESCRIPTION
239 * Send a single MAD (256 byte)
240 */
241
242 ib_api_status_t
osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,IN void * p_mad,IN const osm_mad_addr_t * p_mad_addr)243 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
244 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr)
245 {
246
247 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
248 osm_vendor_t const *p_vend = p_bo->p_vendor;
249 int ret;
250 ibms_mad_msg_t mad_msg;
251 ib_api_status_t status;
252
253 const ib_mad_t *p_mad_hdr = p_mad;
254
255 OSM_LOG_ENTER(p_vend->p_log);
256
257 memset(&mad_msg, 0, sizeof(mad_msg));
258
259 /* Make sure the p_bo object is still relevant */
260 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
261 return IB_INVALID_CALLBACK;
262
263 /*
264 * Copy the MAD over to the sent mad
265 */
266 memcpy(&mad_msg.header, p_mad_hdr, MAD_BLOCK_SIZE);
267
268 /*
269 * For all sends other than directed route SM MADs,
270 * acquire an address vector for the destination.
271 */
272 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) {
273
274 __osmv_ibms_osm_addr_to_mad_addr(p_mad_addr,
275 p_mad_hdr->mgmt_class ==
276 IB_MCLASS_SUBN_LID,
277 &mad_msg.addr);
278 } else {
279 /* is a directed route - we need to construct a permissive address */
280 /* we do not need port number since it is part of the mad_hndl */
281 mad_msg.addr.dlid = IB_LID_PERMISSIVE;
282 mad_msg.addr.slid = IB_LID_PERMISSIVE;
283 mad_msg.addr.sqpn = 0;
284 mad_msg.addr.dqpn = 0;
285 }
286
287 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
288 "osmv_transport_mad_send: "
289 "Sending QPN:%d DLID:0x%04x class:0x%02x "
290 "method:0x%02x attr:0x%04x status:0x%04x "
291 "tid:0x%016" PRIx64 "\n",
292 mad_msg.addr.dqpn,
293 cl_ntoh16(mad_msg.addr.dlid),
294 mad_msg.header.mgmt_class,
295 mad_msg.header.method,
296 cl_ntoh16(mad_msg.header.attr_id),
297 cl_ntoh16(mad_msg.header.status),
298 cl_ntoh64(mad_msg.header.trans_id)
299 );
300
301 /* send it */
302 ret =
303 ibms_send(((osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr))->
304 conHdl, &mad_msg);
305 if (ret) {
306 osm_log(p_vend->p_log, OSM_LOG_ERROR,
307 "osmv_transport_mad_send: ERR 5304: "
308 "Error sending mad (%d).\n", ret);
309 status = IB_ERROR;
310 goto Exit;
311 }
312
313 status = IB_SUCCESS;
314
315 Exit:
316 OSM_LOG_EXIT(p_vend->p_log);
317 return (status);
318 }
319
osmv_transport_done(IN const osm_bind_handle_t h_bind)320 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
321 {
322 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
323 osmv_ibms_transport_mgr_t *p_tpot_mgr =
324 (osmv_ibms_transport_mgr_t *) (p_bo->p_transp_mgr);
325
326 CL_ASSERT(p_bo);
327
328 /* First of all - zero out the magic_ptr, so if a callback is called -
329 it'll know that we are currently closing down, and will not handle the
330 mad. */
331 p_bo->magic_ptr = 0;
332 /* usleep(3000000); */
333
334 ibms_disconnect(p_tpot_mgr->conHdl);
335
336 /* seems the only way to abort a blocking read is to make it read something */
337 free(p_tpot_mgr);
338 }
339
340 static void
__osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,IN uint8_t is_smi,OUT struct _ibms_mad_addr * p_ibms_addr)341 __osmv_ibms_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_osm_addr,
342 IN uint8_t is_smi,
343 OUT struct _ibms_mad_addr *p_ibms_addr)
344 {
345
346 /* For global destination or Multicast address: */
347 p_ibms_addr->dlid = cl_ntoh16(p_osm_addr->dest_lid);
348 p_ibms_addr->sl = p_osm_addr->addr_type.gsi.service_level;
349 if (is_smi) {
350 p_ibms_addr->sqpn = 0;
351 p_ibms_addr->dqpn = 0;
352 } else {
353 p_ibms_addr->sqpn = 1;
354 p_ibms_addr->dqpn =
355 cl_ntoh32(p_osm_addr->addr_type.gsi.remote_qp);
356 }
357 /*
358 HACK we limit to the first PKey Index assuming it will
359 always be the default PKey
360 */
361 p_ibms_addr->pkey_index = 0;
362 }
363
364 static void
__osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const * p_vend,IN struct _ibms_mad_addr * p_ibms_addr,IN uint8_t is_smi,OUT osm_mad_addr_t * p_osm_addr)365 __osmv_ibms_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend,
366 IN struct _ibms_mad_addr *p_ibms_addr,
367 IN uint8_t is_smi,
368 OUT osm_mad_addr_t * p_osm_addr)
369 {
370 memset(p_osm_addr, 0, sizeof(osm_mad_addr_t));
371 p_osm_addr->dest_lid = cl_hton16(p_ibms_addr->slid);
372 p_osm_addr->static_rate = 0;
373 p_osm_addr->path_bits = 0;
374 if (is_smi) {
375 /* SMI */
376 p_osm_addr->addr_type.smi.source_lid =
377 cl_hton16(p_ibms_addr->slid);
378 p_osm_addr->addr_type.smi.port_num = 1; /* TODO add if required p_ibms_addr->port; */
379 } else {
380 /* GSI */
381 p_osm_addr->addr_type.gsi.remote_qp =
382 cl_ntoh32(p_ibms_addr->sqpn);
383 p_osm_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
384 p_osm_addr->addr_type.gsi.pkey_ix = p_ibms_addr->pkey_index;
385 p_osm_addr->addr_type.gsi.service_level = p_ibms_addr->sl;
386
387 p_osm_addr->addr_type.gsi.global_route = FALSE;
388 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */
389 /*
390 if (p_osm_addr->addr_type.gsi.global_route)
391 {
392 p_osm_addr->addr_type.gsi.grh_info.ver_class_flow =
393 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version,
394 p_rcv_desc->grh.traffic_class,
395 p_rcv_desc->grh.flow_label);
396 p_osm_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit;
397 memcpy(&p_osm_addr->addr_type.gsi.grh_info.src_gid.raw,
398 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
399 memcpy(&p_osm_addr->addr_type.gsi.grh_info.dest_gid.raw,
400 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
401 }
402 */
403 }
404 }
405
406 /*
407 * NAME osm_vendor_set_sm
408 *
409 * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit
410 * according to the value given (TRUE or FALSE).
411 */
412
osm_vendor_set_sm(IN osm_bind_handle_t h_bind,IN boolean_t is_sm_val)413 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val)
414 {
415 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
416 osm_vendor_t const *p_vend = p_bo->p_vendor;
417 int ret;
418 ibms_cap_msg_t cap_msg;
419
420 OSM_LOG_ENTER(p_vend->p_log);
421
422 cap_msg.mask = IB_PORT_CAP_IS_SM;
423 if (is_sm_val)
424 cap_msg.capabilities = IB_PORT_CAP_IS_SM;
425 else
426 cap_msg.capabilities = 0;
427
428 ret = ibms_set_cap(((osmv_ibms_transport_mgr_t *) (p_bo->
429 p_transp_mgr))->
430 conHdl, &cap_msg);
431
432 if (ret) {
433 osm_log(p_vend->p_log, OSM_LOG_ERROR,
434 "osm_vendor_set_sm: ERR 5312: "
435 "Unable set 'IS_SM' bit to:%u in port attributes.\n",
436 is_sm_val);
437 }
438 OSM_LOG_EXIT(p_vend->p_log);
439 }
440