1 /*
2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved.
3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved.
4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 */
35
36 /* AUTHOR Edward Bortnikov
37 *
38 * DESCRIPTION
39 * The lower-level MAD transport interface implementation
40 * that allows sending a single MAD/receiving a callback
41 * when a single MAD is received.
42 */
43
44 #if HAVE_CONFIG_H
45 # include <config.h>
46 #endif /* HAVE_CONFIG_H */
47
48 #include <stdlib.h>
49 #include <string.h>
50 #include <ib_mgt.h>
51 #include <complib/cl_event.h>
52 #include <vendor/osm_vendor_mlx_transport.h>
53 #include <vendor/osm_vendor_mlx_dispatcher.h>
54 #include <opensm/osm_log.h>
55
56 typedef struct _osmv_IBMGT_transport_mgr_ {
57 IB_MGT_mad_type_t mad_type;
58 uint8_t mgmt_class; /* for gsi */
59 /* for communication between send call back and send mad */
60 boolean_t is_send_ok;
61 cl_event_t send_done;
62 } osmv_IBMGT_transport_mgr_t;
63
64 typedef struct _osmv_IBMGT_transport_info_ {
65 IB_MGT_mad_hndl_t smi_h;
66 cl_qlist_t *p_smi_list;
67
68 IB_MGT_mad_hndl_t gsi_h;
69 /* holds bind object list for every binded mgmt class */
70 cl_qlist_t *gsi_mgmt_lists[15];
71 } osmv_IBMGT_transport_info_t;
72
73 static void
74 __osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
75 IN uint8_t is_smi,
76 OUT osm_mad_addr_t * p_mad_addr);
77
78 static void
79 __osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr,
80 IN uint8_t is_smi, OUT IB_ud_av_t * p_av);
81
82 void
83 __osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl,
84 IN u_int64_t wrid,
85 IN IB_comp_status_t status, IN void *private_ctx_p);
86
87 void
88 __osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl,
89 IN void *private_ctx_p,
90 IN void *payload_p,
91 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p);
92
93 /*
94 * NAME
95 * osmv_transport_init
96 *
97 * DESCRIPTION
98 * Setup the MAD transport infrastructure (filters, callbacks etc).
99 */
100
101 ib_api_status_t
osmv_transport_init(IN osm_bind_info_t * p_info,IN char hca_id[VENDOR_HCA_MAXNAMES],IN uint8_t hca_idx,IN osmv_bind_obj_t * p_bo)102 osmv_transport_init(IN osm_bind_info_t * p_info,
103 IN char hca_id[VENDOR_HCA_MAXNAMES],
104 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo)
105 {
106 ib_api_status_t st = IB_SUCCESS;
107 IB_MGT_ret_t ret;
108 IB_MGT_mad_type_t mad_type;
109 osmv_IBMGT_transport_mgr_t *p_mgr;
110 osmv_IBMGT_transport_info_t *p_tpot_info;
111 cl_list_obj_t *p_obj = NULL;
112 osm_log_t *p_log = p_bo->p_vendor->p_log;
113 int i;
114
115 UNUSED_PARAM(hca_idx);
116
117 /* if first bind, allocate tranport_info at vendor */
118 if (NULL == p_bo->p_vendor->p_transport_info) {
119 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
120 "osmv_transport_init: first bind() for the vendor\n");
121 p_bo->p_vendor->p_transport_info
122 = (osmv_IBMGT_transport_info_t *)
123 malloc(sizeof(osmv_IBMGT_transport_info_t));
124 if (NULL == p_bo->p_vendor->p_transport_info) {
125 return IB_INSUFFICIENT_MEMORY;
126 }
127 memset(p_bo->p_vendor->p_transport_info, 0,
128 sizeof(osmv_IBMGT_transport_info_t));
129 p_tpot_info =
130 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->
131 p_transport_info);
132
133 p_tpot_info->smi_h = 0xffffffff;
134 p_tpot_info->p_smi_list = NULL;
135
136 p_tpot_info->gsi_h = 0xffffffff;
137 for (i = 0; i < 15; i++) {
138
139 p_tpot_info->gsi_mgmt_lists[i] = NULL;
140 }
141
142 } else {
143
144 p_tpot_info =
145 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->
146 p_transport_info);
147 }
148
149 /* Initialize the magic_ptr to the pointer of the p_bo info.
150 This will be used to signal when the object is being destroyed, so no
151 real action will be done then. */
152 p_bo->magic_ptr = p_bo;
153
154 /* allocate transport mgr */
155 p_mgr = malloc(sizeof(osmv_IBMGT_transport_mgr_t));
156 if (NULL == p_mgr) {
157 free(p_tpot_info);
158 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
159 "osmv_transport_init: ERR 7201: " "alloc failed \n");
160 return IB_INSUFFICIENT_MEMORY;
161 }
162
163 memset(p_mgr, 0, sizeof(osmv_IBMGT_transport_mgr_t));
164
165 p_bo->p_transp_mgr = p_mgr;
166
167 switch (p_info->mad_class) {
168 case IB_MCLASS_SUBN_LID:
169 case IB_MCLASS_SUBN_DIR:
170 mad_type = IB_MGT_SMI;
171 break;
172
173 case IB_MCLASS_SUBN_ADM:
174 default:
175 mad_type = IB_MGT_GSI;
176 break;
177 }
178
179 /* we only support one class registration per SMI/GSI !!! */
180 switch (mad_type) {
181 case IB_MGT_SMI:
182 /* we do not need to bind the handle if already available */
183 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
184 "osmv_transport_init: SMI bind\n");
185
186 if (p_tpot_info->smi_h == 0xffffffff) {
187 ret = IB_MGT_get_handle(hca_id,
188 p_bo->port_num,
189 IB_MGT_SMI,
190 &(p_tpot_info->smi_h));
191 if (IB_MGT_OK != ret) {
192 osm_log(p_log, OSM_LOG_ERROR,
193 "osmv_transport_init: ERR 7202: "
194 "IB_MGT_get_handle for smi failed \n");
195 st = IB_ERROR;
196 free(p_mgr);
197 goto Exit;
198 }
199
200 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
201 "osmv_transport_init: got smi handle:%d \n",
202 p_tpot_info->smi_h);
203
204 ret = IB_MGT_bind_sm(p_tpot_info->smi_h);
205 if (IB_MGT_OK != ret) {
206 osm_log(p_log, OSM_LOG_ERROR,
207 "osmv_transport_init: ERR 7203: "
208 "IB_MGT_bind_sm failed \n");
209 st = IB_ERROR;
210 free(p_mgr);
211 goto Exit;
212 }
213
214 /* init smi list */
215 p_tpot_info->p_smi_list = malloc(sizeof(cl_qlist_t));
216 if (NULL == p_tpot_info->p_smi_list) {
217 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
218 "osmv_transport_init: ERR 7204: "
219 "alloc failed \n");
220 IB_MGT_unbind_sm(p_tpot_info->smi_h);
221 IB_MGT_release_handle(p_tpot_info->smi_h);
222 free(p_mgr);
223 return IB_INSUFFICIENT_MEMORY;
224 }
225 memset(p_tpot_info->p_smi_list, 0, sizeof(cl_qlist_t));
226 cl_qlist_init(p_tpot_info->p_smi_list);
227
228 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
229 "osmv_transport_init: before reg_cb\n");
230 ret = IB_MGT_reg_cb(p_tpot_info->smi_h,
231 &__osmv_IBMGT_rcv_cb,
232 p_bo,
233 &__osmv_IBMGT_send_cb,
234 p_tpot_info->p_smi_list,
235 IB_MGT_RCV_CB_MASK |
236 IB_MGT_SEND_CB_MASK);
237 if (ret != IB_SUCCESS) {
238 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
239 "osmv_transport_init: ERR 7205: "
240 "reg_cb failed with return code:%x \n",
241 ret);
242 IB_MGT_unbind_sm(p_tpot_info->smi_h);
243 IB_MGT_release_handle(p_tpot_info->smi_h);
244 free(p_tpot_info->p_smi_list);
245 free(p_mgr);
246 st = IB_ERROR;
247 goto Exit;
248 }
249
250 }
251 /* insert to list of smi's - for raising callbacks later on */
252 p_obj = malloc(sizeof(cl_list_obj_t));
253 if (p_obj)
254 memset(p_obj, 0, sizeof(cl_list_obj_t));
255 cl_qlist_set_obj(p_obj, p_bo);
256 cl_qlist_insert_tail(p_tpot_info->p_smi_list,
257 &p_obj->list_item);
258
259 break;
260
261 case IB_MGT_GSI:
262 /* we do not need to bind the handle if already available */
263 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
264 "osmv_transport_init: ERR 7206: GSI bind\n");
265 if (p_tpot_info->gsi_h == 0xffffffff) {
266 ret = IB_MGT_get_handle(hca_id,
267 p_bo->port_num,
268 IB_MGT_GSI,
269 &(p_tpot_info->gsi_h));
270 if (IB_MGT_OK != ret) {
271 osm_log(p_log, OSM_LOG_ERROR,
272 "osmv_transport_init: ERR 7207: "
273 "IB_MGT_get_handle for gsi failed \n");
274 st = IB_ERROR;
275 free(p_mgr);
276 goto Exit;
277 }
278 }
279
280 /* this mgmt class was not binded yet */
281 if (p_tpot_info->gsi_mgmt_lists[p_info->mad_class] == NULL) {
282 ret =
283 IB_MGT_bind_gsi_class(p_tpot_info->gsi_h,
284 p_info->mad_class);
285 if (IB_MGT_OK != ret) {
286 osm_log(p_log, OSM_LOG_ERROR,
287 "osmv_transport_init: ERR 7208: "
288 "IB_MGT_bind_gsi_class failed \n");
289 st = IB_ERROR;
290 free(p_mgr);
291 goto Exit;
292 }
293
294 p_tpot_info->gsi_mgmt_lists[p_info->mad_class] =
295 malloc(sizeof(cl_qlist_t));
296 if (NULL ==
297 p_tpot_info->gsi_mgmt_lists[p_info->mad_class]) {
298 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h,
299 p_info->mad_class);
300 free(p_mgr);
301 return IB_INSUFFICIENT_MEMORY;
302 }
303 memset(p_tpot_info->gsi_mgmt_lists[p_info->mad_class],
304 0, sizeof(cl_qlist_t));
305 cl_qlist_init(p_tpot_info->
306 gsi_mgmt_lists[p_info->mad_class]);
307 }
308 /* insert to list of smi's - for raising callbacks later on */
309 p_obj = malloc(sizeof(cl_list_obj_t));
310 if (p_obj)
311 memset(p_obj, 0, sizeof(cl_list_obj_t));
312 cl_qlist_set_obj(p_obj, p_bo);
313 cl_qlist_insert_tail(p_tpot_info->
314 gsi_mgmt_lists[p_info->mad_class],
315 &p_obj->list_item);
316
317 p_mgr->mgmt_class = p_info->mad_class;
318 ret = IB_MGT_reg_cb(p_tpot_info->gsi_h,
319 &__osmv_IBMGT_rcv_cb,
320 p_bo,
321 &__osmv_IBMGT_send_cb,
322 p_bo,
323 IB_MGT_RCV_CB_MASK | IB_MGT_SEND_CB_MASK);
324
325 if (ret != IB_SUCCESS) {
326 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h,
327 p_mgr->mgmt_class);
328 free(p_tpot_info->gsi_mgmt_lists[p_mgr->mgmt_class]);
329 free(p_mgr);
330 st = IB_ERROR;
331 goto Exit;
332 }
333
334 break;
335
336 default:
337 osm_log(p_log, OSM_LOG_ERROR,
338 "osmv_transport_init: ERR 7209: unrecognized mgmt class \n");
339 st = IB_ERROR;
340 free(p_mgr);
341 goto Exit;
342 }
343
344 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG,
345 "osmv_transport_init: GSI bind\n");
346 cl_event_construct(&p_mgr->send_done);
347 cl_event_init(&p_mgr->send_done, TRUE);
348 p_mgr->is_send_ok = FALSE;
349 p_mgr->mad_type = mad_type;
350
351 Exit:
352 /* OSM_LOG_EXIT(p_log ); */
353 return (ib_api_status_t) st;
354 }
355
356 /*
357 * NAME
358 * osmv_transport_send_mad
359 *
360 * DESCRIPTION
361 * Send a single MAD (256 byte)
362 */
363
364 ib_api_status_t
osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,IN void * p_ib_mad,IN const osm_mad_addr_t * p_mad_addr)365 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind,
366 IN void *p_ib_mad, IN const osm_mad_addr_t * p_mad_addr)
367 {
368
369 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
370 osmv_IBMGT_transport_info_t *p_tpot_info =
371 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->p_transport_info);
372 osm_vendor_t const *p_vend = p_bo->p_vendor;
373 ib_api_status_t status;
374 IB_ud_av_t av;
375 IB_MGT_ret_t ret;
376 ib_mad_t *p_mad = p_ib_mad;
377
378 OSM_LOG_ENTER(p_vend->p_log);
379
380 CL_ASSERT(p_bo->p_vendor->p_transport_info);
381
382 /*
383 * For all sends other than directed route SM MADs,
384 * acquire an address vector for the destination.
385 */
386 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) {
387 __osmv_IBMGT_osm_addr_to_ibmgt_addr(p_mad_addr,
388 p_mad->mgmt_class ==
389 IB_MCLASS_SUBN_LID, &av);
390 } else {
391 /* is a directed route - we need to construct a permissive address */
392 memset(&av, 0, sizeof(av));
393 /* we do not need port number since it is part of the mad_hndl */
394 av.dlid = IB_LID_PERMISSIVE;
395 }
396
397 /* send it */
398 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
399 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
400
401 /* SMI CASE */
402 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
403 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
404 "osmv_transport_mad_send: "
405 "av.dlid:0x%X, "
406 "av.static_rate:%d, "
407 "av.path_bits:%d.\n",
408 cl_ntoh16(av.dlid), av.static_rate,
409 av.src_path_bits);
410 }
411
412 ret = IB_MGT_send_mad(p_tpot_info->smi_h, p_mad, /* actual payload */
413 &av, /* address vector */
414 (u_int64_t) CAST_P2LONG(p_bo),
415 IB_MGT_DEFAULT_SEND_TIME);
416 } else {
417 /* GSI CASE - Support Remote QP */
418 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
419 osm_log(p_vend->p_log, OSM_LOG_DEBUG,
420 "osmv_transport_mad_send: "
421 "av.dlid:0x%X, av.static_rate:%d, av.path_bits:%d, remote qp:%d \n",
422 cl_ntoh16(av.dlid), av.static_rate,
423 av.src_path_bits,
424 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp)
425 );
426 }
427
428 ret = IB_MGT_send_mad_to_qp(p_tpot_info->gsi_h, p_mad, /* actual payload */
429 &av, /* address vector */
430 (u_int64_t) CAST_P2LONG(p_bo),
431 IB_MGT_DEFAULT_SEND_TIME,
432 cl_ntoh32(p_mad_addr->addr_type.gsi.
433 remote_qp));
434
435 }
436
437 status = IB_SUCCESS;
438 if (ret != IB_MGT_OK) {
439 osm_log(p_vend->p_log, OSM_LOG_ERROR,
440 "osmv_transport_mad_send: ERR 7210: "
441 "Error sending mad (%d).\n", ret);
442 status = IB_ERROR;
443 } else {
444 osmv_IBMGT_transport_mgr_t *p_mgr =
445 (osmv_IBMGT_transport_mgr_t *) (p_bo->p_transp_mgr);
446
447 /* Let the others work when I am sleeping ... */
448 osmv_txn_unlock(p_bo);
449
450 cl_event_wait_on(&(p_mgr->send_done), 0xffffffff, TRUE);
451
452 /* Re-acquire the lock */
453 osmv_txn_lock(p_bo);
454
455 if (TRUE == p_bo->is_closing) {
456
457 osm_log(p_vend->p_log, OSM_LOG_ERROR,
458 "osmv_transport_mad_send: ERR 7211: "
459 "The handle %p is being unbound, cannot send.\n",
460 h_bind);
461 status = IB_ERROR;
462 }
463
464 if (p_mgr->is_send_ok == FALSE) {
465 status = IB_ERROR;
466 }
467 }
468
469 OSM_LOG_EXIT(p_vend->p_log);
470 return (status);
471 }
472
osmv_transport_done(IN const osm_bind_handle_t h_bind)473 void osmv_transport_done(IN const osm_bind_handle_t h_bind)
474 {
475 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind;
476 osm_log_t *p_log = p_bo->p_vendor->p_log;
477 osmv_IBMGT_transport_mgr_t *p_mgr;
478 osmv_IBMGT_transport_info_t *p_tpot_info;
479 IB_MGT_ret_t ret;
480 cl_list_obj_t *p_obj = NULL;
481 cl_list_item_t *p_item, *p_item_tmp;
482 int i;
483 cl_qlist_t *p_list = NULL;
484
485 OSM_LOG_ENTER(p_log);
486
487 CL_ASSERT(p_bo);
488
489 /* First of all - zero out the magic_ptr, so if a callback is called -
490 it'll know that we are currently closing down, and will not handle the
491 mad. */
492 p_bo->magic_ptr = 0;
493
494 p_mgr = (osmv_IBMGT_transport_mgr_t *) (p_bo->p_transp_mgr);
495 p_tpot_info =
496 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->p_transport_info);
497
498 switch (p_mgr->mad_type) {
499 case IB_MGT_SMI:
500 p_list = p_tpot_info->p_smi_list;
501
502 /* remove from the bindings list */
503 p_item = cl_qlist_head(p_list);
504 while (p_item != cl_qlist_end(p_list)) {
505 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item);
506 if (cl_qlist_obj(p_obj) == h_bind) {
507 break;
508 }
509 p_item_tmp = cl_qlist_next(p_item);
510 p_item = p_item_tmp;
511 }
512
513 CL_ASSERT(p_item != cl_qlist_end(p_list));
514 cl_qlist_remove_item(p_list, p_item);
515 if (p_obj)
516 free(p_obj);
517
518 /* no one is binded to smi anymore - we can free the list, unbind & realease the hndl */
519 if (cl_is_qlist_empty(p_list) == TRUE) {
520 free(p_list);
521 p_list = NULL;
522
523 ret = IB_MGT_unbind_sm(p_tpot_info->smi_h);
524 if (ret != IB_MGT_OK) {
525 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
526 "osmv_transport_done: ERR 7212: "
527 "Failed to unbind sm\n");
528 }
529
530 ret = IB_MGT_release_handle(p_tpot_info->smi_h);
531 if (ret != IB_MGT_OK) {
532 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
533 "osmv_transport_done: ERR 7213: "
534 "Failed to release smi handle\n");
535 }
536 p_tpot_info->smi_h = 0xffffffff;
537 }
538 break;
539
540 case IB_MGT_GSI:
541 p_list = p_tpot_info->gsi_mgmt_lists[p_mgr->mgmt_class];
542 /* remove from the bindings list */
543 p_item = cl_qlist_head(p_list);
544 while (p_item != cl_qlist_end(p_list)) {
545 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item);
546 if (cl_qlist_obj(p_obj) == h_bind) {
547 break;
548 }
549 p_item_tmp = cl_qlist_next(p_item);
550 p_item = p_item_tmp;
551 }
552
553 CL_ASSERT(p_item != cl_qlist_end(p_list));
554 cl_qlist_remove_item(p_list, p_item);
555 if (p_obj)
556 free(p_obj);
557
558 /* no one is binded to this class anymore - we can free the list and unbind this class */
559 if (cl_is_qlist_empty(p_list) == TRUE) {
560 free(p_list);
561 p_list = NULL;
562
563 ret =
564 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h,
565 p_mgr->mgmt_class);
566 if (ret != IB_MGT_OK) {
567 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
568 "osmv_transport_done: ERR 7214: "
569 "Failed to unbind gsi class\n");
570 }
571 }
572
573 /* all the mgmt classes are unbinded - release gsi handle */
574 for (i = 0; i < 15; i++) {
575 if (p_tpot_info->gsi_mgmt_lists[i] != NULL) {
576 break;
577 }
578 }
579
580 if (i == 15) {
581 ret = IB_MGT_release_handle(p_tpot_info->gsi_h);
582 if (ret != IB_MGT_OK) {
583 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR,
584 "osmv_transport_done: ERR 7215: "
585 "Failed to release gsi handle\n");
586 }
587 p_tpot_info->gsi_h = 0xffffffff;
588 }
589 } /* end switch */
590
591 free(p_mgr);
592 }
593
594 /**********************************************************************
595 * IB_MGT Receive callback : invoked after each receive
596 **********************************************************************/
597 void
__osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl,IN void * private_ctx_p,IN void * payload_p,IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)598 __osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl,
599 IN void *private_ctx_p,
600 IN void *payload_p,
601 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p)
602 {
603 osmv_bind_obj_t *p_bo;
604 osm_mad_addr_t mad_addr;
605 cl_list_item_t *p_item;
606 cl_list_obj_t *p_obj;
607 cl_qlist_t *p_list;
608 ib_mad_t *p_mad = (ib_mad_t *) payload_p;
609 osm_vendor_t *p_vendor;
610 osmv_IBMGT_transport_info_t *p_tinfo;
611
612 __osmv_IBMGT_rcv_desc_to_osm_addr(rcv_remote_info_p,
613 ((p_mad->mgmt_class ==
614 IB_MCLASS_SUBN_LID)
615 || (p_mad->mgmt_class ==
616 IB_MCLASS_SUBN_DIR)), &mad_addr);
617
618 /* different handling of SMI and GSI */
619 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) ||
620 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) {
621 /* SMI CASE */
622 p_bo = (osmv_bind_obj_t *) private_ctx_p;
623 /* Make sure the p_bo object is still relevant */
624 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
625 return;
626
627 p_vendor = p_bo->p_vendor;
628 p_tinfo =
629 (osmv_IBMGT_transport_info_t *) p_vendor->p_transport_info;
630 p_list = p_tinfo->p_smi_list;
631 } else {
632 /* GSI CASE */
633 p_bo = (osmv_bind_obj_t *) private_ctx_p;
634 /* Make sure the p_bo object is still relevant */
635 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
636 return;
637
638 p_vendor = p_bo->p_vendor;
639 p_tinfo =
640 (osmv_IBMGT_transport_info_t *) p_vendor->p_transport_info;
641 p_list = p_tinfo->gsi_mgmt_lists[p_mad->mgmt_class];
642 }
643
644 /* go over the bindings list and send the mad, one of them will accept it,
645 the others will drope
646 */
647 p_item = cl_qlist_head(p_list);
648 while (p_item != cl_qlist_end(p_list)) {
649 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item);
650 p_bo = cl_qlist_obj(p_obj);
651 /* give upper layer the mad */
652 osmv_dispatch_mad((osm_bind_handle_t) p_bo, payload_p,
653 &mad_addr);
654 /* Make sure the p_bo object is still relevant */
655 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing)
656 return;
657
658 p_item = cl_qlist_next(p_item);
659 }
660 }
661
662 /**********************************************************************
663 * IB_MGT Send callback : invoked after each send
664 **********************************************************************/
665 void
__osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl,IN u_int64_t wrid,IN IB_comp_status_t status,IN void * private_ctx_p)666 __osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl,
667 IN u_int64_t wrid,
668 IN IB_comp_status_t status, IN void *private_ctx_p)
669 {
670 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) CAST_P2LONG(wrid);
671
672 osmv_IBMGT_transport_mgr_t *p_mgr =
673 (osmv_IBMGT_transport_mgr_t *) p_bo->p_transp_mgr;
674
675 /* Make sure the p_bo object is still relevant */
676 if (p_bo->magic_ptr != p_bo)
677 return;
678
679 /* we assume that each send on a bind object is synchronized, and no paralel sends
680 from diffrent threads with same object can be made */
681 if (status == IB_COMP_SUCCESS) {
682 p_mgr->is_send_ok = TRUE;
683 } else
684 p_mgr->is_send_ok = FALSE;
685 cl_event_signal(&p_mgr->send_done);
686
687 }
688
689 /**********************************************************************
690 * IB_MGT to OSM ADDRESS VECTOR
691 **********************************************************************/
692 static void
__osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,IN uint8_t is_smi,OUT osm_mad_addr_t * p_mad_addr)693 __osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc,
694 IN uint8_t is_smi,
695 OUT osm_mad_addr_t * p_mad_addr)
696 {
697 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */
698 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid);
699 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */
700 p_mad_addr->path_bits = p_rcv_desc->local_path_bits;
701 /* Clear the grh any way to avoid unset fields */
702 memset(&p_mad_addr->addr_type.gsi.grh_info, 0,
703 sizeof(p_mad_addr->addr_type.gsi.grh_info));
704
705 if (is_smi) {
706 /* SMI */
707 p_mad_addr->addr_type.smi.source_lid =
708 cl_hton16(p_rcv_desc->remote_lid);
709 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */
710 } else {
711 /* GSI */
712 /* seems to me there is a IBMGT bug reversing the QPN ... */
713 /* Does IBMGT supposed to provide the QPN is network or HOST ? */
714 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp);
715
716 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
717 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */
718 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */
719 /* the full PKey table - than go by the index. */
720 /* since this does not seem reasonable to me I simply use the default */
721 /* There is a TAVOR limitation that only one P_KEY is supported per */
722 /* QP - so QP1 must use IB_DEFAULT_PKEY */
723 p_mad_addr->addr_type.gsi.pkey_ix = 0;
724 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl;
725
726 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag;
727 /* copy the GRH data if relevant */
728 if (p_mad_addr->addr_type.gsi.global_route) {
729 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow =
730 ib_grh_set_ver_class_flow(p_rcv_desc->grh.
731 IP_version,
732 p_rcv_desc->grh.
733 traffic_class,
734 p_rcv_desc->grh.
735 flow_label);
736 p_mad_addr->addr_type.gsi.grh_info.hop_limit =
737 p_rcv_desc->grh.hop_limit;
738 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw,
739 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t));
740 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
741 p_rcv_desc->grh.dgid, sizeof(ib_net64_t));
742 }
743 }
744 }
745
746 /**********************************************************************
747 * OSM ADDR VECTOR TO IB_MGT
748 **********************************************************************/
749 void
__osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr,IN uint8_t is_smi,OUT IB_ud_av_t * p_av)750 __osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr,
751 IN uint8_t is_smi, OUT IB_ud_av_t * p_av)
752 {
753
754 /* For global destination or Multicast address: */
755 u_int8_t ver;
756
757 memset(p_av, 0, sizeof(IB_ud_av_t));
758
759 p_av->src_path_bits = p_mad_addr->path_bits;
760 p_av->static_rate = p_mad_addr->static_rate;
761 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid);
762
763 if (is_smi) {
764 p_av->sl = 0; /* Just to note we use 0 here. */
765 } else {
766 p_av->sl = p_mad_addr->addr_type.gsi.service_level;
767 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route;
768
769 if (p_mad_addr->addr_type.gsi.global_route) {
770 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi.
771 grh_info.ver_class_flow, &ver,
772 &p_av->traffic_class,
773 &p_av->flow_label);
774 p_av->hop_limit =
775 p_mad_addr->addr_type.gsi.grh_info.hop_limit;
776 p_av->sgid_index = 0; /* we always use source GID 0 */
777 memcpy(&p_av->dgid,
778 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw,
779 sizeof(ib_net64_t));
780
781 }
782 }
783 }
784