1 /* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 /* AUTHOR Edward Bortnikov 37 * 38 * DESCRIPTION 39 * The lower-level MAD transport interface implementation 40 * that allows sending a single MAD/receiving a callback 41 * when a single MAD is received. 42 */ 43 44 #if HAVE_CONFIG_H 45 # include <config.h> 46 #endif /* HAVE_CONFIG_H */ 47 48 #include <stdlib.h> 49 #include <string.h> 50 #include <ib_mgt.h> 51 #include <complib/cl_event.h> 52 #include <vendor/osm_vendor_mlx_transport.h> 53 #include <vendor/osm_vendor_mlx_dispatcher.h> 54 #include <opensm/osm_log.h> 55 56 typedef struct _osmv_IBMGT_transport_mgr_ { 57 IB_MGT_mad_type_t mad_type; 58 uint8_t mgmt_class; /* for gsi */ 59 /* for communication between send call back and send mad */ 60 boolean_t is_send_ok; 61 cl_event_t send_done; 62 } osmv_IBMGT_transport_mgr_t; 63 64 typedef struct _osmv_IBMGT_transport_info_ { 65 IB_MGT_mad_hndl_t smi_h; 66 cl_qlist_t *p_smi_list; 67 68 IB_MGT_mad_hndl_t gsi_h; 69 /* holds bind object list for every binded mgmt class */ 70 cl_qlist_t *gsi_mgmt_lists[15]; 71 } osmv_IBMGT_transport_info_t; 72 73 static void 74 __osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc, 75 IN uint8_t is_smi, 76 OUT osm_mad_addr_t * p_mad_addr); 77 78 static void 79 __osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr, 80 IN uint8_t is_smi, OUT IB_ud_av_t * p_av); 81 82 void 83 __osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl, 84 IN u_int64_t wrid, 85 IN IB_comp_status_t status, IN void *private_ctx_p); 86 87 void 88 __osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl, 89 IN void *private_ctx_p, 90 IN void *payload_p, 91 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p); 92 93 /* 94 * NAME 95 * osmv_transport_init 96 * 97 * DESCRIPTION 98 * Setup the MAD transport infrastructure (filters, callbacks etc). 99 */ 100 101 ib_api_status_t 102 osmv_transport_init(IN osm_bind_info_t * p_info, 103 IN char hca_id[VENDOR_HCA_MAXNAMES], 104 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo) 105 { 106 ib_api_status_t st = IB_SUCCESS; 107 IB_MGT_ret_t ret; 108 IB_MGT_mad_type_t mad_type; 109 osmv_IBMGT_transport_mgr_t *p_mgr; 110 osmv_IBMGT_transport_info_t *p_tpot_info; 111 cl_list_obj_t *p_obj = NULL; 112 osm_log_t *p_log = p_bo->p_vendor->p_log; 113 int i; 114 115 UNUSED_PARAM(hca_idx); 116 117 /* if first bind, allocate tranport_info at vendor */ 118 if (NULL == p_bo->p_vendor->p_transport_info) { 119 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 120 "osmv_transport_init: first bind() for the vendor\n"); 121 p_bo->p_vendor->p_transport_info 122 = (osmv_IBMGT_transport_info_t *) 123 malloc(sizeof(osmv_IBMGT_transport_info_t)); 124 if (NULL == p_bo->p_vendor->p_transport_info) { 125 return IB_INSUFFICIENT_MEMORY; 126 } 127 memset(p_bo->p_vendor->p_transport_info, 0, 128 sizeof(osmv_IBMGT_transport_info_t)); 129 p_tpot_info = 130 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor-> 131 p_transport_info); 132 133 p_tpot_info->smi_h = 0xffffffff; 134 p_tpot_info->p_smi_list = NULL; 135 136 p_tpot_info->gsi_h = 0xffffffff; 137 for (i = 0; i < 15; i++) { 138 139 p_tpot_info->gsi_mgmt_lists[i] = NULL; 140 } 141 142 } else { 143 144 p_tpot_info = 145 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor-> 146 p_transport_info); 147 } 148 149 /* Initialize the magic_ptr to the pointer of the p_bo info. 150 This will be used to signal when the object is being destroyed, so no 151 real action will be done then. */ 152 p_bo->magic_ptr = p_bo; 153 154 /* allocate transport mgr */ 155 p_mgr = malloc(sizeof(osmv_IBMGT_transport_mgr_t)); 156 if (NULL == p_mgr) { 157 free(p_tpot_info); 158 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 159 "osmv_transport_init: ERR 7201: " "alloc failed \n"); 160 return IB_INSUFFICIENT_MEMORY; 161 } 162 163 memset(p_mgr, 0, sizeof(osmv_IBMGT_transport_mgr_t)); 164 165 p_bo->p_transp_mgr = p_mgr; 166 167 switch (p_info->mad_class) { 168 case IB_MCLASS_SUBN_LID: 169 case IB_MCLASS_SUBN_DIR: 170 mad_type = IB_MGT_SMI; 171 break; 172 173 case IB_MCLASS_SUBN_ADM: 174 default: 175 mad_type = IB_MGT_GSI; 176 break; 177 } 178 179 /* we only support one class registration per SMI/GSI !!! */ 180 switch (mad_type) { 181 case IB_MGT_SMI: 182 /* we do not need to bind the handle if already available */ 183 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 184 "osmv_transport_init: SMI bind\n"); 185 186 if (p_tpot_info->smi_h == 0xffffffff) { 187 ret = IB_MGT_get_handle(hca_id, 188 p_bo->port_num, 189 IB_MGT_SMI, 190 &(p_tpot_info->smi_h)); 191 if (IB_MGT_OK != ret) { 192 osm_log(p_log, OSM_LOG_ERROR, 193 "osmv_transport_init: ERR 7202: " 194 "IB_MGT_get_handle for smi failed \n"); 195 st = IB_ERROR; 196 free(p_mgr); 197 goto Exit; 198 } 199 200 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 201 "osmv_transport_init: got smi handle:%d \n", 202 p_tpot_info->smi_h); 203 204 ret = IB_MGT_bind_sm(p_tpot_info->smi_h); 205 if (IB_MGT_OK != ret) { 206 osm_log(p_log, OSM_LOG_ERROR, 207 "osmv_transport_init: ERR 7203: " 208 "IB_MGT_bind_sm failed \n"); 209 st = IB_ERROR; 210 free(p_mgr); 211 goto Exit; 212 } 213 214 /* init smi list */ 215 p_tpot_info->p_smi_list = malloc(sizeof(cl_qlist_t)); 216 if (NULL == p_tpot_info->p_smi_list) { 217 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 218 "osmv_transport_init: ERR 7204: " 219 "alloc failed \n"); 220 IB_MGT_unbind_sm(p_tpot_info->smi_h); 221 IB_MGT_release_handle(p_tpot_info->smi_h); 222 free(p_mgr); 223 return IB_INSUFFICIENT_MEMORY; 224 } 225 memset(p_tpot_info->p_smi_list, 0, sizeof(cl_qlist_t)); 226 cl_qlist_init(p_tpot_info->p_smi_list); 227 228 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 229 "osmv_transport_init: before reg_cb\n"); 230 ret = IB_MGT_reg_cb(p_tpot_info->smi_h, 231 &__osmv_IBMGT_rcv_cb, 232 p_bo, 233 &__osmv_IBMGT_send_cb, 234 p_tpot_info->p_smi_list, 235 IB_MGT_RCV_CB_MASK | 236 IB_MGT_SEND_CB_MASK); 237 if (ret != IB_SUCCESS) { 238 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 239 "osmv_transport_init: ERR 7205: " 240 "reg_cb failed with return code:%x \n", 241 ret); 242 IB_MGT_unbind_sm(p_tpot_info->smi_h); 243 IB_MGT_release_handle(p_tpot_info->smi_h); 244 free(p_tpot_info->p_smi_list); 245 free(p_mgr); 246 st = IB_ERROR; 247 goto Exit; 248 } 249 250 } 251 /* insert to list of smi's - for raising callbacks later on */ 252 p_obj = malloc(sizeof(cl_list_obj_t)); 253 if (p_obj) 254 memset(p_obj, 0, sizeof(cl_list_obj_t)); 255 cl_qlist_set_obj(p_obj, p_bo); 256 cl_qlist_insert_tail(p_tpot_info->p_smi_list, 257 &p_obj->list_item); 258 259 break; 260 261 case IB_MGT_GSI: 262 /* we do not need to bind the handle if already available */ 263 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 264 "osmv_transport_init: ERR 7206: GSI bind\n"); 265 if (p_tpot_info->gsi_h == 0xffffffff) { 266 ret = IB_MGT_get_handle(hca_id, 267 p_bo->port_num, 268 IB_MGT_GSI, 269 &(p_tpot_info->gsi_h)); 270 if (IB_MGT_OK != ret) { 271 osm_log(p_log, OSM_LOG_ERROR, 272 "osmv_transport_init: ERR 7207: " 273 "IB_MGT_get_handle for gsi failed \n"); 274 st = IB_ERROR; 275 free(p_mgr); 276 goto Exit; 277 } 278 } 279 280 /* this mgmt class was not binded yet */ 281 if (p_tpot_info->gsi_mgmt_lists[p_info->mad_class] == NULL) { 282 ret = 283 IB_MGT_bind_gsi_class(p_tpot_info->gsi_h, 284 p_info->mad_class); 285 if (IB_MGT_OK != ret) { 286 osm_log(p_log, OSM_LOG_ERROR, 287 "osmv_transport_init: ERR 7208: " 288 "IB_MGT_bind_gsi_class failed \n"); 289 st = IB_ERROR; 290 free(p_mgr); 291 goto Exit; 292 } 293 294 p_tpot_info->gsi_mgmt_lists[p_info->mad_class] = 295 malloc(sizeof(cl_qlist_t)); 296 if (NULL == 297 p_tpot_info->gsi_mgmt_lists[p_info->mad_class]) { 298 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h, 299 p_info->mad_class); 300 free(p_mgr); 301 return IB_INSUFFICIENT_MEMORY; 302 } 303 memset(p_tpot_info->gsi_mgmt_lists[p_info->mad_class], 304 0, sizeof(cl_qlist_t)); 305 cl_qlist_init(p_tpot_info-> 306 gsi_mgmt_lists[p_info->mad_class]); 307 } 308 /* insert to list of smi's - for raising callbacks later on */ 309 p_obj = malloc(sizeof(cl_list_obj_t)); 310 if (p_obj) 311 memset(p_obj, 0, sizeof(cl_list_obj_t)); 312 cl_qlist_set_obj(p_obj, p_bo); 313 cl_qlist_insert_tail(p_tpot_info-> 314 gsi_mgmt_lists[p_info->mad_class], 315 &p_obj->list_item); 316 317 p_mgr->mgmt_class = p_info->mad_class; 318 ret = IB_MGT_reg_cb(p_tpot_info->gsi_h, 319 &__osmv_IBMGT_rcv_cb, 320 p_bo, 321 &__osmv_IBMGT_send_cb, 322 p_bo, 323 IB_MGT_RCV_CB_MASK | IB_MGT_SEND_CB_MASK); 324 325 if (ret != IB_SUCCESS) { 326 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h, 327 p_mgr->mgmt_class); 328 free(p_tpot_info->gsi_mgmt_lists[p_mgr->mgmt_class]); 329 free(p_mgr); 330 st = IB_ERROR; 331 goto Exit; 332 } 333 334 break; 335 336 default: 337 osm_log(p_log, OSM_LOG_ERROR, 338 "osmv_transport_init: ERR 7209: unrecognized mgmt class \n"); 339 st = IB_ERROR; 340 free(p_mgr); 341 goto Exit; 342 } 343 344 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 345 "osmv_transport_init: GSI bind\n"); 346 cl_event_construct(&p_mgr->send_done); 347 cl_event_init(&p_mgr->send_done, TRUE); 348 p_mgr->is_send_ok = FALSE; 349 p_mgr->mad_type = mad_type; 350 351 Exit: 352 /* OSM_LOG_EXIT(p_log ); */ 353 return (ib_api_status_t) st; 354 } 355 356 /* 357 * NAME 358 * osmv_transport_send_mad 359 * 360 * DESCRIPTION 361 * Send a single MAD (256 byte) 362 */ 363 364 ib_api_status_t 365 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind, 366 IN void *p_ib_mad, IN const osm_mad_addr_t * p_mad_addr) 367 { 368 369 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 370 osmv_IBMGT_transport_info_t *p_tpot_info = 371 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->p_transport_info); 372 osm_vendor_t const *p_vend = p_bo->p_vendor; 373 ib_api_status_t status; 374 IB_ud_av_t av; 375 IB_MGT_ret_t ret; 376 ib_mad_t *p_mad = p_ib_mad; 377 378 OSM_LOG_ENTER(p_vend->p_log); 379 380 CL_ASSERT(p_bo->p_vendor->p_transport_info); 381 382 /* 383 * For all sends other than directed route SM MADs, 384 * acquire an address vector for the destination. 385 */ 386 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { 387 __osmv_IBMGT_osm_addr_to_ibmgt_addr(p_mad_addr, 388 p_mad->mgmt_class == 389 IB_MCLASS_SUBN_LID, &av); 390 } else { 391 /* is a directed route - we need to construct a permissive address */ 392 memset(&av, 0, sizeof(av)); 393 /* we do not need port number since it is part of the mad_hndl */ 394 av.dlid = IB_LID_PERMISSIVE; 395 } 396 397 /* send it */ 398 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) || 399 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) { 400 401 /* SMI CASE */ 402 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 403 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 404 "osmv_transport_mad_send: " 405 "av.dlid:0x%X, " 406 "av.static_rate:%d, " 407 "av.path_bits:%d.\n", 408 cl_ntoh16(av.dlid), av.static_rate, 409 av.src_path_bits); 410 } 411 412 ret = IB_MGT_send_mad(p_tpot_info->smi_h, p_mad, /* actual payload */ 413 &av, /* address vector */ 414 (u_int64_t) CAST_P2LONG(p_bo), 415 IB_MGT_DEFAULT_SEND_TIME); 416 } else { 417 /* GSI CASE - Support Remote QP */ 418 if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { 419 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 420 "osmv_transport_mad_send: " 421 "av.dlid:0x%X, av.static_rate:%d, av.path_bits:%d, remote qp:%d \n", 422 cl_ntoh16(av.dlid), av.static_rate, 423 av.src_path_bits, 424 cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp) 425 ); 426 } 427 428 ret = IB_MGT_send_mad_to_qp(p_tpot_info->gsi_h, p_mad, /* actual payload */ 429 &av, /* address vector */ 430 (u_int64_t) CAST_P2LONG(p_bo), 431 IB_MGT_DEFAULT_SEND_TIME, 432 cl_ntoh32(p_mad_addr->addr_type.gsi. 433 remote_qp)); 434 435 } 436 437 status = IB_SUCCESS; 438 if (ret != IB_MGT_OK) { 439 osm_log(p_vend->p_log, OSM_LOG_ERROR, 440 "osmv_transport_mad_send: ERR 7210: " 441 "Error sending mad (%d).\n", ret); 442 status = IB_ERROR; 443 } else { 444 osmv_IBMGT_transport_mgr_t *p_mgr = 445 (osmv_IBMGT_transport_mgr_t *) (p_bo->p_transp_mgr); 446 447 /* Let the others work when I am sleeping ... */ 448 osmv_txn_unlock(p_bo); 449 450 cl_event_wait_on(&(p_mgr->send_done), 0xffffffff, TRUE); 451 452 /* Re-acquire the lock */ 453 osmv_txn_lock(p_bo); 454 455 if (TRUE == p_bo->is_closing) { 456 457 osm_log(p_vend->p_log, OSM_LOG_ERROR, 458 "osmv_transport_mad_send: ERR 7211: " 459 "The handle %p is being unbound, cannot send.\n", 460 h_bind); 461 status = IB_ERROR; 462 } 463 464 if (p_mgr->is_send_ok == FALSE) { 465 status = IB_ERROR; 466 } 467 } 468 469 OSM_LOG_EXIT(p_vend->p_log); 470 return (status); 471 } 472 473 void osmv_transport_done(IN const osm_bind_handle_t h_bind) 474 { 475 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 476 osm_log_t *p_log = p_bo->p_vendor->p_log; 477 osmv_IBMGT_transport_mgr_t *p_mgr; 478 osmv_IBMGT_transport_info_t *p_tpot_info; 479 IB_MGT_ret_t ret; 480 cl_list_obj_t *p_obj = NULL; 481 cl_list_item_t *p_item, *p_item_tmp; 482 int i; 483 cl_qlist_t *p_list = NULL; 484 485 OSM_LOG_ENTER(p_log); 486 487 CL_ASSERT(p_bo); 488 489 /* First of all - zero out the magic_ptr, so if a callback is called - 490 it'll know that we are currently closing down, and will not handle the 491 mad. */ 492 p_bo->magic_ptr = 0; 493 494 p_mgr = (osmv_IBMGT_transport_mgr_t *) (p_bo->p_transp_mgr); 495 p_tpot_info = 496 (osmv_IBMGT_transport_info_t *) (p_bo->p_vendor->p_transport_info); 497 498 switch (p_mgr->mad_type) { 499 case IB_MGT_SMI: 500 p_list = p_tpot_info->p_smi_list; 501 502 /* remove from the bindings list */ 503 p_item = cl_qlist_head(p_list); 504 while (p_item != cl_qlist_end(p_list)) { 505 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item); 506 if (cl_qlist_obj(p_obj) == h_bind) { 507 break; 508 } 509 p_item_tmp = cl_qlist_next(p_item); 510 p_item = p_item_tmp; 511 } 512 513 CL_ASSERT(p_item != cl_qlist_end(p_list)); 514 cl_qlist_remove_item(p_list, p_item); 515 if (p_obj) 516 free(p_obj); 517 518 /* no one is binded to smi anymore - we can free the list, unbind & realease the hndl */ 519 if (cl_is_qlist_empty(p_list) == TRUE) { 520 free(p_list); 521 p_list = NULL; 522 523 ret = IB_MGT_unbind_sm(p_tpot_info->smi_h); 524 if (ret != IB_MGT_OK) { 525 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 526 "osmv_transport_done: ERR 7212: " 527 "Failed to unbind sm\n"); 528 } 529 530 ret = IB_MGT_release_handle(p_tpot_info->smi_h); 531 if (ret != IB_MGT_OK) { 532 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 533 "osmv_transport_done: ERR 7213: " 534 "Failed to release smi handle\n"); 535 } 536 p_tpot_info->smi_h = 0xffffffff; 537 } 538 break; 539 540 case IB_MGT_GSI: 541 p_list = p_tpot_info->gsi_mgmt_lists[p_mgr->mgmt_class]; 542 /* remove from the bindings list */ 543 p_item = cl_qlist_head(p_list); 544 while (p_item != cl_qlist_end(p_list)) { 545 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item); 546 if (cl_qlist_obj(p_obj) == h_bind) { 547 break; 548 } 549 p_item_tmp = cl_qlist_next(p_item); 550 p_item = p_item_tmp; 551 } 552 553 CL_ASSERT(p_item != cl_qlist_end(p_list)); 554 cl_qlist_remove_item(p_list, p_item); 555 if (p_obj) 556 free(p_obj); 557 558 /* no one is binded to this class anymore - we can free the list and unbind this class */ 559 if (cl_is_qlist_empty(p_list) == TRUE) { 560 free(p_list); 561 p_list = NULL; 562 563 ret = 564 IB_MGT_unbind_gsi_class(p_tpot_info->gsi_h, 565 p_mgr->mgmt_class); 566 if (ret != IB_MGT_OK) { 567 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 568 "osmv_transport_done: ERR 7214: " 569 "Failed to unbind gsi class\n"); 570 } 571 } 572 573 /* all the mgmt classes are unbinded - release gsi handle */ 574 for (i = 0; i < 15; i++) { 575 if (p_tpot_info->gsi_mgmt_lists[i] != NULL) { 576 break; 577 } 578 } 579 580 if (i == 15) { 581 ret = IB_MGT_release_handle(p_tpot_info->gsi_h); 582 if (ret != IB_MGT_OK) { 583 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 584 "osmv_transport_done: ERR 7215: " 585 "Failed to release gsi handle\n"); 586 } 587 p_tpot_info->gsi_h = 0xffffffff; 588 } 589 } /* end switch */ 590 591 free(p_mgr); 592 } 593 594 /********************************************************************** 595 * IB_MGT Receive callback : invoked after each receive 596 **********************************************************************/ 597 void 598 __osmv_IBMGT_rcv_cb(IN IB_MGT_mad_hndl_t mad_hndl, 599 IN void *private_ctx_p, 600 IN void *payload_p, 601 IN IB_MGT_mad_rcv_desc_t * rcv_remote_info_p) 602 { 603 osmv_bind_obj_t *p_bo; 604 osm_mad_addr_t mad_addr; 605 cl_list_item_t *p_item; 606 cl_list_obj_t *p_obj; 607 cl_qlist_t *p_list; 608 ib_mad_t *p_mad = (ib_mad_t *) payload_p; 609 osm_vendor_t *p_vendor; 610 osmv_IBMGT_transport_info_t *p_tinfo; 611 612 __osmv_IBMGT_rcv_desc_to_osm_addr(rcv_remote_info_p, 613 ((p_mad->mgmt_class == 614 IB_MCLASS_SUBN_LID) 615 || (p_mad->mgmt_class == 616 IB_MCLASS_SUBN_DIR)), &mad_addr); 617 618 /* different handling of SMI and GSI */ 619 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) || 620 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) { 621 /* SMI CASE */ 622 p_bo = (osmv_bind_obj_t *) private_ctx_p; 623 /* Make sure the p_bo object is still relevant */ 624 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 625 return; 626 627 p_vendor = p_bo->p_vendor; 628 p_tinfo = 629 (osmv_IBMGT_transport_info_t *) p_vendor->p_transport_info; 630 p_list = p_tinfo->p_smi_list; 631 } else { 632 /* GSI CASE */ 633 p_bo = (osmv_bind_obj_t *) private_ctx_p; 634 /* Make sure the p_bo object is still relevant */ 635 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 636 return; 637 638 p_vendor = p_bo->p_vendor; 639 p_tinfo = 640 (osmv_IBMGT_transport_info_t *) p_vendor->p_transport_info; 641 p_list = p_tinfo->gsi_mgmt_lists[p_mad->mgmt_class]; 642 } 643 644 /* go over the bindings list and send the mad, one of them will accept it, 645 the others will drope 646 */ 647 p_item = cl_qlist_head(p_list); 648 while (p_item != cl_qlist_end(p_list)) { 649 p_obj = PARENT_STRUCT(p_item, cl_list_obj_t, list_item); 650 p_bo = cl_qlist_obj(p_obj); 651 /* give upper layer the mad */ 652 osmv_dispatch_mad((osm_bind_handle_t) p_bo, payload_p, 653 &mad_addr); 654 /* Make sure the p_bo object is still relevant */ 655 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 656 return; 657 658 p_item = cl_qlist_next(p_item); 659 } 660 } 661 662 /********************************************************************** 663 * IB_MGT Send callback : invoked after each send 664 **********************************************************************/ 665 void 666 __osmv_IBMGT_send_cb(IN IB_MGT_mad_hndl_t mad_hndl, 667 IN u_int64_t wrid, 668 IN IB_comp_status_t status, IN void *private_ctx_p) 669 { 670 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) CAST_P2LONG(wrid); 671 672 osmv_IBMGT_transport_mgr_t *p_mgr = 673 (osmv_IBMGT_transport_mgr_t *) p_bo->p_transp_mgr; 674 675 /* Make sure the p_bo object is still relevant */ 676 if (p_bo->magic_ptr != p_bo) 677 return; 678 679 /* we assume that each send on a bind object is synchronized, and no paralel sends 680 from diffrent threads with same object can be made */ 681 if (status == IB_COMP_SUCCESS) { 682 p_mgr->is_send_ok = TRUE; 683 } else 684 p_mgr->is_send_ok = FALSE; 685 cl_event_signal(&p_mgr->send_done); 686 687 } 688 689 /********************************************************************** 690 * IB_MGT to OSM ADDRESS VECTOR 691 **********************************************************************/ 692 static void 693 __osmv_IBMGT_rcv_desc_to_osm_addr(IN IB_MGT_mad_rcv_desc_t * p_rcv_desc, 694 IN uint8_t is_smi, 695 OUT osm_mad_addr_t * p_mad_addr) 696 { 697 /* p_mad_addr->dest_lid = p_osm->subn.sm_base_lid; - for resp we use the dest lid ... */ 698 p_mad_addr->dest_lid = cl_hton16(p_rcv_desc->remote_lid); 699 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */ 700 p_mad_addr->path_bits = p_rcv_desc->local_path_bits; 701 /* Clear the grh any way to avoid unset fields */ 702 memset(&p_mad_addr->addr_type.gsi.grh_info, 0, 703 sizeof(p_mad_addr->addr_type.gsi.grh_info)); 704 705 if (is_smi) { 706 /* SMI */ 707 p_mad_addr->addr_type.smi.source_lid = 708 cl_hton16(p_rcv_desc->remote_lid); 709 p_mad_addr->addr_type.smi.port_num = 99; /* HACK - if used - should fail */ 710 } else { 711 /* GSI */ 712 /* seems to me there is a IBMGT bug reversing the QPN ... */ 713 /* Does IBMGT supposed to provide the QPN is network or HOST ? */ 714 p_mad_addr->addr_type.gsi.remote_qp = cl_hton32(p_rcv_desc->qp); 715 716 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 717 /* we do have the p_mad_addr->pkey_ix but how to get the PKey by index ? */ 718 /* the only way seems to be to use VAPI_query_hca_pkey_tbl and obtain */ 719 /* the full PKey table - than go by the index. */ 720 /* since this does not seem reasonable to me I simply use the default */ 721 /* There is a TAVOR limitation that only one P_KEY is supported per */ 722 /* QP - so QP1 must use IB_DEFAULT_PKEY */ 723 p_mad_addr->addr_type.gsi.pkey_ix = 0; 724 p_mad_addr->addr_type.gsi.service_level = p_rcv_desc->sl; 725 726 p_mad_addr->addr_type.gsi.global_route = p_rcv_desc->grh_flag; 727 /* copy the GRH data if relevant */ 728 if (p_mad_addr->addr_type.gsi.global_route) { 729 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 730 ib_grh_set_ver_class_flow(p_rcv_desc->grh. 731 IP_version, 732 p_rcv_desc->grh. 733 traffic_class, 734 p_rcv_desc->grh. 735 flow_label); 736 p_mad_addr->addr_type.gsi.grh_info.hop_limit = 737 p_rcv_desc->grh.hop_limit; 738 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 739 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 740 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 741 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 742 } 743 } 744 } 745 746 /********************************************************************** 747 * OSM ADDR VECTOR TO IB_MGT 748 **********************************************************************/ 749 void 750 __osmv_IBMGT_osm_addr_to_ibmgt_addr(IN const osm_mad_addr_t * p_mad_addr, 751 IN uint8_t is_smi, OUT IB_ud_av_t * p_av) 752 { 753 754 /* For global destination or Multicast address: */ 755 u_int8_t ver; 756 757 memset(p_av, 0, sizeof(IB_ud_av_t)); 758 759 p_av->src_path_bits = p_mad_addr->path_bits; 760 p_av->static_rate = p_mad_addr->static_rate; 761 p_av->dlid = cl_ntoh16(p_mad_addr->dest_lid); 762 763 if (is_smi) { 764 p_av->sl = 0; /* Just to note we use 0 here. */ 765 } else { 766 p_av->sl = p_mad_addr->addr_type.gsi.service_level; 767 p_av->grh_flag = p_mad_addr->addr_type.gsi.global_route; 768 769 if (p_mad_addr->addr_type.gsi.global_route) { 770 ib_grh_get_ver_class_flow(p_mad_addr->addr_type.gsi. 771 grh_info.ver_class_flow, &ver, 772 &p_av->traffic_class, 773 &p_av->flow_label); 774 p_av->hop_limit = 775 p_mad_addr->addr_type.gsi.grh_info.hop_limit; 776 p_av->sgid_index = 0; /* we always use source GID 0 */ 777 memcpy(&p_av->dgid, 778 &p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 779 sizeof(ib_net64_t)); 780 781 } 782 } 783 } 784