1 /* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 #undef __init 37 #if HAVE_CONFIG_H 38 # include <config.h> 39 #endif /* HAVE_CONFIG_H */ 40 41 #include <stdlib.h> 42 #include <string.h> 43 #include <vendor/osm_vendor_ts.h> 44 #include <vendor/osm_vendor_api.h> 45 #include <vendor/osm_ts_useraccess.h> 46 #include <opensm/osm_subnet.h> 47 #include <opensm/osm_opensm.h> 48 49 /* 50 Since a race can accure on requests. Meaning - a response is received before 51 the send_callback is called - we will save both the madw_p and the fact 52 whether or not it is a response. A race can occure only on requests that did 53 not fail, and then the madw_p will be put back in the pool before the 54 callback. 55 */ 56 uint64_t __osm_set_wrid_by_p_madw(IN osm_madw_t * p_madw) 57 { 58 uint64_t wrid = 0; 59 60 CL_ASSERT(p_madw->p_mad); 61 62 memcpy(&wrid, &p_madw, sizeof(osm_madw_t *)); 63 wrid = (wrid << 1) | 64 ib_mad_is_response(p_madw->p_mad); 65 return wrid; 66 } 67 68 void 69 __osm_set_p_madw_and_resp_by_wrid(IN uint64_t wrid, 70 OUT uint8_t * is_resp, 71 OUT osm_madw_t ** pp_madw) 72 { 73 *is_resp = wrid & 0x0000000000000001; 74 wrid = wrid >> 1; 75 memcpy(pp_madw, &wrid, sizeof(osm_madw_t *)); 76 } 77 78 /********************************************************************** 79 * TS MAD to OSM ADDRESS VECTOR 80 **********************************************************************/ 81 void 82 __osm_ts_conv_mad_rcv_desc_to_osm_addr(IN osm_vendor_t * const p_vend, 83 IN struct ib_mad *p_mad, 84 IN uint8_t is_smi, 85 OUT osm_mad_addr_t * p_mad_addr) 86 { 87 p_mad_addr->dest_lid = cl_hton16(p_mad->slid); 88 p_mad_addr->static_rate = 0; /* HACK - we do not know the rate ! */ 89 p_mad_addr->path_bits = 0; /* HACK - no way to know in TS */ 90 if (is_smi) { 91 /* SMI */ 92 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid); 93 p_mad_addr->addr_type.smi.port_num = p_mad->port; 94 } else { 95 /* GSI */ 96 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn; 97 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 98 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index; 99 p_mad_addr->addr_type.gsi.service_level = 0; /* HACK no way to know */ 100 101 p_mad_addr->addr_type.gsi.global_route = FALSE; /* HACK no way to know */ 102 /* copy the GRH data if relevant */ 103 /* 104 if (p_mad_addr->addr_type.gsi.global_route) 105 { 106 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 107 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version, 108 p_rcv_desc->grh.traffic_class, 109 p_rcv_desc->grh.flow_label); 110 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit; 111 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 112 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 113 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 114 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 115 } 116 */ 117 } 118 } 119 120 /********************************************************************** 121 * OSM ADDR VECTOR TO TS MAD: 122 **********************************************************************/ 123 void 124 __osm_ts_conv_osm_addr_to_ts_addr(IN osm_mad_addr_t * p_mad_addr, 125 IN uint8_t is_smi, OUT struct ib_mad *p_mad) 126 { 127 128 /* For global destination or Multicast address: */ 129 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid); 130 p_mad->sl = 0; 131 if (is_smi) { 132 p_mad->sqpn = 0; 133 p_mad->dqpn = 0; 134 } else { 135 p_mad->sqpn = 1; 136 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp; 137 } 138 } 139 140 void __osm_vendor_clear_sm(IN osm_bind_handle_t h_bind) 141 { 142 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 143 osm_vendor_t *p_vend = p_bind->p_vend; 144 VAPI_ret_t status; 145 VAPI_hca_attr_t attr_mod; 146 VAPI_hca_attr_mask_t attr_mask; 147 148 OSM_LOG_ENTER(p_vend->p_log); 149 150 memset(&attr_mod, 0, sizeof(attr_mod)); 151 memset(&attr_mask, 0, sizeof(attr_mask)); 152 153 attr_mod.is_sm = FALSE; 154 attr_mask = HCA_ATTR_IS_SM; 155 156 status = 157 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 158 &attr_mask); 159 if (status != VAPI_OK) { 160 osm_log(p_vend->p_log, OSM_LOG_ERROR, 161 "__osm_vendor_clear_sm: ERR 5021: " 162 "Unable set 'IS_SM' bit in port attributes (%d).\n", 163 status); 164 } 165 166 OSM_LOG_EXIT(p_vend->p_log); 167 } 168 169 /********************************************************************** 170 * ANY CONSTRUCTION OF THE osm_vendor_t OBJECT 171 **********************************************************************/ 172 void osm_vendor_construct(IN osm_vendor_t * const p_vend) 173 { 174 memset(p_vend, 0, sizeof(*p_vend)); 175 cl_thread_construct(&(p_vend->smi_bind.poller)); 176 cl_thread_construct(&(p_vend->gsi_bind.poller)); 177 } 178 179 /********************************************************************** 180 * DEALOCATE osm_vendor_t 181 **********************************************************************/ 182 void osm_vendor_destroy(IN osm_vendor_t * const p_vend) 183 { 184 OSM_LOG_ENTER(p_vend->p_log); 185 osm_transaction_mgr_destroy(p_vend); 186 187 /* Destroy the poller threads */ 188 /* HACK: can you destroy an un-initialized thread ? */ 189 pthread_cancel(p_vend->smi_bind.poller.osd.id); 190 pthread_cancel(p_vend->gsi_bind.poller.osd.id); 191 cl_thread_destroy(&(p_vend->smi_bind.poller)); 192 cl_thread_destroy(&(p_vend->gsi_bind.poller)); 193 OSM_LOG_EXIT(p_vend->p_log); 194 } 195 196 /********************************************************************** 197 DEALLOCATE A POINTER TO osm_vendor_t 198 **********************************************************************/ 199 void osm_vendor_delete(IN osm_vendor_t ** const pp_vend) 200 { 201 CL_ASSERT(pp_vend); 202 203 osm_vendor_destroy(*pp_vend); 204 free(*pp_vend); 205 *pp_vend = NULL; 206 } 207 208 /********************************************************************** 209 Initializes the vendor: 210 **********************************************************************/ 211 212 ib_api_status_t 213 osm_vendor_init(IN osm_vendor_t * const p_vend, 214 IN osm_log_t * const p_log, IN const uint32_t timeout) 215 { 216 ib_api_status_t status = IB_SUCCESS; 217 218 OSM_LOG_ENTER(p_log); 219 220 p_vend->p_log = p_log; 221 p_vend->p_transaction_mgr = NULL; 222 osm_transaction_mgr_init(p_vend); 223 p_vend->timeout = timeout; 224 225 /* we use the file handle to track the binding */ 226 p_vend->smi_bind.ul_dev_fd = -1; 227 p_vend->gsi_bind.ul_dev_fd = -1; 228 229 OSM_LOG_EXIT(p_log); 230 return (status); 231 } 232 233 /********************************************************************** 234 * Create and Initialize osm_vendor_t Object 235 **********************************************************************/ 236 osm_vendor_t *osm_vendor_new(IN osm_log_t * const p_log, 237 IN const uint32_t timeout) 238 { 239 ib_api_status_t status; 240 osm_vendor_t *p_vend; 241 242 OSM_LOG_ENTER(p_log); 243 244 CL_ASSERT(p_log); 245 246 p_vend = malloc(sizeof(*p_vend)); 247 if (p_vend != NULL) { 248 memset(p_vend, 0, sizeof(*p_vend)); 249 250 status = osm_vendor_init(p_vend, p_log, timeout); 251 if (status != IB_SUCCESS) { 252 osm_vendor_delete(&p_vend); 253 } 254 } else { 255 osm_log(p_vend->p_log, OSM_LOG_ERROR, 256 "osm_vendor_new: ERR 5007: " 257 "Fail to allocate vendor object.\n"); 258 } 259 260 OSM_LOG_EXIT(p_log); 261 return (p_vend); 262 } 263 264 /********************************************************************** 265 * TS RCV Thread callback 266 * HACK: - we need to make this support arbitrary size mads. 267 **********************************************************************/ 268 void 269 __osm_ts_rcv_callback(IN osm_ts_bind_info_t * p_bind, 270 IN osm_mad_addr_t * p_mad_addr, 271 IN uint32_t mad_size, IN void *p_mad) 272 { 273 ib_api_status_t status; 274 osm_madw_t *p_req_madw = NULL; 275 osm_madw_t *p_madw; 276 osm_vend_wrap_t *p_new_vw; 277 ib_mad_t *p_mad_buf; 278 osm_log_t *const p_log = p_bind->p_vend->p_log; 279 280 OSM_LOG_ENTER(p_log); 281 282 /* if it is a response MAD we mustbe able to get the request */ 283 if (ib_mad_is_response((ib_mad_t *) p_mad)) { 284 /* can we find a matching madw by this payload TID */ 285 status = 286 osm_transaction_mgr_get_madw_for_tid(p_bind->p_vend, 287 (ib_mad_t *) p_mad, 288 &p_req_madw); 289 if (status != IB_SUCCESS) { 290 osm_log(p_log, OSM_LOG_ERROR, 291 "__osm_ts_rcv_callback: ERR 5008: " 292 "Error obtaining request madw by TID (%d).\n", 293 status); 294 p_req_madw = NULL; 295 } 296 297 if (p_req_madw == NULL) { 298 osm_log(p_log, OSM_LOG_ERROR, 299 "__osm_ts_rcv_callback: ERR 5009: " 300 "Fail to obtain request madw for receined MAD. Aborting CB.\n"); 301 goto Exit; 302 } 303 } 304 305 /* do we have a request ??? */ 306 if (p_req_madw == NULL) { 307 308 /* if not - get new osm_madw and arrange it. */ 309 /* create the new madw in the pool */ 310 p_madw = osm_mad_pool_get(p_bind->p_osm_pool, 311 (osm_bind_handle_t) p_bind, 312 mad_size, p_mad_addr); 313 if (p_madw == NULL) { 314 osm_log(p_log, OSM_LOG_ERROR, 315 "__osm_ts_rcv_callback: ERR 5010: " 316 "Error request for a new madw.\n"); 317 goto Exit; 318 } 319 /* HACK: we cust to avoid the const ??? */ 320 p_mad_buf = (void *)p_madw->p_mad; 321 } else { 322 /* we have the madw defined during the send and stored in the vend_wrap */ 323 /* we need to make sure the wrapper is correctly init there */ 324 CL_ASSERT(p_req_madw->vend_wrap.p_resp_madw != 0); 325 p_madw = p_req_madw->vend_wrap.p_resp_madw; 326 327 CL_ASSERT(p_madw->h_bind); 328 p_mad_buf = 329 osm_vendor_get(p_madw->h_bind, mad_size, 330 &p_madw->vend_wrap); 331 332 if (p_mad_buf == NULL) { 333 osm_log(p_log, OSM_LOG_ERROR, 334 "__osm_ts_rcv_callback: ERR 5011: " 335 "Unable to acquire wire MAD.\n"); 336 337 goto Exit; 338 } 339 340 /* 341 Finally, attach the wire MAD to this wrapper. 342 */ 343 osm_madw_set_mad(p_madw, p_mad_buf); 344 } 345 346 /* init some fields of the vendor wrapper */ 347 p_new_vw = osm_madw_get_vend_ptr(p_madw); 348 p_new_vw->h_bind = p_bind; 349 p_new_vw->size = mad_size; 350 p_new_vw->p_resp_madw = NULL; 351 p_new_vw->p_mad_buf = p_mad_buf; 352 353 memcpy(p_new_vw->p_mad_buf, p_mad, mad_size); 354 355 /* attach the buffer to the wrapper */ 356 p_madw->p_mad = p_mad_buf; 357 358 /* we can also make sure we marked the size and bind on the returned madw */ 359 p_madw->h_bind = p_new_vw->h_bind; 360 361 /* call the CB */ 362 (*(osm_vend_mad_recv_callback_t) p_bind->rcv_callback) 363 (p_madw, p_bind->client_context, p_req_madw); 364 365 Exit: 366 OSM_LOG_EXIT(p_log); 367 } 368 369 /********************************************************************** 370 * TS Send callback : invoked after each send 371 * 372 **********************************************************************/ 373 void 374 __osm_ts_send_callback(IN osm_ts_bind_info_t * bind_info_p, 375 IN boolean_t is_resp, 376 IN osm_madw_t * madw_p, IN IB_comp_status_t status) 377 { 378 osm_log_t *const p_log = bind_info_p->p_vend->p_log; 379 osm_vend_wrap_t *p_vw; 380 381 OSM_LOG_ENTER(p_log); 382 383 osm_log(p_log, OSM_LOG_DEBUG, 384 "__osm_ts_send_callback: INFO 1008: " 385 "Handling Send of MADW:%p Is Resp:%d.\n", madw_p, is_resp); 386 387 /* we need to handle requests and responses differently */ 388 if (is_resp) { 389 if (status != IB_COMP_SUCCESS) { 390 osm_log(p_log, OSM_LOG_ERROR, 391 "__osm_ts_send_callback: ERR 5012: " 392 "Error Sending Response MADW:%p.\n", madw_p); 393 } else { 394 osm_log(p_log, OSM_LOG_DEBUG, 395 "__osm_ts_send_callback: DBG 1008: " 396 "Completed Sending Response MADW:%p.\n", 397 madw_p); 398 } 399 400 /* if we are a response - we need to clean it up */ 401 osm_mad_pool_put(bind_info_p->p_osm_pool, madw_p); 402 } else { 403 404 /* this call back is invoked on completion of send - error or not */ 405 if (status != IB_COMP_SUCCESS) { 406 407 osm_log(p_log, OSM_LOG_ERROR, 408 "__osm_ts_send_callback: ERR 5013: " 409 "Received an Error from IB_MGT Send (%d).\n", 410 status); 411 412 p_vw = osm_madw_get_vend_ptr(madw_p); 413 CL_ASSERT(p_vw); 414 415 /* 416 Return any wrappers to the pool that may have been 417 pre-emptively allocated to handle a receive. 418 */ 419 if (p_vw->p_resp_madw) { 420 osm_mad_pool_put(bind_info_p->p_osm_pool, 421 p_vw->p_resp_madw); 422 p_vw->p_resp_madw = NULL; 423 } 424 425 /* invoke the CB */ 426 (*(osm_vend_mad_send_err_callback_t) bind_info_p-> 427 send_err_callback) 428 (bind_info_p->client_context, madw_p); 429 } else { 430 /* successful request send - do nothing - the response will need the 431 out mad */ 432 osm_log(p_log, OSM_LOG_DEBUG, 433 "__osm_ts_send_callback: DBG 1008: " 434 "Completed Sending Request MADW:%p.\n", madw_p); 435 } 436 } 437 438 OSM_LOG_EXIT(p_log); 439 } 440 441 /********************************************************************** 442 * Poller thread: 443 * Always receive 256byte mads from the devcie file 444 **********************************************************************/ 445 void __osm_vendor_ts_poller(IN void *p_ptr) 446 { 447 int ts_ret_code; 448 struct ib_mad mad; 449 osm_mad_addr_t mad_addr; 450 osm_ts_bind_info_t *const p_bind = (osm_ts_bind_info_t *) p_ptr; 451 452 OSM_LOG_ENTER(p_bind->p_vend->p_log); 453 /* we set the type of cancelation for this thread */ 454 pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); 455 456 while (1) { 457 /* we read one mad at a time and pass it to the read callback function */ 458 ts_ret_code = read(p_bind->ul_dev_fd, &mad, sizeof(mad)); 459 if (ts_ret_code != sizeof(mad)) { 460 osm_log(p_bind->p_vend->p_log, OSM_LOG_ERROR, 461 "__osm_vendor_ts_poller: ERR 5003: " 462 "error with read, bytes = %d, errno = %d\n", 463 ts_ret_code, errno); 464 } else { 465 osm_log(p_bind->p_vend->p_log, OSM_LOG_DEBUG, 466 "__osm_vendor_ts_poller: " 467 "MAD QPN:%d SLID:0x%04x class:0x%02x " 468 "__osm_vendor_ts_poller:0x%02x attr:0x%04x status:0x%04x " 469 "__osm_vendor_ts_poller:0x%016" PRIx64 "\n", 470 cl_ntoh32(mad.dqpn), 471 cl_ntoh16(mad.slid), 472 mad.mgmt_class, 473 mad.r_method, 474 cl_ntoh16(mad.attribute_id), 475 cl_ntoh16(mad.status), 476 cl_ntoh64(mad.transaction_id)); 477 478 /* first arrange an address */ 479 __osm_ts_conv_mad_rcv_desc_to_osm_addr(p_bind->p_vend, 480 &mad, 481 (((ib_mad_t *) & 482 mad)-> 483 mgmt_class == 484 IB_MCLASS_SUBN_LID) 485 || 486 (((ib_mad_t *) & 487 mad)-> 488 mgmt_class == 489 IB_MCLASS_SUBN_DIR), 490 &mad_addr); 491 492 /* call the receiver callback */ 493 /* HACK: this should be replaced with a call to the RMPP Assembly ... */ 494 __osm_ts_rcv_callback(p_bind, &mad_addr, 256, &mad); 495 } 496 } 497 498 OSM_LOG_EXIT(p_bind->p_vend->p_log); 499 } 500 501 /********************************************************************** 502 * BINDs a callback (rcv and send error) for a given class and method 503 * defined by the given: osm_bind_info_t 504 **********************************************************************/ 505 osm_bind_handle_t 506 osm_vendor_bind(IN osm_vendor_t * const p_vend, 507 IN osm_bind_info_t * const p_user_bind, 508 IN osm_mad_pool_t * const p_mad_pool, 509 IN osm_vend_mad_recv_callback_t mad_recv_callback, 510 IN osm_vend_mad_send_err_callback_t send_err_callback, 511 IN void *context) 512 { 513 ib_net64_t port_guid; 514 osm_ts_bind_info_t *p_bind = NULL; 515 VAPI_hca_hndl_t hca_hndl; 516 VAPI_hca_id_t hca_id; 517 uint32_t port_num; 518 ib_api_status_t status; 519 int device_fd; 520 char device_file[16]; 521 osm_ts_user_mad_filter filter; 522 int ts_ioctl_ret; 523 int qpn; 524 525 OSM_LOG_ENTER(p_vend->p_log); 526 527 CL_ASSERT(p_mad_pool); 528 529 port_guid = p_user_bind->port_guid; 530 531 osm_log(p_vend->p_log, OSM_LOG_INFO, 532 "osm_vendor_bind: " 533 "Binding to port 0x%" PRIx64 ".\n", cl_ntoh64(port_guid)); 534 535 switch (p_user_bind->mad_class) { 536 case IB_MCLASS_SUBN_LID: 537 case IB_MCLASS_SUBN_DIR: 538 p_bind = &(p_vend->smi_bind); 539 qpn = 0; 540 break; 541 542 case IB_MCLASS_SUBN_ADM: 543 default: 544 p_bind = &(p_vend->gsi_bind); 545 qpn = 1; 546 break; 547 } 548 549 /* Make sure we did not previously opened the file */ 550 if (p_bind->ul_dev_fd >= 0) { 551 osm_log(p_vend->p_log, OSM_LOG_ERROR, 552 "osm_vendor_bind: ERR 5004: " 553 "Already binded to port %u\n", p_bind->port_num); 554 goto Exit; 555 } 556 557 /* 558 We need to figure out what is the TS file name to attach to. 559 I guess it is following the index of the port in the table of 560 ports. 561 */ 562 563 /* obtain the hca name and port num from the guid */ 564 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 565 "osm_vendor_bind: " 566 "Finding CA and Port that owns port guid 0x%" PRIx64 ".\n", 567 cl_ntoh64(port_guid)); 568 status = 569 osm_vendor_get_guid_ca_and_port(p_vend, port_guid, &hca_hndl, 570 &hca_id, &port_num); 571 if (status != IB_SUCCESS) { 572 osm_log(p_vend->p_log, OSM_LOG_ERROR, 573 "osm_vendor_bind: ERR 5005: " 574 "Fail to find port number of port guid:0x%016" PRIx64 575 "\n", port_guid); 576 goto Exit; 577 } 578 579 /* the file name is just /dev/ts_ua0: */ 580 strcpy(device_file, "/dev/ts_ua0"); 581 582 osm_log(p_vend->p_log, OSM_LOG_ERROR, 583 "osm_vendor_bind: " "Opening TS UL dev file:%s\n", device_file); 584 585 /* Open the file ... */ 586 device_fd = open(device_file, O_RDWR); 587 if (device_fd < 0) { 588 osm_log(p_vend->p_log, OSM_LOG_ERROR, 589 "osm_vendor_bind: ERR 5006: " 590 "Fail to open TS UL dev file:%s\n", device_file); 591 goto Exit; 592 } 593 594 /* track this bind request info */ 595 p_bind->ul_dev_fd = device_fd; 596 p_bind->port_num = port_num; 597 p_bind->p_vend = p_vend; 598 p_bind->client_context = context; 599 p_bind->rcv_callback = mad_recv_callback; 600 p_bind->send_err_callback = send_err_callback; 601 p_bind->p_osm_pool = p_mad_pool; 602 p_bind->hca_hndl = hca_hndl; 603 604 /* 605 * Create the MAD filter on this file handle. 606 */ 607 filter.port = port_num; 608 609 filter.qpn = qpn; 610 filter.mgmt_class = p_user_bind->mad_class; 611 filter.direction = TS_IB_MAD_DIRECTION_IN; 612 filter.mask = 613 TS_IB_MAD_FILTER_DIRECTION | 614 TS_IB_MAD_FILTER_PORT | 615 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS; 616 617 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter); 618 if (ts_ioctl_ret < 0) { 619 osm_log(p_vend->p_log, OSM_LOG_ERROR, 620 "osm_vendor_bind: ERR 5014: " 621 "Fail to register MAD filter with err:%u\n", 622 ts_ioctl_ret); 623 goto Exit; 624 } 625 626 /* Initialize the listener thread for this port */ 627 status = cl_thread_init(&p_bind->poller, 628 __osm_vendor_ts_poller, p_bind, 629 "osm ts poller"); 630 if (status != IB_SUCCESS) 631 goto Exit; 632 633 Exit: 634 OSM_LOG_EXIT(p_vend->p_log); 635 return ((osm_bind_handle_t) p_bind); 636 } 637 638 /********************************************************************** 639 Get a mad from the lower level. 640 The osm_vend_wrap_t is a wrapper used to connect the mad to the response. 641 **********************************************************************/ 642 ib_mad_t *osm_vendor_get(IN osm_bind_handle_t h_bind, 643 IN const uint32_t mad_size, 644 IN osm_vend_wrap_t * const p_vw) 645 { 646 ib_mad_t *p_mad; 647 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 648 osm_vendor_t *p_vend = p_bind->p_vend; 649 650 OSM_LOG_ENTER(p_vend->p_log); 651 652 CL_ASSERT(p_vw); 653 654 p_vw->size = mad_size; 655 656 /* allocate it */ 657 p_mad = (ib_mad_t *) malloc(p_vw->size); 658 if (p_mad == NULL) { 659 osm_log(p_vend->p_log, OSM_LOG_ERROR, 660 "osm_vendor_get: ERR 5022: " 661 "Error Obtaining MAD buffer.\n"); 662 goto Exit; 663 } 664 665 memset(p_mad, 0, p_vw->size); 666 667 /* track locally */ 668 p_vw->p_mad_buf = p_mad; 669 p_vw->h_bind = h_bind; 670 p_vw->p_resp_madw = NULL; 671 672 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 673 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 674 "osm_vendor_get: " 675 "Acquired MAD %p, size = %u.\n", p_mad, p_vw->size); 676 } 677 678 Exit: 679 OSM_LOG_EXIT(p_vend->p_log); 680 return (p_mad); 681 } 682 683 /********************************************************************** 684 * Return a MAD by providing it's wrapper object. 685 **********************************************************************/ 686 void 687 osm_vendor_put(IN osm_bind_handle_t h_bind, IN osm_vend_wrap_t * const p_vw) 688 { 689 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 690 osm_vendor_t *p_vend = p_bind->p_vend; 691 osm_madw_t *p_madw; 692 693 OSM_LOG_ENTER(p_vend->p_log); 694 695 CL_ASSERT(p_vw); 696 CL_ASSERT(p_vw->p_mad_buf); 697 698 if (osm_log_get_level(p_vend->p_log) >= OSM_LOG_DEBUG) { 699 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 700 "osm_vendor_put: " "Retiring MAD %p.\n", 701 p_vw->p_mad_buf); 702 } 703 704 /* 705 * We moved the removal of the transaction to immediatly after 706 * it was looked up. 707 */ 708 709 /* free the mad but the wrapper is part of the madw object */ 710 free(p_vw->p_mad_buf); 711 p_vw->p_mad_buf = NULL; 712 p_madw = PARENT_STRUCT(p_vw, osm_madw_t, vend_wrap); 713 p_madw->p_mad = NULL; 714 715 OSM_LOG_EXIT(p_vend->p_log); 716 } 717 718 /********************************************************************** 719 Actually Send a MAD 720 721 MADs are buffers of type: struct ib_mad - so they are limited by size. 722 This is for internal use by osm_vendor_send and the transaction mgr 723 retry too. 724 **********************************************************************/ 725 ib_api_status_t 726 osm_ts_send_mad(IN osm_ts_bind_info_t * p_bind, IN osm_madw_t * const p_madw) 727 { 728 osm_vendor_t *const p_vend = p_bind->p_vend; 729 osm_mad_addr_t *const p_mad_addr = osm_madw_get_mad_addr_ptr(p_madw); 730 ib_mad_t *const p_mad = osm_madw_get_mad_ptr(p_madw); 731 struct ib_mad ts_mad; 732 int ret; 733 ib_api_status_t status; 734 735 OSM_LOG_ENTER(p_vend->p_log); 736 737 /* 738 * Copy the MAD over to the sent mad 739 */ 740 memcpy(&ts_mad, p_mad, 256); 741 742 /* 743 * For all sends other than directed route SM MADs, 744 * acquire an address vector for the destination. 745 */ 746 if (p_mad->mgmt_class != IB_MCLASS_SUBN_DIR) { 747 __osm_ts_conv_osm_addr_to_ts_addr(p_mad_addr, 748 p_mad->mgmt_class == 749 IB_MCLASS_SUBN_LID, &ts_mad); 750 } else { 751 /* is a directed route - we need to construct a permissive address */ 752 /* we do not need port number since it is part of the mad_hndl */ 753 ts_mad.dlid = IB_LID_PERMISSIVE; 754 ts_mad.slid = IB_LID_PERMISSIVE; 755 } 756 if ((p_mad->mgmt_class == IB_MCLASS_SUBN_DIR) || 757 (p_mad->mgmt_class == IB_MCLASS_SUBN_LID)) { 758 ts_mad.sqpn = 0; 759 ts_mad.dqpn = 0; 760 } else { 761 ts_mad.sqpn = 1; 762 ts_mad.dqpn = 1; 763 } 764 ts_mad.port = p_bind->port_num; 765 766 /* send it */ 767 ret = write(p_bind->ul_dev_fd, &ts_mad, sizeof(ts_mad)); 768 769 if (ret != sizeof(ts_mad)) { 770 osm_log(p_vend->p_log, OSM_LOG_ERROR, 771 "osm_ts_send_mad: ERR 5026: " 772 "Error sending mad (%d).\n", ret); 773 status = IB_ERROR; 774 goto Exit; 775 } 776 777 status = IB_SUCCESS; 778 779 Exit: 780 OSM_LOG_EXIT(p_vend->p_log); 781 return (status); 782 } 783 784 /********************************************************************** 785 Send a MAD through. 786 787 What is unclear to me is the need for the setting of all the MAD Wrapper 788 fields. Seems like the OSM uses these values during it's processing... 789 **********************************************************************/ 790 ib_api_status_t 791 osm_vendor_send(IN osm_bind_handle_t h_bind, 792 IN osm_madw_t * const p_madw, IN boolean_t const resp_expected) 793 { 794 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 795 osm_vendor_t *const p_vend = p_bind->p_vend; 796 osm_vend_wrap_t *const p_vw = osm_madw_get_vend_ptr(p_madw); 797 ib_api_status_t status; 798 799 OSM_LOG_ENTER(p_vend->p_log); 800 801 /* 802 * If a response is expected to this MAD, then preallocate 803 * a mad wrapper to contain the wire MAD received in the 804 * response. Allocating a wrapper here allows for easier 805 * failure paths than after we already received the wire mad. 806 */ 807 if (resp_expected == TRUE) { 808 /* we track it in the vendor wrapper */ 809 p_vw->p_resp_madw = 810 osm_mad_pool_get_wrapper_raw(p_bind->p_osm_pool); 811 if (p_vw->p_resp_madw == NULL) { 812 osm_log(p_vend->p_log, OSM_LOG_ERROR, 813 "osm_vendor_send: ERR 5024: " 814 "Unable to allocate MAD wrapper.\n"); 815 status = IB_INSUFFICIENT_RESOURCES; 816 goto Exit; 817 } 818 819 /* put some minimal info on that wrapper */ 820 ((osm_madw_t *) (p_vw->p_resp_madw))->h_bind = h_bind; 821 822 /* we also want to track it in the TID based map */ 823 status = osm_transaction_mgr_insert_madw((osm_bind_handle_t *) 824 p_bind, p_madw); 825 if (status != IB_SUCCESS) { 826 osm_log(p_vend->p_log, OSM_LOG_ERROR, 827 "osm_vendor_send: ERR 5025: " 828 "Error inserting request madw by TID (%d).\n", 829 status); 830 } 831 } else 832 p_vw->p_resp_madw = NULL; 833 834 /* do the actual send */ 835 /* HACK: to be replaced by call to RMPP Segmentation */ 836 status = osm_ts_send_mad(p_bind, p_madw); 837 838 /* we do not get an asycn callback so call it ourselves */ 839 /* this will handle all cleanup if neccessary */ 840 __osm_ts_send_callback(p_bind, !resp_expected, p_madw, status); 841 842 Exit: 843 OSM_LOG_EXIT(p_vend->p_log); 844 return (status); 845 } 846 847 /********************************************************************** 848 * the idea here is to change the content of the bind such that it 849 * will hold the local address used for sending directed route by the SMA. 850 **********************************************************************/ 851 ib_api_status_t osm_vendor_local_lid_change(IN osm_bind_handle_t h_bind) 852 { 853 osm_vendor_t *p_vend = ((osm_ts_bind_info_t *) h_bind)->p_vend; 854 855 OSM_LOG_ENTER(p_vend->p_log); 856 857 osm_log(p_vend->p_log, OSM_LOG_DEBUG, 858 "osm_vendor_local_lid_change: DEBUG 2202: " "Change of LID.\n"); 859 860 OSM_LOG_EXIT(p_vend->p_log); 861 862 return (IB_SUCCESS); 863 } 864 865 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 866 { 867 osm_ts_bind_info_t *p_bind = (osm_ts_bind_info_t *) h_bind; 868 osm_vendor_t *p_vend = p_bind->p_vend; 869 VAPI_ret_t status; 870 VAPI_hca_attr_t attr_mod; 871 VAPI_hca_attr_mask_t attr_mask; 872 873 OSM_LOG_ENTER(p_vend->p_log); 874 875 memset(&attr_mod, 0, sizeof(attr_mod)); 876 memset(&attr_mask, 0, sizeof(attr_mask)); 877 878 attr_mod.is_sm = is_sm_val; 879 attr_mask = HCA_ATTR_IS_SM; 880 881 status = 882 VAPI_modify_hca_attr(p_bind->hca_hndl, p_bind->port_num, &attr_mod, 883 &attr_mask); 884 if (status != VAPI_OK) { 885 osm_log(p_vend->p_log, OSM_LOG_ERROR, 886 "osm_vendor_set_sm: ERR 5027: " 887 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n", 888 is_sm_val, status); 889 } 890 891 OSM_LOG_EXIT(p_vend->p_log); 892 } 893 894 void osm_vendor_set_debug(IN osm_vendor_t * const p_vend, IN int32_t level) 895 { 896 897 } 898