1 /* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 /* AUTHOR Edward Bortnikov 37 * 38 * DESCRIPTION 39 * The lower-level MAD transport interface implementation 40 * that allows sending a single MAD/receiving a callback 41 * when a single MAD is received. 42 */ 43 44 #if HAVE_CONFIG_H 45 # include <config.h> 46 #endif /* HAVE_CONFIG_H */ 47 48 #include <sys/types.h> 49 #include <sys/stat.h> 50 #include <sys/ioctl.h> 51 #include <fcntl.h> 52 #include <errno.h> 53 #include <stdlib.h> 54 #include <string.h> 55 56 #include <vendor/osm_vendor_api.h> 57 #include <vendor/osm_vendor_mlx_transport.h> 58 #include <vendor/osm_vendor_mlx_dispatcher.h> 59 #include <vendor/osm_vendor_mlx_svc.h> 60 #include <vendor/osm_ts_useraccess.h> 61 62 typedef struct _osmv_TOPSPIN_transport_mgr_ { 63 int device_fd; 64 osm_ts_user_mad_filter filter; 65 cl_thread_t receiver; 66 } osmv_TOPSPIN_transport_mgr_t; 67 68 static void 69 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend, 70 IN struct ib_mad *p_mad, 71 IN uint8_t is_smi, 72 OUT osm_mad_addr_t * p_mad_addr); 73 74 static void 75 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr, 76 IN uint8_t is_smi, 77 OUT struct ib_mad *p_mad); 78 79 void __osmv_TOPSPIN_receiver_thr(void *p_ctx) 80 { 81 int ts_ret_code; 82 struct ib_mad mad; 83 osm_mad_addr_t mad_addr; 84 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx; 85 ib_api_status_t status = IB_SUCCESS; 86 87 OSM_LOG_ENTER(p_bo->p_vendor->p_log); 88 89 /* Make sure the p_bo object is still relevant */ 90 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 91 return; 92 93 /* we set the type of cancelation for this thread */ 94 /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */ 95 96 while (1) { 97 /* Make sure the p_bo object is still relevant */ 98 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 99 return; 100 101 /* we read one mad at a time and pass it to the read callback function */ 102 ts_ret_code = 103 read(((osmv_TOPSPIN_transport_mgr_t *) (p_bo-> 104 p_transp_mgr))-> 105 device_fd, &mad, sizeof(mad)); 106 /* Make sure the p_bo object is still relevant */ 107 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 108 return; 109 110 if (ts_ret_code != sizeof(mad)) { 111 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 112 "__osmv_TOPSPIN_receiver_thr: ERR 6803: " 113 "error with read, bytes = %d, errno = %d\n", 114 ts_ret_code, errno); 115 break; 116 } else { 117 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 118 "__osmv_TOPSPIN_receiver_thr: " 119 "MAD QPN:%d SLID:0x%04x class:0x%02x " 120 "method:0x%02x attr:0x%04x status:0x%04x " 121 "tid:0x%016" PRIx64 "\n", 122 mad.dqpn, 123 cl_ntoh16(mad.slid), 124 mad.mgmt_class, 125 mad.r_method, 126 cl_ntoh16(mad.attribute_id), 127 cl_ntoh16(mad.status), 128 cl_ntoh64(mad.transaction_id)); 129 130 /* first arrange an address */ 131 __osmv_TOPSPIN_mad_addr_to_osm_addr(p_bo->p_vendor, 132 &mad, 133 (((ib_mad_t *) & 134 mad)-> 135 mgmt_class == 136 IB_MCLASS_SUBN_LID) 137 || 138 (((ib_mad_t *) & 139 mad)-> 140 mgmt_class == 141 IB_MCLASS_SUBN_DIR), 142 &mad_addr); 143 144 /* call the receiver callback */ 145 146 status = 147 osmv_dispatch_mad((osm_bind_handle_t) p_bo, 148 (void *)&mad, &mad_addr); 149 150 /* Make sure the p_bo object is still relevant */ 151 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 152 return; 153 154 if (IB_INTERRUPTED == status) { 155 156 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 157 "__osmv_TOPSPIN_receiver_thr: " 158 "The bind handle %p is being closed. " 159 "Breaking the loop.\n", p_bo); 160 break; 161 } 162 } 163 } 164 165 OSM_LOG_EXIT(p_bo->p_vendor->p_log); 166 } 167 168 /* 169 * NAME 170 * osmv_transport_init 171 * 172 * DESCRIPTION 173 * Setup the MAD transport infrastructure (filters, callbacks etc). 174 */ 175 176 ib_api_status_t 177 osmv_transport_init(IN osm_bind_info_t * p_info, 178 IN char hca_id[VENDOR_HCA_MAXNAMES], 179 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo) 180 { 181 cl_status_t cl_st; 182 char device_file[16]; 183 int device_fd; 184 int ts_ioctl_ret; 185 osmv_TOPSPIN_transport_mgr_t *p_mgr = 186 malloc(sizeof(osmv_TOPSPIN_transport_mgr_t)); 187 int qpn; 188 189 if (!p_mgr) { 190 return IB_INSUFFICIENT_MEMORY; 191 } 192 193 memset(p_mgr, 0, sizeof(osmv_TOPSPIN_transport_mgr_t)); 194 195 /* open TopSpin file device */ 196 /* HACK: assume last char in hostid is the HCA index */ 197 sprintf(device_file, "/dev/ts_ua%u", hca_idx); 198 device_fd = open(device_file, O_RDWR); 199 if (device_fd < 0) { 200 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n", 201 device_file, errno); 202 return IB_ERROR; 203 } 204 205 /* 206 * Create the MAD filter on this file handle. 207 */ 208 209 p_mgr->filter.port = p_bo->port_num; 210 p_mgr->filter.direction = TS_IB_MAD_DIRECTION_IN; 211 p_mgr->filter.mask = 212 TS_IB_MAD_FILTER_DIRECTION | 213 TS_IB_MAD_FILTER_PORT | 214 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS; 215 216 switch (p_info->mad_class) { 217 case IB_MCLASS_SUBN_LID: 218 case IB_MCLASS_SUBN_DIR: 219 qpn = 0; 220 p_mgr->filter.qpn = qpn; 221 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_LID; 222 ts_ioctl_ret = 223 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter); 224 if (ts_ioctl_ret < 0) { 225 return IB_ERROR; 226 } 227 228 p_mgr->filter.mgmt_class = IB_MCLASS_SUBN_DIR; 229 ts_ioctl_ret = 230 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter); 231 if (ts_ioctl_ret < 0) { 232 return IB_ERROR; 233 } 234 235 break; 236 237 case IB_MCLASS_SUBN_ADM: 238 default: 239 qpn = 1; 240 p_mgr->filter.qpn = qpn; 241 p_mgr->filter.mgmt_class = p_info->mad_class; 242 ts_ioctl_ret = 243 ioctl(device_fd, TS_IB_IOCSMADFILTADD, &p_mgr->filter); 244 if (ts_ioctl_ret < 0) { 245 return IB_ERROR; 246 } 247 break; 248 } 249 250 p_mgr->device_fd = device_fd; 251 252 p_bo->p_transp_mgr = p_mgr; 253 254 /* Initialize the magic_ptr to the pointer of the p_bo info. 255 This will be used to signal when the object is being destroyed, so no 256 real action will be done then. */ 257 p_bo->magic_ptr = p_bo; 258 259 /* init receiver thread */ 260 cl_st = 261 cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_receiver_thr, 262 (void *)p_bo, "osmv TOPSPIN rcv thr"); 263 264 return (ib_api_status_t) cl_st; 265 } 266 267 /* 268 * NAME 269 * osmv_transport_send_mad 270 * 271 * DESCRIPTION 272 * Send a single MAD (256 byte) 273 */ 274 275 ib_api_status_t 276 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind, 277 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr) 278 { 279 280 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 281 osm_vendor_t const *p_vend = p_bo->p_vendor; 282 struct ib_mad ts_mad; 283 int ret; 284 ib_api_status_t status; 285 286 const ib_mad_t *p_mad_hdr = p_mad; 287 288 OSM_LOG_ENTER(p_vend->p_log); 289 290 memset(&ts_mad, 0, sizeof(ts_mad)); 291 292 /* Make sure the p_bo object is still relevant */ 293 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 294 return IB_INVALID_CALLBACK; 295 296 /* 297 * Copy the MAD over to the sent mad 298 */ 299 memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE); 300 301 /* 302 * For all sends other than directed route SM MADs, 303 * acquire an address vector for the destination. 304 */ 305 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) { 306 307 __osmv_TOPSPIN_osm_addr_to_mad_addr(p_mad_addr, 308 p_mad_hdr->mgmt_class == 309 IB_MCLASS_SUBN_LID, 310 &ts_mad); 311 } else { 312 /* is a directed route - we need to construct a permissive address */ 313 /* we do not need port number since it is part of the mad_hndl */ 314 ts_mad.dlid = IB_LID_PERMISSIVE; 315 ts_mad.slid = IB_LID_PERMISSIVE; 316 ts_mad.sqpn = 0; 317 ts_mad.dqpn = 0; 318 } 319 320 ts_mad.port = p_bo->port_num; 321 322 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 323 "osmv_transport_mad_send: " 324 "Sending QPN:%d DLID:0x%04x class:0x%02x " 325 "method:0x%02x attr:0x%04x status:0x%04x " 326 "tid:0x%016" PRIx64 "\n", 327 ts_mad.dqpn, 328 cl_ntoh16(ts_mad.dlid), 329 ts_mad.mgmt_class, 330 ts_mad.r_method, 331 cl_ntoh16(ts_mad.attribute_id), 332 cl_ntoh16(ts_mad.status), cl_ntoh64(ts_mad.transaction_id) 333 ); 334 335 /* send it */ 336 ret = 337 write(((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))-> 338 device_fd, &ts_mad, sizeof(ts_mad)); 339 340 if (ret != sizeof(ts_mad)) { 341 osm_log(p_vend->p_log, OSM_LOG_ERROR, 342 "osmv_transport_mad_send: ERR 6804: " 343 "Error sending mad (%d).\n", ret); 344 status = IB_ERROR; 345 goto Exit; 346 } 347 348 status = IB_SUCCESS; 349 350 Exit: 351 OSM_LOG_EXIT(p_vend->p_log); 352 return (status); 353 } 354 355 /* 356 register a new mad type to the opened device file 357 and send a mad through - the main idea is to make 358 the filter catch it such that the read unblocks 359 */ 360 void __osm_transport_gen_dummy_mad(osmv_bind_obj_t * p_bo) 361 { 362 struct ib_mad ts_mad; 363 osmv_TOPSPIN_transport_mgr_t *p_mgr = 364 (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr); 365 struct ib_get_port_info_ioctl port_data; 366 int ts_ioctl_ret; 367 368 /* prepare the mad fields following the stored filter on the bind */ 369 memset(&ts_mad, 0, sizeof(ts_mad)); 370 ts_mad.format_version = 1; 371 ts_mad.mgmt_class = p_mgr->filter.mgmt_class; 372 ts_mad.attribute_id = 0x2; 373 ts_mad.class_version = 1; 374 ts_mad.r_method = cl_ntoh16(0x2); 375 ts_mad.port = p_bo->port_num; 376 ts_mad.sqpn = p_mgr->filter.qpn; 377 ts_mad.dqpn = p_mgr->filter.qpn; 378 ts_mad.slid = 0xffff; 379 /* we must send to our local lid ... */ 380 port_data.port = p_bo->port_num; 381 ts_ioctl_ret = ioctl(p_mgr->device_fd, TS_IB_IOCGPORTINFO, &port_data); 382 ts_mad.dlid = port_data.port_info.lid; 383 ts_mad.transaction_id = 0x9999; 384 write(p_mgr->device_fd, &ts_mad, sizeof(ts_mad)); 385 } 386 387 void osmv_transport_done(IN const osm_bind_handle_t h_bind) 388 { 389 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 390 osmv_TOPSPIN_transport_mgr_t *p_tpot_mgr = 391 (osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr); 392 393 CL_ASSERT(p_bo); 394 395 /* First of all - zero out the magic_ptr, so if a callback is called - 396 it'll know that we are currently closing down, and will not handle the 397 mad. */ 398 p_bo->magic_ptr = 0; 399 /* usleep(3000000); */ 400 401 /* seems the only way to abort a blocking read is to make it read something */ 402 __osm_transport_gen_dummy_mad(p_bo); 403 cl_thread_destroy(&(p_tpot_mgr->receiver)); 404 free(p_tpot_mgr); 405 } 406 407 static void 408 __osmv_TOPSPIN_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr, 409 IN uint8_t is_smi, OUT struct ib_mad *p_mad) 410 { 411 412 /* For global destination or Multicast address: */ 413 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid); 414 p_mad->sl = p_mad_addr->addr_type.gsi.service_level; 415 if (is_smi) { 416 p_mad->sqpn = 0; 417 p_mad->dqpn = 0; 418 } else { 419 p_mad->sqpn = 1; 420 p_mad->dqpn = cl_ntoh32(p_mad_addr->addr_type.gsi.remote_qp); 421 } 422 /* 423 HACK we limit to the first PKey Index assuming it will 424 always be the default PKey 425 */ 426 p_mad->pkey_index = 0; 427 } 428 429 static void 430 __osmv_TOPSPIN_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend, 431 IN struct ib_mad *p_mad, 432 IN uint8_t is_smi, 433 OUT osm_mad_addr_t * p_mad_addr) 434 { 435 p_mad_addr->dest_lid = cl_hton16(p_mad->slid); 436 p_mad_addr->static_rate = 0; 437 p_mad_addr->path_bits = 0; 438 if (is_smi) { 439 /* SMI */ 440 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid); 441 p_mad_addr->addr_type.smi.port_num = p_mad->port; 442 } else { 443 /* GSI */ 444 p_mad_addr->addr_type.gsi.remote_qp = cl_ntoh32(p_mad->sqpn); 445 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 446 /* There is a TAVOR limitation that only one P_KEY is supported per */ 447 /* QP - so QP1 must use IB_DEFAULT_PKEY */ 448 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index; 449 p_mad_addr->addr_type.gsi.service_level = p_mad->sl; 450 451 p_mad_addr->addr_type.gsi.global_route = FALSE; 452 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */ 453 /* 454 if (p_mad_addr->addr_type.gsi.global_route) 455 { 456 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 457 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version, 458 p_rcv_desc->grh.traffic_class, 459 p_rcv_desc->grh.flow_label); 460 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit; 461 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 462 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 463 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 464 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 465 } 466 */ 467 } 468 } 469 470 /* 471 * NAME osm_vendor_set_sm 472 * 473 * DESCRIPTION Modifies the port info for the bound port to set the "IS_SM" bit 474 * according to the value given (TRUE or FALSE). 475 */ 476 #if (defined(OSM_VENDOR_INTF_TS_NO_VAPI) || defined(OSM_VENDOR_INTF_TS)) 477 478 void osm_vendor_set_sm(IN osm_bind_handle_t h_bind, IN boolean_t is_sm_val) 479 { 480 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 481 osm_vendor_t const *p_vend = p_bo->p_vendor; 482 int ts_ioctl_ret; 483 int device_fd = 484 ((osmv_TOPSPIN_transport_mgr_t *) (p_bo->p_transp_mgr))->device_fd; 485 struct ib_set_port_info_ioctl set_port_data; 486 487 OSM_LOG_ENTER(p_vend->p_log); 488 489 memset(&set_port_data, 0, sizeof(set_port_data)); 490 491 set_port_data.port = p_bo->port_num; 492 set_port_data.port_info.valid_fields = IB_PORT_IS_SM; 493 set_port_data.port_info.is_sm = is_sm_val; 494 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSPORTINFO, &set_port_data); 495 if (ts_ioctl_ret < 0) { 496 osm_log(p_vend->p_log, OSM_LOG_ERROR, 497 "osm_vendor_set_sm: ERR 6805: " 498 "Unable set 'IS_SM' bit to:%u in port attributes (%d).\n", 499 is_sm_val, ts_ioctl_ret); 500 } 501 502 OSM_LOG_EXIT(p_vend->p_log); 503 } 504 505 #endif 506