1 /* 2 * Copyright (c) 2004-2008 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2002-2005 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 1996-2003 Intel Corporation. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 * 34 */ 35 36 /* AUTHOR Edward Bortnikov 37 * 38 * DESCRIPTION 39 * The lower-level MAD transport interface implementation 40 * that allows sending a single MAD/receiving a callback 41 * when a single MAD is received. 42 */ 43 44 #if HAVE_CONFIG_H 45 # include <config.h> 46 #endif /* HAVE_CONFIG_H */ 47 48 #include <sys/types.h> 49 #include <sys/stat.h> 50 #include <sys/ioctl.h> 51 #include <fcntl.h> 52 #include <stddef.h> 53 #include <errno.h> 54 #include <stdlib.h> 55 #include <string.h> 56 57 #include <vendor/osm_vendor_api.h> 58 #include <vendor/osm_vendor_mlx_transport.h> 59 #include <vendor/osm_vendor_mlx_transport_anafa.h> 60 #include <vendor/osm_vendor_mlx_dispatcher.h> 61 #include <vendor/osm_vendor_mlx_svc.h> 62 #include <vendor/osm_ts_useraccess.h> 63 64 static void 65 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend, 66 IN struct ib_mad *p_mad, 67 IN uint8_t is_smi, 68 OUT osm_mad_addr_t * p_mad_addr); 69 70 static void 71 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * 72 p_mad_addr, IN uint8_t is_smi, 73 OUT struct ib_mad *p_mad); 74 75 void __osmv_TOPSPIN_ANAFA_receiver_thr(void *p_ctx) 76 { 77 int ts_ret_code; 78 struct ib_mad mad; 79 osm_mad_addr_t mad_addr; 80 osmv_bind_obj_t *const p_bo = (osmv_bind_obj_t *) p_ctx; 81 ib_api_status_t status = IB_SUCCESS; 82 83 OSM_LOG_ENTER(p_bo->p_vendor->p_log); 84 85 /* Make sure the p_bo object is still relevant */ 86 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 87 return; 88 89 /* we set the type of cancelation for this thread */ 90 /* pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); */ 91 92 while (1) { 93 /* Make sure the p_bo object is still relevant */ 94 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 95 return; 96 97 /* we read one mad at a time and pass it to the read callback function */ 98 ts_ret_code = 99 read(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo-> 100 p_transp_mgr))-> 101 device_fd, &mad, sizeof(mad)); 102 103 /* Make sure the p_bo object is still relevant */ 104 if ((p_bo->magic_ptr != p_bo) || p_bo->is_closing) 105 return; 106 107 if (ts_ret_code != sizeof(mad)) { 108 osm_log(p_bo->p_vendor->p_log, OSM_LOG_ERROR, 109 "__osmv_TOPSPIN_ANAFA_receiver_thr: ERR 6903: " 110 "error with read, bytes = %d\n", ts_ret_code); 111 break; 112 } else { 113 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 114 "__osmv_TOPSPIN_ANAFA_receiver_thr: " 115 "MAD QPN:%d SLID:0x%04x class:0x%02x " 116 "method:0x%02x attr:0x%04x status:0x%04x " 117 "tid:0x%016" PRIx64 "\n", 118 mad.dqpn, 119 cl_ntoh16(mad.slid), 120 mad.mgmt_class, 121 mad.r_method, 122 cl_ntoh16(mad.attribute_id), 123 cl_ntoh16(mad.status), 124 cl_ntoh64(mad.transaction_id)); 125 126 /* first arrange an address */ 127 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr 128 (p_bo->p_vendor, &mad, 129 (((ib_mad_t *) & mad)->mgmt_class == 130 IB_MCLASS_SUBN_LID) 131 || (((ib_mad_t *) & mad)->mgmt_class == 132 IB_MCLASS_SUBN_DIR), &mad_addr); 133 134 /* call the receiver callback */ 135 136 status = 137 osmv_dispatch_mad((osm_bind_handle_t) p_bo, 138 (void *)&mad, &mad_addr); 139 140 /* Make sure the p_bo object is still relevant */ 141 if (p_bo->magic_ptr != p_bo) 142 return; 143 144 if (IB_INTERRUPTED == status) { 145 146 osm_log(p_bo->p_vendor->p_log, OSM_LOG_DEBUG, 147 "__osmv_TOPSPIN_ANAFA_receiver_thr: " 148 "The bind handle %p is being closed. " 149 "Breaking the loop.\n", p_bo); 150 break; 151 } 152 } 153 } 154 155 OSM_LOG_EXIT(p_bo->p_vendor->p_log); 156 } 157 158 /* 159 * NAME 160 * osmv_transport_init 161 * 162 * DESCRIPTION 163 * Setup the MAD transport infrastructure (filters, callbacks etc). 164 */ 165 166 ib_api_status_t 167 osmv_transport_init(IN osm_bind_info_t * p_info, 168 IN char hca_id[VENDOR_HCA_MAXNAMES], 169 IN uint8_t hca_idx, IN osmv_bind_obj_t * p_bo) 170 { 171 cl_status_t cl_st; 172 173 int ts_ioctl_ret; 174 int device_fd; 175 char *device_file = "/dev/ts_ua0"; 176 osm_ts_user_mad_filter filter; 177 osmv_TOPSPIN_ANAFA_transport_mgr_t *p_mgr; 178 osmv_TOPSPIN_ANAFA_transport_info_t *p_tpot_info; 179 p_tpot_info = 180 (osmv_TOPSPIN_ANAFA_transport_info_t *) p_bo->p_vendor-> 181 p_transport_info; 182 183 p_mgr = malloc(sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t)); 184 if (!p_mgr) { 185 return IB_INSUFFICIENT_MEMORY; 186 } 187 188 memset(p_mgr, 0, sizeof(osmv_TOPSPIN_ANAFA_transport_mgr_t)); 189 190 /* open TopSpin file device */ 191 device_fd = open(device_file, O_RDWR); 192 if (device_fd < 0) { 193 fprintf(stderr, "Fatal: Fail to open the file:%s err:%d\n", 194 device_file, errno); 195 return IB_ERROR; 196 } 197 p_mgr->device_fd = device_fd; 198 199 /* 200 * Create the MAD filter on this file handle. 201 */ 202 203 filter.port = 0; /* Victor */ 204 filter.direction = TS_IB_MAD_DIRECTION_IN; 205 filter.mask = 206 TS_IB_MAD_FILTER_DIRECTION | 207 TS_IB_MAD_FILTER_PORT | 208 TS_IB_MAD_FILTER_QPN | TS_IB_MAD_FILTER_MGMT_CLASS; 209 210 switch (p_info->mad_class) { 211 case IB_MCLASS_SUBN_LID: 212 case IB_MCLASS_SUBN_DIR: 213 filter.qpn = 0; 214 filter.mgmt_class = IB_MCLASS_SUBN_LID; 215 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter); 216 if (ts_ioctl_ret < 0) { 217 return IB_ERROR; 218 } 219 220 filter.mgmt_class = IB_MCLASS_SUBN_DIR; 221 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter); 222 if (ts_ioctl_ret < 0) { 223 return IB_ERROR; 224 } 225 226 break; 227 228 case IB_MCLASS_SUBN_ADM: 229 default: 230 filter.qpn = 1; 231 filter.mgmt_class = p_info->mad_class; 232 ts_ioctl_ret = ioctl(device_fd, TS_IB_IOCSMADFILTADD, &filter); 233 if (ts_ioctl_ret < 0) { 234 return IB_ERROR; 235 } 236 break; 237 } 238 239 p_bo->p_transp_mgr = p_mgr; 240 241 /* Initialize the magic_ptr to the pointer of the p_bo info. 242 This will be used to signal when the object is being destroyed, so no 243 real action will be done then. */ 244 p_bo->magic_ptr = p_bo; 245 246 /* init receiver thread */ 247 cl_st = 248 cl_thread_init(&p_mgr->receiver, __osmv_TOPSPIN_ANAFA_receiver_thr, 249 (void *)p_bo, "osmv TOPSPIN_ANAFA rcv thr"); 250 251 return (ib_api_status_t) cl_st; 252 } 253 254 /* 255 * NAME 256 * osmv_transport_send_mad 257 * 258 * DESCRIPTION 259 * Send a single MAD (256 byte) 260 */ 261 262 ib_api_status_t 263 osmv_transport_mad_send(IN const osm_bind_handle_t h_bind, 264 IN void *p_mad, IN const osm_mad_addr_t * p_mad_addr) 265 { 266 267 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 268 osm_vendor_t const *p_vend = p_bo->p_vendor; 269 struct ib_mad ts_mad = { 0 }; 270 int ret; 271 ib_api_status_t status; 272 273 const ib_mad_t *p_mad_hdr = p_mad; 274 275 OSM_LOG_ENTER(p_vend->p_log); 276 277 /* Make sure the p_bo object is still relevant */ 278 if (p_bo->magic_ptr != p_bo) 279 return IB_INVALID_CALLBACK; 280 281 /* 282 * Copy the MAD over to the sent mad 283 */ 284 memcpy(&ts_mad, p_mad_hdr, MAD_BLOCK_SIZE); 285 286 /* 287 * For all sends other than directed route SM MADs, 288 * acquire an address vector for the destination. 289 */ 290 if (p_mad_hdr->mgmt_class != IB_MCLASS_SUBN_DIR) { 291 292 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(p_mad_addr, 293 p_mad_hdr-> 294 mgmt_class == 295 IB_MCLASS_SUBN_LID, 296 &ts_mad); 297 } else { 298 /* is a directed route - we need to construct a permissive address */ 299 /* we do not need port number since it is part of the mad_hndl */ 300 ts_mad.dlid = IB_LID_PERMISSIVE; 301 ts_mad.slid = IB_LID_PERMISSIVE; 302 } 303 if ((p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_DIR) || 304 (p_mad_hdr->mgmt_class == IB_MCLASS_SUBN_LID)) { 305 ts_mad.sqpn = 0; 306 ts_mad.dqpn = 0; 307 } else { 308 ts_mad.sqpn = 1; 309 ts_mad.dqpn = 1; 310 } 311 312 /* ts_mad.port = p_bo->port_num; */ 313 ts_mad.port = 0; /* Victor */ 314 315 /* send it */ 316 ret = 317 write(((osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo-> 318 p_transp_mgr))-> 319 device_fd, &ts_mad, sizeof(ts_mad)); 320 321 if (ret != sizeof(ts_mad)) { 322 osm_log(p_vend->p_log, OSM_LOG_ERROR, 323 "osmv_transport_mad_send: ERR 6904: " 324 "Error sending mad (%d).\n", ret); 325 status = IB_ERROR; 326 goto Exit; 327 } 328 329 status = IB_SUCCESS; 330 331 Exit: 332 OSM_LOG_EXIT(p_vend->p_log); 333 return (status); 334 } 335 336 void osmv_transport_done(IN const osm_bind_handle_t h_bind) 337 { 338 osmv_bind_obj_t *p_bo = (osmv_bind_obj_t *) h_bind; 339 osmv_TOPSPIN_ANAFA_transport_mgr_t *p_tpot_mgr = 340 (osmv_TOPSPIN_ANAFA_transport_mgr_t *) (p_bo->p_transp_mgr); 341 342 CL_ASSERT(p_bo); 343 344 /* First of all - zero out the magic_ptr, so if a callback is called - 345 it'll know that we are currently closing down, and will not handle the 346 mad. */ 347 p_bo->magic_ptr = 0; 348 349 /* usleep(3000000); */ 350 351 /* pthread_cancel (p_tpot_mgr->receiver.osd.id); */ 352 cl_thread_destroy(&(p_tpot_mgr->receiver)); 353 free(p_tpot_mgr); 354 } 355 356 static void 357 __osmv_TOPSPIN_ANAFA_osm_addr_to_mad_addr(IN const osm_mad_addr_t * p_mad_addr, 358 IN uint8_t is_smi, 359 OUT struct ib_mad *p_mad) 360 { 361 362 /* For global destination or Multicast address: */ 363 p_mad->dlid = cl_ntoh16(p_mad_addr->dest_lid); 364 p_mad->sl = p_mad_addr->addr_type.gsi.service_level; 365 if (is_smi) { 366 p_mad->sqpn = 0; 367 p_mad->dqpn = 0; 368 } else { 369 p_mad->sqpn = 1; 370 p_mad->dqpn = p_mad_addr->addr_type.gsi.remote_qp; 371 } 372 /* 373 HACK we limit to the first PKey Index assuming it will 374 always be the default PKey 375 */ 376 p_mad->pkey_index = 0; 377 } 378 379 static void 380 __osmv_TOPSPIN_ANAFA_mad_addr_to_osm_addr(IN osm_vendor_t const *p_vend, 381 IN struct ib_mad *p_mad, 382 IN uint8_t is_smi, 383 OUT osm_mad_addr_t * p_mad_addr) 384 { 385 p_mad_addr->dest_lid = cl_hton16(p_mad->slid); 386 p_mad_addr->static_rate = 0; 387 p_mad_addr->path_bits = 0; 388 if (is_smi) { 389 /* SMI */ 390 p_mad_addr->addr_type.smi.source_lid = cl_hton16(p_mad->slid); 391 p_mad_addr->addr_type.smi.port_num = p_mad->port; 392 } else { 393 /* GSI */ 394 p_mad_addr->addr_type.gsi.remote_qp = p_mad->sqpn; 395 p_mad_addr->addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY; 396 p_mad_addr->addr_type.gsi.pkey_ix = p_mad->pkey_index; 397 p_mad_addr->addr_type.gsi.service_level = p_mad->sl; 398 399 p_mad_addr->addr_type.gsi.global_route = FALSE; 400 /* copy the GRH data if relevant - TopSpin imp doesnt relate to GRH!!! */ 401 /* 402 if (p_mad_addr->addr_type.gsi.global_route) 403 { 404 p_mad_addr->addr_type.gsi.grh_info.ver_class_flow = 405 ib_grh_set_ver_class_flow(p_rcv_desc->grh.IP_version, 406 p_rcv_desc->grh.traffic_class, 407 p_rcv_desc->grh.flow_label); 408 p_mad_addr->addr_type.gsi.grh_info.hop_limit = p_rcv_desc->grh.hop_limit; 409 memcpy(&p_mad_addr->addr_type.gsi.grh_info.src_gid.raw, 410 &p_rcv_desc->grh.sgid, sizeof(ib_net64_t)); 411 memcpy(&p_mad_addr->addr_type.gsi.grh_info.dest_gid.raw, 412 p_rcv_desc->grh.dgid, sizeof(ib_net64_t)); 413 } 414 */ 415 } 416 } 417