1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/ib/mgt/ibcm/ibcm_impl.h> 29 #include <sys/ib/ibtl/ibti.h> 30 #include <sys/ib/mgt/ibcm/ibcm_arp.h> 31 32 /* 33 * ibcm_ti.c 34 * These routines implement the Communication Manager's interfaces to IBTL. 35 */ 36 37 /* CM rc recycle task args structure definition */ 38 typedef struct ibcm_taskq_recycle_arg_s { 39 ibt_channel_hdl_t rc_chan; 40 ibt_cep_flags_t control; 41 uint8_t hca_port_num; 42 ibt_recycle_handler_t func; 43 void *arg; 44 } ibcm_taskq_recycle_arg_t; 45 46 _NOTE(READ_ONLY_DATA(ibcm_taskq_recycle_arg_s)) 47 48 static ibt_status_t ibcm_init_reply_addr(ibcm_hca_info_t *hcap, 49 ibcm_mad_addr_t *reply_addr, ibt_chan_open_args_t *chan_args, 50 ibt_chan_open_flags_t flags, ib_time_t *cm_pkt_lt, ib_lid_t prim_slid); 51 static void ibcm_process_abort_via_taskq(void *args); 52 static ibt_status_t ibcm_process_rc_recycle_ret(void *recycle_arg); 53 static ibt_status_t ibcm_process_join_mcg(void *taskq_arg); 54 static void ibcm_process_async_join_mcg(void *tq_arg); 55 56 static ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *, 57 uint64_t c_mask, void *, size_t *); 58 59 static ibt_status_t ibcm_close_rc_channel(ibt_channel_hdl_t channel, 60 ibcm_state_data_t *statep, ibt_execution_mode_t mode); 61 62 /* Address Record management definitions */ 63 #define IBCM_DAPL_ATS_NAME "DAPL Address Translation Service" 64 #define IBCM_DAPL_ATS_SID 0x10000CE100415453ULL 65 #define IBCM_DAPL_ATS_NBYTES 16 66 ibcm_svc_info_t *ibcm_ar_svcinfop; 67 ibcm_ar_t *ibcm_ar_list; 68 69 /* 70 * Tunable parameter to turnoff the overriding of pi_path_mtu value. 71 * 1 By default override the path record's pi_path_mtu value to 72 * IB_MTU_1K for all RC channels. This is done only for the 73 * channels established on Tavor HCA and the path's pi_path_mtu 74 * is greater than IB_MTU_1K. 75 * 0 Do not override, use pi_path_mtu by default. 76 */ 77 int ibcm_override_path_mtu = 1; 78 79 #ifdef DEBUG 80 static void ibcm_print_reply_addr(ibt_channel_hdl_t channel, 81 ibcm_mad_addr_t *cm_reply_addr); 82 #endif 83 84 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_hdl})) 85 86 /* access is controlled between ibcm_sm.c and ibcm_ti.c by CVs */ 87 _NOTE(SCHEME_PROTECTS_DATA("Serialized access by CV", {ibt_rc_returns_t 88 ibt_ud_returns_t ibt_ap_returns_t ibt_ar_t})) 89 90 /* 91 * Typically, clients initialize these args in one api call, and use in 92 * another api 93 */ 94 _NOTE(SCHEME_PROTECTS_DATA("Expected usage of ibtl api by client", 95 {ibt_path_info_s ibt_cep_path_s ibt_adds_vect_s ibt_mcg_info_s ib_gid_s 96 ibt_ud_dest_attr_s ibt_ud_dest_s ibt_srv_data_s ibt_redirect_info_s})) 97 98 /* 99 * ibt_open_rc_channel() 100 * ibt_open_rc_channel opens a communication channel on the specified 101 * channel to the specified service. For connection service type qp's 102 * the CM initiates the CEP to establish the connection and transitions 103 * the QP/EEC to the "Ready to send" State modifying the QP/EEC's 104 * attributes as necessary. 105 * The implementation of this function assumes that alt path is different 106 * from primary path. It is assumed that the Path functions ensure that. 107 * 108 * RETURN VALUES: 109 * IBT_SUCCESS on success (or respective failure on error) 110 */ 111 ibt_status_t 112 ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags, 113 ibt_execution_mode_t mode, ibt_chan_open_args_t *chan_args, 114 ibt_rc_returns_t *ret_args) 115 { 116 /* all fields that are related to REQ MAD formation */ 117 118 ib_pkey_t prim_pkey; 119 ib_lid_t primary_slid, alternate_slid; 120 ib_qpn_t local_qpn = 0; 121 ib_guid_t hca_guid; 122 ib_qkey_t local_qkey = 0; 123 ib_eecn_t local_eecn = 0; 124 ib_eecn_t remote_eecn = 0; 125 boolean_t primary_grh; 126 boolean_t alternate_grh = B_FALSE; 127 ib_lid_t base_lid; 128 ib_com_id_t local_comid; 129 ibmf_msg_t *ibmf_msg, *ibmf_msg_dreq; 130 ibcm_req_msg_t *req_msgp; 131 132 uint8_t rdma_in, rdma_out; 133 uint8_t cm_retries; 134 uint64_t local_cm_proc_time; /* In usec */ 135 uint8_t local_cm_resp_time; /* IB time */ 136 uint64_t remote_cm_resp_time; /* In usec */ 137 uint32_t starting_psn = 0; 138 139 /* CM path related fields */ 140 ibmf_handle_t ibmf_hdl; 141 ibcm_qp_list_t *cm_qp_entry; 142 ibcm_mad_addr_t cm_reply_addr; 143 144 uint8_t cm_pkt_lt; 145 146 /* Local args for ibtl/internal CM functions called within */ 147 ibt_status_t status; 148 ibcm_status_t lkup_status; 149 ibt_qp_query_attr_t qp_query_attr; 150 151 /* Other misc local args */ 152 ibt_priv_data_len_t len; 153 ibcm_hca_info_t *hcap; 154 ibcm_state_data_t *statep; 155 uint8_t port_no; 156 157 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel(chan %p, %X, %x, %p, %p)", 158 channel, flags, mode, chan_args, ret_args); 159 160 if (IBCM_INVALID_CHANNEL(channel)) { 161 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: invalid channel"); 162 return (IBT_CHAN_HDL_INVALID); 163 } 164 165 /* cm handler should always be specified */ 166 if (chan_args->oc_cm_handler == NULL) { 167 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 168 "CM handler is not be specified", channel); 169 return (IBT_INVALID_PARAM); 170 } 171 172 if (mode == IBT_NONBLOCKING) { 173 if (ret_args != NULL) { 174 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 175 " ret_args should be NULL when called in " 176 "non-blocking mode", channel); 177 return (IBT_INVALID_PARAM); 178 } 179 } else if (mode == IBT_BLOCKING) { 180 if (ret_args == NULL) { 181 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 182 " ret_args should be Non-NULL when called in " 183 "blocking mode", channel); 184 return (IBT_INVALID_PARAM); 185 } 186 if (ret_args->rc_priv_data_len > IBT_REP_PRIV_DATA_SZ) { 187 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 188 " private data length is too large", channel); 189 return (IBT_INVALID_PARAM); 190 } 191 if ((ret_args->rc_priv_data_len > 0) && 192 (ret_args->rc_priv_data == NULL)) { 193 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 194 " rc_priv_data_len > 0, but rc_priv_data NULL", 195 channel); 196 return (IBT_INVALID_PARAM); 197 } 198 } else { /* any other mode is not valid for ibt_open_rc_channel */ 199 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 200 "invalid mode %x specified", channel, mode); 201 return (IBT_INVALID_PARAM); 202 } 203 204 /* 205 * XXX: no support yet for ibt_chan_open_flags_t - IBT_OCHAN_DUP 206 */ 207 if (flags & IBT_OCHAN_DUP) { 208 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 209 "Unsupported Flags specified: 0x%X", channel, flags); 210 return (IBT_INVALID_PARAM); 211 } 212 213 if ((flags & IBT_OCHAN_REDIRECTED) && 214 (flags & IBT_OCHAN_PORT_REDIRECTED)) { 215 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 216 "Illegal to specify IBT_OCHAN_REDIRECTED and " 217 "IBT_OCHAN_PORT_REDIRECTED flags together", channel); 218 return (IBT_INVALID_PARAM); 219 } 220 221 if (((flags & IBT_OCHAN_REDIRECTED) && 222 (chan_args->oc_cm_redirect_info == NULL)) || 223 ((flags & IBT_OCHAN_PORT_REDIRECTED) && 224 (chan_args->oc_cm_cep_path == NULL))) { 225 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 226 "Redirect flag specified, but respective arg is NULL", 227 channel); 228 return (IBT_INVALID_PARAM); 229 } 230 231 if ((flags & IBT_OCHAN_REDIRECTED) && 232 (chan_args->oc_cm_redirect_info->rdi_dlid == 0) && 233 (chan_args->oc_cm_redirect_info->rdi_gid.gid_guid == 0)) { 234 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 235 "Either rdi_dlid or rdi_gid must be specified for" 236 " IBT_OCHAN_REDIRECTED", channel); 237 return (IBT_INVALID_PARAM); 238 } 239 240 /* primary dlid and hca_port_num should never be zero */ 241 port_no = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 242 243 if ((IBCM_PRIM_ADDS_VECT(chan_args).av_dlid == 0) && (port_no == 0)) { 244 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 245 "Primary Path's information is not valid", channel); 246 return (IBT_INVALID_PARAM); 247 } 248 249 /* validate SID */ 250 if (chan_args->oc_path->pi_sid == 0) { 251 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 252 "ERROR: Service ID in path information is 0", channel); 253 return (IBT_INVALID_PARAM); 254 } 255 256 /* validate rnr_retry_cnt (enum has more than 3 bits) */ 257 if ((uint_t)chan_args->oc_path_rnr_retry_cnt > IBT_RNR_INFINITE_RETRY) { 258 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 259 "ERROR: oc_path_rnr_retry_cnt(%d) is out of range", 260 channel, chan_args->oc_path_rnr_retry_cnt); 261 return (IBT_INVALID_PARAM); 262 } 263 264 /* 265 * Ensure that client is not re-using a QP that is still associated 266 * with a statep 267 */ 268 IBCM_GET_CHAN_PRIVATE(channel, statep); 269 if (statep != NULL) { 270 IBCM_RELEASE_CHAN_PRIVATE(channel); 271 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 272 "Channel being re-used on active side", channel); 273 return (IBT_CHAN_IN_USE); 274 } 275 276 /* Get GUID from Channel */ 277 hca_guid = ibt_channel_to_hca_guid(channel); 278 279 /* validate QP's hca guid with that from primary path */ 280 if (hca_guid != chan_args->oc_path->pi_hca_guid) { 281 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 282 "GUID from Channel and primary path don't match", channel); 283 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 284 "Channel GUID %llX primary path GUID %llX", channel, 285 hca_guid, chan_args->oc_path->pi_hca_guid); 286 return (IBT_CHAN_HDL_INVALID); 287 } 288 289 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 290 "Local HCA GUID %llX", channel, hca_guid); 291 292 status = ibt_query_qp(channel, &qp_query_attr); 293 if (status != IBT_SUCCESS) { 294 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 295 "ibt_query_qp failed %d", channel, status); 296 return (status); 297 } 298 299 /* If client specified "no port change on QP" */ 300 if ((qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 301 port_no) && (flags & IBT_OCHAN_PORT_FIXED)) { 302 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 303 "chan port %d and path port %d does not match", channel, 304 qp_query_attr.qp_info.qp_transport.rc.rc_path. \ 305 cep_hca_port_num, port_no); 306 return (IBT_INVALID_PARAM); 307 } 308 309 if (qp_query_attr.qp_info.qp_trans != IBT_RC_SRV) { 310 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 311 "Invalid Channel type: Applicable only to RC Channel", 312 channel); 313 return (IBT_CHAN_SRV_TYPE_INVALID); 314 } 315 316 /* Check if QP is in INIT state or not */ 317 if (qp_query_attr.qp_info.qp_state != IBT_STATE_INIT) { 318 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 319 "QP is not in INIT state %x", channel, 320 qp_query_attr.qp_info.qp_state); 321 return (IBT_CHAN_STATE_INVALID); 322 } 323 324 local_qpn = qp_query_attr.qp_qpn; 325 326 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p Active QPN 0x%x", 327 channel, local_qpn); 328 329 #ifdef NO_EEC_SUPPORT_YET 330 331 if (flags & IBT_OCHAN_RDC_EXISTS) { 332 ibt_eec_query_attr_t eec_query_attr; 333 334 local_qkey = qp_query_attr.qp_info.qp_transport.rd_qkey; 335 336 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: RD"); 337 338 status = ibt_query_eec(channel, &eec_query_attr); 339 if (status != IBT_SUCCESS) { 340 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 341 " ibt_query_eec failed %d", channel, status); 342 return (status); 343 } 344 local_eecn = eec_query_attr.eec_eecn; 345 } 346 347 #endif 348 349 /* If no HCA found return failure */ 350 if ((hcap = ibcm_find_hca_entry(hca_guid)) == NULL) { 351 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 352 "hcap is NULL. Probably hca is not in active state", 353 channel); 354 return (IBT_CHAN_HDL_INVALID); 355 } 356 357 rdma_out = chan_args->oc_rdma_ra_out; 358 rdma_in = chan_args->oc_rdma_ra_in; 359 360 if ((rdma_in > hcap->hca_max_rdma_in_qp) || 361 (rdma_out > hcap->hca_max_rdma_out_qp)) { 362 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 363 "rdma in %d/out %d values exceed hca limits", channel, 364 rdma_in, rdma_out); 365 ibcm_dec_hca_acc_cnt(hcap); 366 return (IBT_INVALID_PARAM); 367 } 368 369 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 370 "rdma_in %d rdma_out %d", channel, rdma_in, rdma_out); 371 372 if (chan_args->oc_path->pi_prim_pkt_lt > ibcm_max_ib_pkt_lt) { 373 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 374 "Huge Primary Pkt lt %d", channel, 375 chan_args->oc_path->pi_prim_pkt_lt); 376 ibcm_dec_hca_acc_cnt(hcap); 377 return (IBT_PATH_PKT_LT_TOO_HIGH); 378 } 379 380 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, 381 NULL, &base_lid); 382 if (status != IBT_SUCCESS) { 383 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 384 "primary port_num %d not active", channel, port_no); 385 ibcm_dec_hca_acc_cnt(hcap); 386 return (status); 387 } 388 389 /* Validate P_KEY Index */ 390 status = ibt_index2pkey_byguid(hcap->hca_guid, port_no, 391 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix, &prim_pkey); 392 if (status != IBT_SUCCESS) { 393 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 394 "Invalid Primary PKeyIx %x", channel, 395 IBCM_PRIM_CEP_PATH(chan_args).cep_pkey_ix); 396 ibcm_dec_hca_acc_cnt(hcap); 397 return (status); 398 } 399 400 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 401 "primary_port_num %d primary_pkey 0x%x", channel, port_no, 402 prim_pkey); 403 404 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 405 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 406 != IBT_SUCCESS)) { 407 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 408 "ibmf reg or callback setup failed during re-initialize", 409 channel); 410 ibcm_dec_hca_acc_cnt(hcap); 411 return (status); 412 } 413 414 ibmf_hdl = hcap->hca_port_info[port_no - 1].port_ibmf_hdl; 415 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 416 "primary ibmf_hdl = 0x%p", channel, ibmf_hdl); 417 418 419 primary_slid = base_lid + IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 420 421 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: channel 0x%p " 422 "primary SLID = %x", channel, primary_slid); 423 424 /* check first if alternate path exists or not as it is OPTIONAL */ 425 if (IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num != 0) { 426 uint8_t alt_port_no; 427 428 alt_port_no = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 429 430 if (chan_args->oc_path->pi_alt_pkt_lt > ibcm_max_ib_pkt_lt) { 431 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 432 "Huge Alt Pkt lt %d", channel, 433 chan_args->oc_path->pi_alt_pkt_lt); 434 ibcm_dec_hca_acc_cnt(hcap); 435 return (IBT_PATH_PKT_LT_TOO_HIGH); 436 } 437 438 if (port_no != alt_port_no) { 439 440 status = ibt_get_port_state_byguid(hcap->hca_guid, 441 alt_port_no, NULL, &base_lid); 442 if (status != IBT_SUCCESS) { 443 444 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 445 "chan 0x%p alt_port_num %d inactive %d", 446 channel, alt_port_no, status); 447 ibcm_dec_hca_acc_cnt(hcap); 448 return (status); 449 } 450 451 } 452 alternate_slid = 453 base_lid + IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 454 455 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan %0xp " 456 "alternate SLID = %x", channel, alternate_slid); 457 } 458 459 /* 460 * only pkey needs to be zero'ed, because all other fields are set in 461 * in ibcm_init_reply_addr. But, let's bzero the complete struct for 462 * any future modifications. 463 */ 464 bzero(&cm_reply_addr, sizeof (cm_reply_addr)); 465 466 /* Initialize the MAD destination address in stored_reply_addr */ 467 if ((status = ibcm_init_reply_addr(hcap, &cm_reply_addr, chan_args, 468 flags, &cm_pkt_lt, primary_slid)) != IBT_SUCCESS) { 469 470 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 471 "ibcm_init_reply_addr failed status %d ", channel, status); 472 ibcm_dec_hca_acc_cnt(hcap); 473 return (status); 474 } 475 476 477 /* Initialize the pkey for CM MAD communication */ 478 if (cm_reply_addr.rcvd_addr.ia_p_key == 0) 479 cm_reply_addr.rcvd_addr.ia_p_key = prim_pkey; 480 481 #ifdef DEBUG 482 ibcm_print_reply_addr(channel, &cm_reply_addr); 483 #endif 484 485 /* Retrieve an ibmf qp for sending CM MADs */ 486 if ((cm_qp_entry = ibcm_find_qp(hcap, port_no, 487 cm_reply_addr.rcvd_addr.ia_p_key)) == NULL) { 488 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p " 489 "unable to allocate ibmf qp for CM MADs", channel); 490 ibcm_dec_hca_acc_cnt(hcap); 491 return (IBT_INSUFF_RESOURCE); 492 } 493 494 495 if (ibcm_alloc_comid(hcap, &local_comid) != IBCM_SUCCESS) { 496 ibcm_release_qp(cm_qp_entry); 497 ibcm_dec_hca_acc_cnt(hcap); 498 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan 0x%p" 499 " Unable to allocate comid", channel); 500 return (IBT_INSUFF_KERNEL_RESOURCE); 501 } 502 503 /* allocate an IBMF mad buffer (REQ) */ 504 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, 505 MAD_METHOD_SEND)) != IBT_SUCCESS) { 506 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 507 "chan 0x%p ibcm_alloc_out_msg failed", channel); 508 ibcm_release_qp(cm_qp_entry); 509 ibcm_free_comid(hcap, local_comid); 510 ibcm_dec_hca_acc_cnt(hcap); 511 return (status); 512 } 513 514 /* allocate an IBMF mad buffer (DREQ) */ 515 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg_dreq, 516 MAD_METHOD_SEND)) != IBT_SUCCESS) { 517 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 518 "chan 0x%p ibcm_alloc_out_msg failed", channel); 519 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 520 ibcm_release_qp(cm_qp_entry); 521 ibcm_free_comid(hcap, local_comid); 522 ibcm_dec_hca_acc_cnt(hcap); 523 return (status); 524 } 525 526 /* Init to Init, if QP's port does not match with path information */ 527 if (qp_query_attr.qp_info.qp_transport.rc.rc_path.cep_hca_port_num != 528 IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num) { 529 530 ibt_qp_info_t qp_info; 531 ibt_cep_modify_flags_t cep_flags; 532 533 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 534 "chan 0x%p chan port %d", channel, 535 qp_query_attr.qp_info.qp_transport.rc.rc_path.\ 536 cep_hca_port_num); 537 538 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 539 "chan 0x%p path port %d", channel, port_no); 540 541 bzero(&qp_info, sizeof (qp_info)); 542 /* For now, set it to RC type */ 543 544 qp_info.qp_trans = IBT_RC_SRV; 545 qp_info.qp_state = IBT_STATE_INIT; 546 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = port_no; 547 548 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 549 550 status = ibt_modify_qp(channel, cep_flags, &qp_info, NULL); 551 552 if (status != IBT_SUCCESS) { 553 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: " 554 "chan 0x%p ibt_modify_qp() = %d", channel, status); 555 ibcm_release_qp(cm_qp_entry); 556 ibcm_free_comid(hcap, local_comid); 557 ibcm_dec_hca_acc_cnt(hcap); 558 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg); 559 (void) ibcm_free_out_msg(ibmf_hdl, &ibmf_msg_dreq); 560 return (status); 561 } else 562 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: " 563 "chan 0x%p ibt_modify_qp() = %d", channel, status); 564 } 565 566 /* allocate ibcm_state_data_t before grabbing the WRITER lock */ 567 statep = kmem_zalloc(sizeof (ibcm_state_data_t), KM_SLEEP); 568 rw_enter(&hcap->hca_state_rwlock, RW_WRITER); 569 lkup_status = ibcm_lookup_msg(IBCM_OUTGOING_REQ, local_comid, 0, 0, 570 hcap, &statep); 571 rw_exit(&hcap->hca_state_rwlock); 572 573 /* CM should be seeing this for the first time */ 574 ASSERT(lkup_status == IBCM_LOOKUP_NEW); 575 576 /* Increment the hca's resource count */ 577 ibcm_inc_hca_res_cnt(hcap); 578 579 /* Once a resource created on hca, no need to hold the acc cnt */ 580 ibcm_dec_hca_acc_cnt(hcap); 581 582 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 583 584 statep->timerid = 0; 585 statep->local_hca_guid = hca_guid; 586 statep->local_qpn = local_qpn; 587 statep->stored_reply_addr.cm_qp_entry = cm_qp_entry; 588 statep->prim_port = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 589 statep->alt_port = IBCM_ALT_CEP_PATH(chan_args).cep_hca_port_num; 590 591 592 /* Save "statep" as channel's CM private data. */ 593 statep->channel = channel; 594 IBCM_SET_CHAN_PRIVATE(statep->channel, statep); 595 596 statep->stored_msg = ibmf_msg; 597 statep->dreq_msg = ibmf_msg_dreq; 598 599 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*req_msgp)) 600 601 /* Start filling in the REQ MAD */ 602 req_msgp = (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg); 603 req_msgp->req_local_comm_id = h2b32(local_comid); 604 req_msgp->req_svc_id = h2b64(chan_args->oc_path->pi_sid); 605 req_msgp->req_local_ca_guid = h2b64(hca_guid); 606 req_msgp->req_local_qkey = h2b32(local_qkey); /* for EEC/RD */ 607 608 /* Bytes 32-35 are req_local_qpn and req_off_resp_resources */ 609 req_msgp->req_local_qpn_plus = h2b32(local_qpn << 8 | rdma_in); 610 611 /* Bytes 36-39 are req_local_eec_no and req_off_initiator_depth */ 612 req_msgp->req_local_eec_no_plus = h2b32(local_eecn << 8 | rdma_out); 613 614 if (flags & IBT_OCHAN_REMOTE_CM_TM) 615 remote_cm_resp_time = chan_args->oc_remote_cm_time; 616 else 617 remote_cm_resp_time = ibcm_remote_response_time; 618 619 /* 620 * Bytes 40-43 - remote_eecn, remote_cm_resp_time, tran_type, 621 * IBT_CM_FLOW_CONTROL is always set by default. 622 */ 623 req_msgp->req_remote_eecn_plus = h2b32( 624 remote_eecn << 8 | (ibt_usec2ib(remote_cm_resp_time) & 0x1f) << 3 | 625 IBT_RC_SRV << 1 | IBT_CM_FLOW_CONTROL); 626 627 if (flags & IBT_OCHAN_LOCAL_CM_TM) 628 local_cm_proc_time = chan_args->oc_local_cm_time; 629 else 630 local_cm_proc_time = ibcm_local_processing_time; 631 632 local_cm_resp_time = ibt_usec2ib(local_cm_proc_time + 633 2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt) + 634 ibcm_sw_delay); 635 636 /* save retry count */ 637 statep->cep_retry_cnt = chan_args->oc_path_retry_cnt; 638 639 if (flags & IBT_OCHAN_STARTING_PSN) 640 starting_psn = chan_args->oc_starting_psn; 641 642 if (local_cm_resp_time > 0x1f) 643 local_cm_resp_time = 0x1f; 644 645 /* Bytes 44-47 are req_starting_psn, local_cm_resp_time and retry_cnt */ 646 req_msgp->req_starting_psn_plus = h2b32(starting_psn << 8 | 647 local_cm_resp_time << 3 | statep->cep_retry_cnt); 648 649 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 650 "Prim Pkt lt (IB time) 0x%x", channel, 651 chan_args->oc_path->pi_prim_pkt_lt); 652 653 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 654 "local_cm_proc_time(usec) %d ", channel, local_cm_proc_time); 655 656 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 657 "local_cm_resp_time(ib_time) %d", channel, local_cm_resp_time); 658 659 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 660 "remote_cm_resp_time (usec) %d", channel, remote_cm_resp_time); 661 662 statep->starting_psn = starting_psn; 663 664 /* Pkey - bytes 48-49 */ 665 req_msgp->req_part_key = h2b16(prim_pkey); 666 667 if (flags & IBT_OCHAN_CM_RETRY) 668 cm_retries = chan_args->oc_cm_retry_cnt; 669 else 670 cm_retries = ibcm_max_retries; 671 672 statep->max_cm_retries = statep->remaining_retry_cnt = cm_retries; 673 req_msgp->req_max_cm_retries_plus = statep->max_cm_retries << 4; 674 675 /* 676 * Check whether SRQ is associated with this Channel, if yes, then 677 * set the SRQ Exists bit in the REQ. 678 */ 679 if (qp_query_attr.qp_srq != NULL) { 680 req_msgp->req_max_cm_retries_plus |= (1 << 3); 681 } 682 683 /* 684 * By default on Tavor, we override the PathMTU to 1K. 685 * To turn this off, set ibcm_override_path_mtu = 0. 686 */ 687 if (ibcm_override_path_mtu && IBCM_IS_HCA_TAVOR(hcap) && 688 (chan_args->oc_path->pi_path_mtu > IB_MTU_1K)) { 689 req_msgp->req_mtu_plus = IB_MTU_1K << 4 | 690 chan_args->oc_path_rnr_retry_cnt; 691 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p PathMTU" 692 " overidden to IB_MTU_1K(%d) from %d", channel, IB_MTU_1K, 693 chan_args->oc_path->pi_path_mtu); 694 } else 695 req_msgp->req_mtu_plus = chan_args->oc_path->pi_path_mtu << 4 | 696 chan_args->oc_path_rnr_retry_cnt; 697 698 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p CM retry cnt %d" 699 " staring PSN %x", channel, cm_retries, starting_psn); 700 701 702 #ifdef NO_EEC_SUPPORT_YET 703 if (flags & IBT_OCHAN_RDC_EXISTS) 704 req_msgp->req_mtu_plus |= 8; 705 #endif 706 707 /* Initialize the "primary" port stuff next - bytes 52-95 */ 708 req_msgp->req_primary_l_port_lid = h2b16(primary_slid); 709 req_msgp->req_primary_r_port_lid = 710 h2b16(IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 711 req_msgp->req_primary_l_port_gid.gid_prefix = 712 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_prefix); 713 req_msgp->req_primary_l_port_gid.gid_guid = 714 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_sgid.gid_guid); 715 req_msgp->req_primary_r_port_gid.gid_prefix = 716 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix); 717 req_msgp->req_primary_r_port_gid.gid_guid = 718 h2b64(IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 719 primary_grh = IBCM_PRIM_ADDS_VECT(chan_args).av_send_grh; 720 721 statep->remote_hca_guid = /* not correct, but helpful for debugging */ 722 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid; 723 724 /* Bytes 88-91 - primary_flowlbl, and primary_srate */ 725 req_msgp->req_primary_flow_label_plus = 726 h2b32(((primary_grh == B_TRUE) ? 727 (IBCM_PRIM_ADDS_VECT(chan_args).av_flow << 12) : 0) | 728 IBCM_PRIM_ADDS_VECT(chan_args).av_srate); 729 req_msgp->req_primary_traffic_class = (primary_grh == B_TRUE) ? 730 IBCM_PRIM_ADDS_VECT(chan_args).av_tclass : 0; 731 req_msgp->req_primary_hop_limit = (primary_grh == B_TRUE) ? 732 IBCM_PRIM_ADDS_VECT(chan_args).av_hop : 1; 733 req_msgp->req_primary_sl_plus = 734 IBCM_PRIM_ADDS_VECT(chan_args).av_srvl << 4 | 735 ((primary_grh == B_TRUE) ? 0 : 8); 736 737 req_msgp->req_primary_localtime_plus = 738 ibt_usec2ib((2 * ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt)) + 739 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 740 741 IBTF_DPRINTF_L2(cmlog, "ibt_open_rc_channel: chan %p statep %p", 742 channel, statep); 743 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 744 "active hca_ack_delay (usec) %d", channel, 745 req_msgp->req_primary_localtime_plus); 746 747 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 748 "Sent primary cep timeout (IB Time) %d", channel, 749 hcap->hca_ack_delay); 750 751 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p prim_dlid %x ", 752 channel, IBCM_PRIM_ADDS_VECT(chan_args).av_dlid); 753 754 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 755 "prim GID %llX:%llX", channel, 756 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_prefix, 757 IBCM_PRIM_ADDS_VECT(chan_args).av_dgid.gid_guid); 758 759 /* Initialize the "alternate" port stuff - optional */ 760 if (chan_args->oc_path->pi_alt_cep_path.cep_hca_port_num != 0) { 761 ib_gid_t tmp_gid; 762 763 req_msgp->req_alt_l_port_lid = h2b16(alternate_slid); 764 req_msgp->req_alt_r_port_lid = 765 h2b16(IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 766 /* 767 * doing all this as req_alt_r/l_port_gid is at offset 768 * 100, 116 which is not divisible by 8 769 */ 770 771 tmp_gid.gid_prefix = 772 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix); 773 tmp_gid.gid_guid = 774 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 775 bcopy(&tmp_gid, &req_msgp->req_alt_r_port_gid[0], 776 sizeof (ib_gid_t)); 777 tmp_gid.gid_prefix = 778 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_prefix); 779 tmp_gid.gid_guid = 780 h2b64(IBCM_ALT_ADDS_VECT(chan_args).av_sgid.gid_guid); 781 782 bcopy(&tmp_gid, &req_msgp->req_alt_l_port_gid[0], 783 sizeof (ib_gid_t)); 784 alternate_grh = IBCM_ALT_ADDS_VECT(chan_args).av_send_grh; 785 786 /* Bytes 132-135 - alternate_flow_label, and alternate srate */ 787 req_msgp->req_alt_flow_label_plus = h2b32( 788 (((alternate_grh == B_TRUE) ? 789 (IBCM_ALT_ADDS_VECT(chan_args).av_flow << 12) : 0) | 790 IBCM_ALT_ADDS_VECT(chan_args).av_srate)); 791 req_msgp->req_alt_traffic_class = (alternate_grh == B_TRUE) ? 792 IBCM_ALT_ADDS_VECT(chan_args).av_tclass : 0; 793 req_msgp->req_alt_hop_limit = (alternate_grh == B_TRUE) ? 794 IBCM_ALT_ADDS_VECT(chan_args).av_hop : 1; 795 req_msgp->req_alt_sl_plus = 796 IBCM_ALT_ADDS_VECT(chan_args).av_srvl << 4 | 797 ((alternate_grh == B_TRUE) ? 0 : 8); 798 req_msgp->req_alt_localtime_plus = ibt_usec2ib((2 * 799 ibt_ib2usec(chan_args->oc_path->pi_alt_pkt_lt)) + 800 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 801 802 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 803 "alt_dlid %x ", channel, 804 IBCM_ALT_ADDS_VECT(chan_args).av_dlid); 805 806 IBTF_DPRINTF_L5(cmlog, "ibt_open_rc_channel: chan 0x%p " 807 "alt GID %llX:%llX", channel, 808 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_prefix, 809 IBCM_ALT_ADDS_VECT(chan_args).av_dgid.gid_guid); 810 } 811 812 len = min(chan_args->oc_priv_data_len, IBT_REQ_PRIV_DATA_SZ); 813 if ((len > 0) && chan_args->oc_priv_data) 814 bcopy(chan_args->oc_priv_data, req_msgp->req_private_data, len); 815 816 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*req_msgp)) 817 818 /* return_data is filled up in the state machine code */ 819 if (ret_args != NULL) { 820 statep->open_return_data = ret_args; 821 } 822 823 /* initialize some statep fields here */ 824 statep->mode = IBCM_ACTIVE_MODE; 825 statep->hcap = hcap; 826 827 statep->cm_handler = chan_args->oc_cm_handler; 828 statep->state_cm_private = chan_args->oc_cm_clnt_private; 829 830 statep->pkt_life_time = 831 ibt_ib2usec(chan_args->oc_path->pi_prim_pkt_lt); 832 833 statep->timer_value = ibt_ib2usec(ibt_usec2ib( 834 2 * ibt_ib2usec(cm_pkt_lt) + remote_cm_resp_time)); 835 836 /* Initialize statep->stored_reply_addr */ 837 statep->stored_reply_addr.ibmf_hdl = ibmf_hdl; 838 839 /* Initialize stored reply addr fields */ 840 statep->stored_reply_addr.grh_hdr = cm_reply_addr.grh_hdr; 841 statep->stored_reply_addr.rcvd_addr = cm_reply_addr.rcvd_addr; 842 statep->stored_reply_addr.grh_exists = cm_reply_addr.grh_exists; 843 statep->stored_reply_addr.port_num = cm_reply_addr.port_num; 844 845 /* 846 * The IPD on local/active side is calculated by path functions, 847 * hence available in the args of ibt_open_rc_channel 848 */ 849 statep->local_srate = IBCM_PRIM_ADDS_VECT(chan_args).av_srate; 850 statep->local_alt_srate = IBCM_ALT_ADDS_VECT(chan_args).av_srate; 851 852 /* Store the source path bits for primary and alt paths */ 853 statep->prim_src_path_bits = IBCM_PRIM_ADDS_VECT(chan_args).av_src_path; 854 statep->alt_src_path_bits = IBCM_ALT_ADDS_VECT(chan_args).av_src_path; 855 856 statep->open_flow = 1; 857 statep->open_done = B_FALSE; 858 statep->state = statep->timer_stored_state = IBCM_STATE_REQ_SENT; 859 IBCM_REF_CNT_INCR(statep); /* Decremented before return */ 860 IBCM_REF_CNT_INCR(statep); /* Decremented after REQ is posted */ 861 statep->send_mad_flags |= IBCM_REQ_POST_BUSY; 862 863 IBCM_OUT_HDRP(statep->stored_msg)->AttributeID = 864 h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID); 865 866 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID = 867 h2b64(ibcm_generate_tranid(IBCM_INCOMING_REQ, statep->local_comid, 868 0)); 869 870 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 871 872 ibcm_open_enqueue(statep); 873 874 mutex_enter(&statep->state_mutex); 875 876 if (mode == IBT_BLOCKING) { 877 878 /* wait for REQ/REP/RTU */ 879 while (statep->open_done != B_TRUE) { 880 cv_wait(&statep->block_client_cv, &statep->state_mutex); 881 } 882 883 /* 884 * In the case that open_channel() fails because of a 885 * REJ or timeout, change retval to IBT_CM_FAILURE 886 */ 887 if (statep->open_return_data->rc_status != IBT_CM_ACCEPT) 888 status = IBT_CM_FAILURE; 889 890 IBTF_DPRINTF_L3(cmlog, "ibt_open_rc_channel: chan 0x%p " 891 "ret status %d cm status %d", channel, status, 892 statep->open_return_data->rc_status); 893 } 894 895 /* decrement the ref-count before leaving here */ 896 IBCM_REF_CNT_DECR(statep); 897 898 mutex_exit(&statep->state_mutex); 899 900 IBTF_DPRINTF_L4(cmlog, "ibt_open_rc_channel: chan 0x%p done", channel); 901 return (status); 902 } 903 904 /* 905 * ibcm_init_reply_addr: 906 * 907 * The brief description of functionality below. 908 * 909 * For IBT_OCHAN_PORT_REDIRECTED (ie., port redirected case): 910 * Build CM path from chan_args->oc_cm_cep_path 911 * Set CM pkt lt (ie.,life time) to chan_args->oc_cm_pkt_lt 912 * 913 * For IBT_OCHAN_REDIRECTED (ie., port and CM redirected case): 914 * If Redirect LID is specified, 915 * If Redirect GID is not specified or specified to be on the same 916 * subnet, then 917 * Build CM path from chan_args->oc_cm_redirect_info 918 * Set CM pkt lt to subnet timeout 919 * Else (ie., GID specified, but on a different subnet) 920 * Do a path lookup to build CM Path and set CM pkt lt 921 * 922 */ 923 static ibt_status_t 924 ibcm_init_reply_addr(ibcm_hca_info_t *hcap, ibcm_mad_addr_t *reply_addr, 925 ibt_chan_open_args_t *chan_args, ibt_chan_open_flags_t flags, 926 ib_time_t *cm_pkt_lt, ib_lid_t prim_slid) 927 { 928 ibt_adds_vect_t *cm_adds; 929 ibt_path_info_t path; 930 boolean_t cm_grh; 931 ibt_status_t status; 932 933 IBTF_DPRINTF_L5(cmlog, "ibcm_init_reply_addr:"); 934 935 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*reply_addr)) 936 937 /* 938 * sending side CM lid/gid/port num are not based on any redirect 939 * params. These values are set to primary RC path lid/gid/port num. 940 * In the future, these values can be set based on framework policy 941 * decisions ensuring reachability. 942 */ 943 reply_addr->grh_hdr.ig_sender_gid = 944 IBCM_PRIM_ADDS_VECT(chan_args).av_sgid; 945 reply_addr->rcvd_addr.ia_local_lid = prim_slid; 946 reply_addr->port_num = IBCM_PRIM_CEP_PATH(chan_args).cep_hca_port_num; 947 948 if (flags & IBT_OCHAN_PORT_REDIRECTED) { 949 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 950 "IBT_OCHAN_PORT_REDIRECTED specified"); 951 952 status = ibt_index2pkey_byguid(hcap->hca_guid, 953 chan_args->oc_cm_cep_path->cep_hca_port_num, 954 chan_args->oc_cm_cep_path->cep_pkey_ix, 955 &reply_addr->rcvd_addr.ia_p_key); 956 957 if (status != IBT_SUCCESS) { 958 IBTF_DPRINTF_L2(cmlog, "ibcm_init_rely_addr: Invalid " 959 "CM PKeyIx %x port_num %x", 960 chan_args->oc_cm_cep_path->cep_pkey_ix, 961 chan_args->oc_cm_cep_path->cep_hca_port_num); 962 return (status); 963 } 964 965 cm_adds = &(chan_args->oc_cm_cep_path->cep_adds_vect); 966 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: dlid = %x", 967 cm_adds->av_dlid); 968 969 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 970 reply_addr->rcvd_addr.ia_remote_qno = 1; 971 *cm_pkt_lt = chan_args->oc_cm_pkt_lt; 972 973 } else if (flags & IBT_OCHAN_REDIRECTED) { 974 ibt_redirect_info_t *redirect_info; 975 ibt_hca_portinfo_t *port_infop; 976 uint_t psize, nports; 977 978 IBTF_DPRINTF_L4(cmlog, "ibcm_init_rely_addr: " 979 "IBT_OCHAN_REDIRECTED specified"); 980 981 redirect_info = chan_args->oc_cm_redirect_info; 982 983 if ((redirect_info->rdi_gid.gid_prefix == 0) || 984 (redirect_info->rdi_gid.gid_guid == 0)) { 985 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 986 "ERROR: Re-direct GID value NOT Provided."); 987 return (IBT_INVALID_PARAM); 988 } 989 990 /* As per spec definition 1.1, it's always IB_GSI_QKEY */ 991 reply_addr->rcvd_addr.ia_q_key = redirect_info->rdi_qkey; 992 reply_addr->rcvd_addr.ia_remote_qno = redirect_info->rdi_qpn; 993 reply_addr->rcvd_addr.ia_p_key = redirect_info->rdi_pkey; 994 995 /* 996 * if LID is non-zero in classportinfo then use classportinfo 997 * fields to form CM MAD destination address. 998 */ 999 if (redirect_info->rdi_dlid != 0) { 1000 status = ibtl_cm_query_hca_ports_byguid(hcap->hca_guid, 1001 reply_addr->port_num, &port_infop, &nports, &psize); 1002 if ((status != IBT_SUCCESS) || (nports == 0)) { 1003 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 1004 "Query Ports Failed: %d", status); 1005 return (status); 1006 } else if (port_infop->p_subnet_timeout > 1007 ibcm_max_ib_pkt_lt) { 1008 IBTF_DPRINTF_L2(cmlog, "ibcm_init_reply_addr: " 1009 "large subnet timeout %x port_no %x", 1010 port_infop->p_subnet_timeout, 1011 reply_addr->port_num); 1012 ibt_free_portinfo(port_infop, psize); 1013 return (IBT_PATH_PKT_LT_TOO_HIGH); 1014 } else { 1015 IBTF_DPRINTF_L3(cmlog, "ibcm_init_reply_addr: " 1016 "subnet timeout %x port_no %x", 1017 port_infop->p_subnet_timeout, 1018 reply_addr->port_num); 1019 1020 *cm_pkt_lt = 1021 ibt_ib2usec(min(ibcm_max_ib_mad_pkt_lt, 1022 port_infop->p_subnet_timeout)); 1023 1024 ibt_free_portinfo(port_infop, psize); 1025 } 1026 1027 reply_addr->rcvd_addr.ia_remote_lid = 1028 redirect_info->rdi_dlid; 1029 reply_addr->rcvd_addr.ia_service_level = 1030 redirect_info->rdi_sl; 1031 reply_addr->grh_exists = B_TRUE; 1032 reply_addr->grh_hdr.ig_recver_gid = 1033 redirect_info->rdi_gid; 1034 reply_addr->grh_hdr.ig_tclass = 1035 redirect_info->rdi_tclass; 1036 reply_addr->grh_hdr.ig_flow_label = 1037 redirect_info->rdi_flow; 1038 1039 /* Classportinfo doesn't have hoplimit field */ 1040 reply_addr->grh_hdr.ig_hop_limit = 1; 1041 return (IBT_SUCCESS); 1042 1043 } else { 1044 ibt_path_attr_t path_attr; 1045 ib_gid_t path_dgid[1]; 1046 1047 /* 1048 * If GID is specified, and LID is zero in classportinfo 1049 * do a path lookup using specified GID, Pkey, 1050 * in classportinfo 1051 */ 1052 1053 bzero(&path_attr, sizeof (path_attr)); 1054 1055 path_attr.pa_dgids = &path_dgid[0]; 1056 path_attr.pa_dgids[0] = redirect_info->rdi_gid; 1057 1058 /* 1059 * use reply_addr below, as sender_gid in reply_addr 1060 * may have been set above based on some policy decision 1061 * for originating end point for CM MADs above 1062 */ 1063 path_attr.pa_sgid = reply_addr->grh_hdr.ig_sender_gid; 1064 path_attr.pa_num_dgids = 1; 1065 path_attr.pa_pkey = redirect_info->rdi_pkey; 1066 1067 if ((status = ibt_get_paths(ibcm_ibt_handle, 1068 IBT_PATH_PKEY, &path_attr, 1, &path, NULL)) != 1069 IBT_SUCCESS) 1070 return (status); 1071 1072 /* Initialize cm_adds */ 1073 cm_adds = &path.pi_prim_cep_path.cep_adds_vect; 1074 *cm_pkt_lt = path.pi_prim_pkt_lt; 1075 } 1076 1077 } else { /* cm_pkey initialized in ibt_open_rc_channel */ 1078 reply_addr->rcvd_addr.ia_q_key = IB_GSI_QKEY; 1079 reply_addr->rcvd_addr.ia_remote_qno = 1; 1080 *cm_pkt_lt = chan_args->oc_path->pi_prim_pkt_lt; 1081 cm_adds = &(IBCM_PRIM_ADDS_VECT(chan_args)); 1082 } 1083 1084 1085 cm_grh = cm_adds->av_send_grh; 1086 reply_addr->grh_exists = cm_grh; 1087 1088 reply_addr->rcvd_addr.ia_remote_lid = 1089 cm_adds->av_dlid; 1090 reply_addr->grh_hdr.ig_recver_gid = 1091 cm_adds->av_dgid; 1092 reply_addr->grh_hdr.ig_flow_label = 1093 cm_adds->av_flow & IB_GRH_FLOW_LABEL_MASK; 1094 reply_addr->grh_hdr.ig_tclass = 1095 (cm_grh == B_TRUE) ? cm_adds->av_tclass : 0; 1096 reply_addr->grh_hdr.ig_hop_limit = 1097 (cm_grh == B_TRUE) ? cm_adds->av_hop : 1; 1098 reply_addr->rcvd_addr.ia_service_level = 1099 cm_adds->av_srvl; 1100 1101 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*reply_addr)) 1102 1103 return (IBT_SUCCESS); 1104 } 1105 1106 1107 /* 1108 * ibt_prime_close_rc_channel() 1109 * It allocates resources required for close channel operation, so 1110 * ibt_close_rc_channel can be called from interrupt routine. 1111 * 1112 * INPUTS: 1113 * channel The address of an ibt_channel_t struct that 1114 * specifies the channel to open. 1115 * 1116 * RETURN VALUES: 1117 * IBT_SUCCESS on success(or respective failure on error) 1118 * 1119 * Clients are typically expected to call this function in established state 1120 */ 1121 ibt_status_t 1122 ibt_prime_close_rc_channel(ibt_channel_hdl_t channel) 1123 { 1124 ibcm_state_data_t *statep; 1125 ibt_status_t status = IBT_SUCCESS; 1126 1127 IBTF_DPRINTF_L3(cmlog, "ibt_prime_close_rc_channel(%p)", channel); 1128 1129 /* validate channel, first */ 1130 if (IBCM_INVALID_CHANNEL(channel)) { 1131 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1132 "invalid channel", channel); 1133 return (IBT_CHAN_HDL_INVALID); 1134 } 1135 1136 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1137 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1138 "Invalid Channel type: Applicable only to RC Channel", 1139 channel); 1140 return (IBT_CHAN_SRV_TYPE_INVALID); 1141 } 1142 1143 /* get the statep */ 1144 IBCM_GET_CHAN_PRIVATE(channel, statep); 1145 1146 /* 1147 * This can happen, if the statep is already gone by a DREQ from 1148 * the remote side 1149 */ 1150 1151 if (statep == NULL) { 1152 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1153 "statep NULL", channel); 1154 return (IBT_SUCCESS); 1155 } 1156 1157 mutex_enter(&statep->state_mutex); 1158 IBCM_RELEASE_CHAN_PRIVATE(channel); 1159 if (statep->state != IBCM_STATE_ESTABLISHED) { 1160 mutex_exit(&statep->state_mutex); 1161 return (IBT_CHAN_STATE_INVALID); 1162 } 1163 IBCM_REF_CNT_INCR(statep); 1164 IBTF_DPRINTF_L4(cmlog, "ibt_prime_close_rc_channel: chan 0x%p statep %p" 1165 " state %x", channel, statep, statep->state); 1166 mutex_exit(&statep->state_mutex); 1167 1168 /* clients could pre-allocate dreq mad, even before connection est */ 1169 if (statep->dreq_msg == NULL) 1170 status = ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, 1171 &statep->dreq_msg, MAD_METHOD_SEND); 1172 1173 mutex_enter(&statep->state_mutex); 1174 IBCM_REF_CNT_DECR(statep); 1175 mutex_exit(&statep->state_mutex); 1176 1177 if (status != IBT_SUCCESS) { 1178 IBTF_DPRINTF_L2(cmlog, "ibt_prime_close_rc_channel: chan 0x%p " 1179 "ibcm_alloc_out_msg failed ", channel); 1180 return (status); 1181 } 1182 1183 /* If this message isn't seen then ibt_prime_close_rc_channel failed */ 1184 IBTF_DPRINTF_L5(cmlog, "ibt_prime_close_rc_channel: chan 0x%p done", 1185 channel); 1186 1187 return (IBT_SUCCESS); 1188 } 1189 1190 /* 1191 * ibt_close_rc_channel() 1192 * It closes an established channel. 1193 * 1194 * RETURN VALUES: 1195 * IBT_SUCCESS on success(or respective failure on error) 1196 */ 1197 ibt_status_t 1198 ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 1199 void *priv_data, ibt_priv_data_len_t priv_data_len, uint8_t *ret_status, 1200 void *ret_priv_data, ibt_priv_data_len_t *ret_priv_data_len_p) 1201 { 1202 ibcm_state_data_t *statep; 1203 1204 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel(%p, %x, %p, %d, %p)", 1205 channel, mode, priv_data, priv_data_len, 1206 (ret_priv_data_len_p == NULL) ? 0 : *ret_priv_data_len_p); 1207 1208 /* validate channel, first */ 1209 if (IBCM_INVALID_CHANNEL(channel)) { 1210 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1211 "invalid channel", channel); 1212 return (IBT_CHAN_HDL_INVALID); 1213 } 1214 1215 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 1216 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1217 "Invalid Channel type: Applicable only to RC Channel", 1218 channel); 1219 return (IBT_CHAN_SRV_TYPE_INVALID); 1220 } 1221 1222 if (mode == IBT_BLOCKING) { 1223 /* valid only for BLOCKING MODE */ 1224 if ((ret_priv_data_len_p != NULL) && 1225 (*ret_priv_data_len_p > IBT_DREP_PRIV_DATA_SZ)) { 1226 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p" 1227 " private data len %d is too large", channel, 1228 *ret_priv_data_len_p); 1229 return (IBT_INVALID_PARAM); 1230 } 1231 } else if ((mode != IBT_NONBLOCKING) && (mode != IBT_NOCALLBACKS)) { 1232 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1233 "invalid mode %x specified", channel, mode); 1234 return (IBT_INVALID_PARAM); 1235 } 1236 1237 if (ibtl_cm_is_chan_closing(channel) || 1238 ibtl_cm_is_chan_closed(channel)) { 1239 if (ret_status) 1240 *ret_status = IBT_CM_CLOSED_ALREADY; 1241 1242 /* No private data to return to the client */ 1243 if (ret_priv_data_len_p != NULL) 1244 *ret_priv_data_len_p = 0; 1245 1246 if ((mode == IBT_BLOCKING) || 1247 (mode == IBT_NOCALLBACKS)) { 1248 IBCM_GET_CHAN_PRIVATE(channel, statep); 1249 if (statep == NULL) 1250 return (IBT_SUCCESS); 1251 mutex_enter(&statep->state_mutex); 1252 IBCM_RELEASE_CHAN_PRIVATE(channel); 1253 IBCM_REF_CNT_INCR(statep); 1254 while (statep->close_done != B_TRUE) 1255 cv_wait(&statep->block_client_cv, 1256 &statep->state_mutex); 1257 IBCM_REF_CNT_DECR(statep); 1258 mutex_exit(&statep->state_mutex); 1259 } 1260 1261 IBTF_DPRINTF_L3(cmlog, "ibt_close_rc_channel: chan 0x%p " 1262 "already marked for closing", channel); 1263 1264 return (IBT_SUCCESS); 1265 } 1266 1267 /* get the statep */ 1268 IBCM_GET_CHAN_PRIVATE(channel, statep); 1269 if (statep == NULL) { 1270 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1271 "statep NULL", channel); 1272 return (IBT_CHAN_STATE_INVALID); 1273 } 1274 1275 mutex_enter(&statep->state_mutex); 1276 1277 if (statep->dreq_msg == NULL) { 1278 IBTF_DPRINTF_L2(cmlog, "ibt_close_rc_channel: chan 0x%p " 1279 "Fatal Error: dreq_msg is NULL", channel); 1280 IBCM_RELEASE_CHAN_PRIVATE(channel); 1281 mutex_exit(&statep->state_mutex); 1282 return (IBT_CHAN_STATE_INVALID); 1283 } 1284 1285 if ((ret_priv_data == NULL) || (ret_priv_data_len_p == NULL)) { 1286 statep->close_ret_priv_data = NULL; 1287 statep->close_ret_priv_data_len = NULL; 1288 } else { 1289 statep->close_ret_priv_data = ret_priv_data; 1290 statep->close_ret_priv_data_len = ret_priv_data_len_p; 1291 } 1292 1293 priv_data_len = min(priv_data_len, IBT_DREQ_PRIV_DATA_SZ); 1294 if ((priv_data != NULL) && (priv_data_len > 0)) { 1295 bcopy(priv_data, ((ibcm_dreq_msg_t *) 1296 IBCM_OUT_MSGP(statep->dreq_msg))->dreq_private_data, 1297 priv_data_len); 1298 } 1299 statep->close_ret_status = ret_status; 1300 1301 IBCM_RELEASE_CHAN_PRIVATE(channel); 1302 IBCM_REF_CNT_INCR(statep); 1303 1304 if (mode != IBT_NONBLOCKING) { 1305 return (ibcm_close_rc_channel(channel, statep, mode)); 1306 } 1307 1308 /* IBT_NONBLOCKING */ 1309 ibcm_close_enqueue(statep); 1310 mutex_exit(&statep->state_mutex); 1311 1312 return (IBT_SUCCESS); 1313 } 1314 1315 void 1316 ibcm_close_start(ibcm_state_data_t *statep) 1317 { 1318 mutex_enter(&statep->state_mutex); 1319 (void) ibcm_close_rc_channel(statep->channel, statep, IBT_NONBLOCKING); 1320 } 1321 1322 static 1323 ibt_status_t 1324 ibcm_close_rc_channel(ibt_channel_hdl_t channel, ibcm_state_data_t *statep, 1325 ibt_execution_mode_t mode) 1326 { 1327 ibcm_hca_info_t *hcap; 1328 1329 _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&statep->state_mutex)); 1330 ASSERT(MUTEX_HELD(&statep->state_mutex)); 1331 1332 IBTF_DPRINTF_L3(cmlog, "ibcm_close_rc_channel: chan 0x%p statep %p", 1333 channel, statep); 1334 1335 hcap = statep->hcap; 1336 1337 /* HCA must have been in active state. If not, it's a client bug */ 1338 if (!IBCM_ACCESS_HCA_OK(hcap)) { 1339 IBTF_DPRINTF_L2(cmlog, "ibcm_close_rc_channel: chan 0x%p " 1340 "hcap 0x%p not active", channel, hcap); 1341 IBCM_REF_CNT_DECR(statep); 1342 mutex_exit(&statep->state_mutex); 1343 return (IBT_CHAN_HDL_INVALID); 1344 } 1345 1346 if (statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) { 1347 while (statep->cep_in_rts == IBCM_BLOCK) 1348 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1349 } 1350 1351 /* Do TRANSIENT_DREQ check after TRANSIENT_ESTABLISHED check */ 1352 while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT) 1353 cv_wait(&statep->block_mad_cv, &statep->state_mutex); 1354 1355 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: chan 0x%p " 1356 "connection state is %x", channel, statep->state); 1357 1358 /* If state is in pre-established states, abort the connection est */ 1359 if (statep->state != IBCM_STATE_ESTABLISHED) { 1360 statep->cm_retries++; /* ensure connection trace is dumped */ 1361 1362 /* No DREP private data possible */ 1363 if (statep->close_ret_priv_data_len != NULL) 1364 *statep->close_ret_priv_data_len = 0; 1365 1366 /* 1367 * If waiting for a response mad, then cancel the timer, 1368 * and delete the connection 1369 */ 1370 if (statep->state == IBCM_STATE_REQ_SENT || 1371 statep->state == IBCM_STATE_REP_SENT || 1372 statep->state == IBCM_STATE_REP_WAIT || 1373 statep->state == IBCM_STATE_MRA_REP_RCVD) { 1374 timeout_id_t timer_val = statep->timerid; 1375 ibcm_conn_state_t old_state; 1376 1377 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: " 1378 "chan 0x%p connection aborted in state %x", channel, 1379 statep->state); 1380 1381 old_state = statep->state; 1382 statep->state = IBCM_STATE_DELETE; 1383 1384 if (mode == IBT_NONBLOCKING) { 1385 if (taskq_dispatch(ibcm_taskq, 1386 ibcm_process_abort_via_taskq, statep, 1387 TQ_NOSLEEP) == 0) { 1388 1389 IBCM_REF_CNT_DECR(statep); 1390 statep->state = old_state; 1391 mutex_exit(&statep->state_mutex); 1392 return (IBT_INSUFF_KERNEL_RESOURCE); 1393 } /* if taskq_dispatch succeeds */ 1394 /* Cancel the timer */ 1395 statep->timerid = 0; 1396 mutex_exit(&statep->state_mutex); 1397 } else { 1398 /* Cancel the timer */ 1399 statep->timerid = 0; 1400 mutex_exit(&statep->state_mutex); 1401 (void) taskq_dispatch(ibcm_taskq, 1402 ibcm_process_abort_via_taskq, statep, 1403 TQ_SLEEP); 1404 } 1405 1406 /* cancel the currently running timer */ 1407 if (timer_val != 0) 1408 (void) untimeout(timer_val); 1409 1410 /* wait until cm handler returns for BLOCKING cases */ 1411 mutex_enter(&statep->state_mutex); 1412 if ((mode == IBT_BLOCKING) || 1413 (mode == IBT_NOCALLBACKS)) { 1414 while (statep->close_done != B_TRUE) 1415 cv_wait(&statep->block_client_cv, 1416 &statep->state_mutex); 1417 } 1418 1419 if (statep->close_ret_status) 1420 *statep->close_ret_status = IBT_CM_CLOSED_ABORT; 1421 mutex_exit(&statep->state_mutex); 1422 1423 /* 1424 * It would ideal to post a REJ MAD, but that would 1425 * be non-conformance to spec. Hence, delete the state 1426 * data. Assuming that happens quickly, any retransmits 1427 * from the remote are replied by CM with reject 1428 * reason " no valid com id". That would stop remote 1429 * sending any more MADs. 1430 */ 1431 ibcm_delete_state_data(statep); 1432 return (IBT_SUCCESS); 1433 1434 /* if CM busy in cm handler, wait until cm handler returns */ 1435 } else if (statep->state == IBCM_STATE_REQ_RCVD || 1436 statep->state == IBCM_STATE_REP_RCVD || 1437 statep->state == IBCM_STATE_MRA_SENT || 1438 statep->state == IBCM_STATE_MRA_REP_SENT) { 1439 1440 /* take control of statep */ 1441 statep->abort_flag |= IBCM_ABORT_CLIENT; 1442 1443 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: " 1444 "chan 0x%p connection aborted in state = %x", 1445 channel, statep->state); 1446 1447 /* 1448 * wait until state machine modifies qp state to error, 1449 * including disassociating statep and QP 1450 */ 1451 if ((mode == IBT_BLOCKING) || (mode == IBT_NOCALLBACKS)) 1452 while (statep->close_done != B_TRUE) 1453 cv_wait(&statep->block_client_cv, 1454 &statep->state_mutex); 1455 1456 /* a sanity setting */ 1457 if (mode == IBT_NOCALLBACKS) 1458 statep->cm_handler = NULL; 1459 IBCM_REF_CNT_DECR(statep); 1460 1461 /* 1462 * In rare situations, connection attempt could be 1463 * terminated for some other reason, before abort is 1464 * processed, but CM still returns ret_status as abort 1465 */ 1466 if (statep->close_ret_status) 1467 *statep->close_ret_status = IBT_CM_CLOSED_ABORT; 1468 mutex_exit(&statep->state_mutex); 1469 1470 /* 1471 * REJ MAD is posted by the CM state machine for this 1472 * case, hence state structure is deleted in the 1473 * state machine processing. 1474 */ 1475 return (IBT_SUCCESS); 1476 1477 } else if ((statep->state == IBCM_STATE_TIMEWAIT) || 1478 (statep->state == IBCM_STATE_DELETE)) { 1479 1480 /* State already in timewait, so no return priv data */ 1481 IBCM_REF_CNT_DECR(statep); 1482 1483 /* The teardown has already been done */ 1484 if (statep->close_ret_status) 1485 *statep->close_ret_status = 1486 IBT_CM_CLOSED_ALREADY; 1487 mutex_exit(&statep->state_mutex); 1488 1489 return (IBT_SUCCESS); 1490 1491 } else if ((statep->state == IBCM_STATE_DREQ_RCVD) || 1492 (statep->state == IBCM_STATE_DREQ_SENT) || 1493 (statep->state == IBCM_STATE_DREP_RCVD) || 1494 ((statep->state == IBCM_STATE_TIMED_OUT) && 1495 (statep->timedout_state == IBCM_STATE_DREQ_SENT))) { 1496 1497 /* 1498 * Either the remote or local client has already 1499 * initiated the teardown. IBCM_STATE_DREP_RCVD is 1500 * possible, if CM initiated teardown without client's 1501 * knowledge, for stale handling, etc., 1502 */ 1503 if (mode == IBT_NOCALLBACKS) { 1504 if (statep->close_nocb_state == IBCM_UNBLOCK) { 1505 statep->close_nocb_state = IBCM_FAIL; 1506 /* enable free qp after return */ 1507 ibtl_cm_chan_is_closing( 1508 statep->channel); 1509 } else while (statep->close_nocb_state == 1510 IBCM_BLOCK) 1511 cv_wait(&statep->block_client_cv, 1512 &statep->state_mutex); 1513 statep->cm_handler = NULL; /* sanity setting */ 1514 if (statep->close_ret_status) 1515 *statep->close_ret_status = 1516 IBT_CM_CLOSED_ALREADY; 1517 } else if (mode == IBT_BLOCKING) { 1518 /* wait until state is moved to timewait */ 1519 while (statep->close_done != B_TRUE) 1520 cv_wait(&statep->block_client_cv, 1521 &statep->state_mutex); 1522 } 1523 1524 IBCM_REF_CNT_DECR(statep); 1525 mutex_exit(&statep->state_mutex); 1526 1527 /* ret_status is set in state machine code */ 1528 return (IBT_SUCCESS); 1529 1530 } else if (statep->state == IBCM_STATE_TIMED_OUT) { 1531 1532 if ((mode == IBT_BLOCKING) || 1533 (mode == IBT_NOCALLBACKS)) { 1534 1535 /* 1536 * wait until cm handler invocation and 1537 * disassociation between statep and channel 1538 * is complete 1539 */ 1540 while (statep->close_done != B_TRUE) 1541 cv_wait(&statep->block_client_cv, 1542 &statep->state_mutex); 1543 } 1544 1545 if (statep->close_ret_status) 1546 *statep->close_ret_status = IBT_CM_CLOSED_ABORT; 1547 IBCM_REF_CNT_DECR(statep); 1548 mutex_exit(&statep->state_mutex); 1549 1550 return (IBT_SUCCESS); 1551 } else { 1552 IBCM_REF_CNT_DECR(statep); 1553 mutex_exit(&statep->state_mutex); 1554 1555 return (IBT_CM_FAILURE); 1556 } 1557 } 1558 1559 ASSERT(statep->close_nocb_state != IBCM_BLOCK); 1560 1561 if (mode == IBT_NOCALLBACKS) { 1562 statep->close_nocb_state = IBCM_FAIL; 1563 statep->cm_handler = NULL; 1564 ibtl_cm_chan_is_closing(statep->channel); 1565 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: " 1566 "NOCALLBACKS on in statep = %p", statep); 1567 } 1568 1569 if (statep->state != IBCM_STATE_ESTABLISHED) { 1570 goto lost_race; 1571 } 1572 1573 /* 1574 * Cancel/wait for any pending ibt_set_alt_path, and 1575 * release state mutex 1576 */ 1577 ibcm_sync_lapr_idle(statep); 1578 1579 ibcm_close_enter(); 1580 1581 mutex_enter(&statep->state_mutex); 1582 if (statep->state != IBCM_STATE_ESTABLISHED) { 1583 ibcm_close_exit(); 1584 goto lost_race; 1585 } 1586 1587 statep->state = IBCM_STATE_TRANSIENT_DREQ_SENT; 1588 statep->timerid = 0; 1589 statep->close_done = B_FALSE; 1590 statep->close_flow = 1; 1591 mutex_exit(&statep->state_mutex); 1592 1593 ibcm_post_dreq_mad(statep); 1594 1595 mutex_enter(&statep->state_mutex); 1596 1597 lost_race: 1598 if (mode == IBT_BLOCKING) { 1599 1600 /* wait for DREP */ 1601 while (statep->close_done != B_TRUE) 1602 cv_wait(&statep->block_client_cv, 1603 &statep->state_mutex); 1604 1605 IBTF_DPRINTF_L4(cmlog, "ibcm_close_rc_channel: chan 0x%p " 1606 "done blocking", channel); 1607 } 1608 1609 IBCM_REF_CNT_DECR(statep); 1610 mutex_exit(&statep->state_mutex); 1611 1612 /* If this message isn't seen then ibt_close_rc_channel failed */ 1613 IBTF_DPRINTF_L5(cmlog, "ibcm_close_rc_channel: chan 0x%p done", 1614 channel); 1615 1616 return (IBT_SUCCESS); 1617 } 1618 1619 ibt_status_t 1620 ibt_recycle_rc(ibt_channel_hdl_t rc_chan, ibt_cep_flags_t control, 1621 uint8_t hca_port_num, ibt_recycle_handler_t func, void *arg) 1622 { 1623 ibcm_state_data_t *statep; 1624 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg; 1625 ibt_qp_query_attr_t qp_attr; 1626 ibt_status_t retval; 1627 1628 IBTF_DPRINTF_L3(cmlog, "ibt_recycle_rc (%p, 0x%X, %d, %p, %p)", rc_chan, 1629 control, hca_port_num, func, arg); 1630 1631 if (IBCM_INVALID_CHANNEL(rc_chan)) { 1632 IBTF_DPRINTF_L2(cmlog, "ibt_recycle_rc: invalid channel"); 1633 return (IBT_CHAN_HDL_INVALID); 1634 } 1635 1636 /* check qp state */ 1637 retval = ibt_query_qp(rc_chan, &qp_attr); 1638 1639 if (retval != IBT_SUCCESS) 1640 return (retval); 1641 1642 if (qp_attr.qp_info.qp_trans != IBT_RC_SRV) 1643 return (IBT_CHAN_SRV_TYPE_INVALID); 1644 1645 if (qp_attr.qp_info.qp_state != IBT_STATE_ERROR) 1646 return (IBT_CHAN_STATE_INVALID); 1647 1648 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1649 1650 ibcm_tq_recycle_arg = kmem_alloc(sizeof (ibcm_taskq_recycle_arg_t), 1651 KM_SLEEP); 1652 1653 ibcm_tq_recycle_arg->rc_chan = rc_chan; 1654 ibcm_tq_recycle_arg->control = control; 1655 ibcm_tq_recycle_arg->hca_port_num = hca_port_num; 1656 ibcm_tq_recycle_arg->func = func; 1657 ibcm_tq_recycle_arg->arg = arg; 1658 1659 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ibcm_tq_recycle_arg)) 1660 1661 IBCM_GET_CHAN_PRIVATE(rc_chan, statep); 1662 1663 /* 1664 * If non-blocking ie., func specified and channel has not yet completed 1665 * the timewait, then schedule the work for later 1666 */ 1667 if ((func != NULL) && (statep != NULL)) { 1668 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1669 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1670 statep->recycle_arg = ibcm_tq_recycle_arg; 1671 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(statep->recycle_arg)) 1672 return (IBT_SUCCESS); 1673 } 1674 1675 /* 1676 * if blocking ie., func specified, and channel has not yet completed 1677 * the timewait, then block until the channel completes the timewait 1678 */ 1679 if (statep != NULL) 1680 IBCM_RELEASE_CHAN_PRIVATE(rc_chan); 1681 IBCM_WAIT_CHAN_PRIVATE(rc_chan); 1682 1683 if (func) { /* NON BLOCKING case. Taskq for QP state change */ 1684 (void) taskq_dispatch(ibcm_taskq, ibcm_process_rc_recycle, 1685 ibcm_tq_recycle_arg, TQ_SLEEP); 1686 return (IBT_SUCCESS); 1687 } else /* BLOCKING case */ 1688 return (ibcm_process_rc_recycle_ret(ibcm_tq_recycle_arg)); 1689 } 1690 1691 void 1692 ibcm_process_rc_recycle(void *recycle_arg) 1693 { 1694 (void) ibcm_process_rc_recycle_ret(recycle_arg); 1695 } 1696 1697 static ibt_status_t 1698 ibcm_process_rc_recycle_ret(void *recycle_arg) 1699 { 1700 ibt_qp_info_t qp_info; 1701 ibt_status_t ibt_status = IBT_SUCCESS; 1702 ibt_cep_modify_flags_t cep_flags; 1703 ibt_qp_query_attr_t qp_attr; 1704 ibcm_taskq_recycle_arg_t *ibcm_tq_recycle_arg = 1705 (ibcm_taskq_recycle_arg_t *)recycle_arg; 1706 1707 /* QP must have been in error state */ 1708 ibt_status = ibt_query_qp(ibcm_tq_recycle_arg->rc_chan, &qp_attr); 1709 if (ibt_status != IBT_SUCCESS) 1710 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1711 "chanp %p ibt_query_qp() = %d", 1712 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1713 else { 1714 /* perform the QP state change from ERROR to RESET */ 1715 bzero(&qp_info, sizeof (qp_info)); 1716 1717 qp_info.qp_trans = IBT_RC_SRV; 1718 qp_info.qp_state = IBT_STATE_RESET; 1719 1720 /* Call modify_qp to move to RESET state */ 1721 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1722 IBT_CEP_SET_STATE, &qp_info, NULL); 1723 1724 if (ibt_status != IBT_SUCCESS) 1725 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1726 "chanp %p ibt_modify_qp() = %d for ERROR to RESET", 1727 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1728 } 1729 1730 if (ibt_status == IBT_SUCCESS) { 1731 1732 qp_info.qp_state = IBT_STATE_INIT; 1733 1734 /* set flags for all mandatory args from RESET to INIT */ 1735 cep_flags = IBT_CEP_SET_STATE | IBT_CEP_SET_PORT; 1736 cep_flags |= IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W; 1737 cep_flags |= IBT_CEP_SET_ATOMIC; 1738 1739 qp_info.qp_transport.rc.rc_path.cep_hca_port_num = 1740 ibcm_tq_recycle_arg->hca_port_num; 1741 qp_info.qp_flags |= 1742 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_RD; 1743 qp_info.qp_flags |= 1744 ibcm_tq_recycle_arg->control & IBT_CEP_RDMA_WR; 1745 qp_info.qp_flags |= 1746 ibcm_tq_recycle_arg->control & IBT_CEP_ATOMIC; 1747 1748 /* Always use the existing pkey */ 1749 qp_info.qp_transport.rc.rc_path.cep_pkey_ix = 1750 qp_attr. qp_info.qp_transport.rc.rc_path.cep_pkey_ix; 1751 1752 /* Call modify_qp to move to INIT state */ 1753 ibt_status = ibt_modify_qp(ibcm_tq_recycle_arg->rc_chan, 1754 cep_flags, &qp_info, NULL); 1755 1756 if (ibt_status != IBT_SUCCESS) 1757 IBTF_DPRINTF_L2(cmlog, "ibcm_process_rc_recycle_ret: " 1758 "chanp %p ibt_modify_qp() = %d for RESET to INIT", 1759 ibcm_tq_recycle_arg->rc_chan, ibt_status); 1760 } 1761 1762 /* Change the QP CM state to indicate QP being re-used */ 1763 if (ibt_status == IBT_SUCCESS) 1764 ibtl_cm_chan_is_reused(ibcm_tq_recycle_arg->rc_chan); 1765 1766 /* Call func, if defined */ 1767 if (ibcm_tq_recycle_arg->func) 1768 (*(ibcm_tq_recycle_arg->func))(ibt_status, 1769 ibcm_tq_recycle_arg->arg); 1770 1771 kmem_free(ibcm_tq_recycle_arg, sizeof (ibcm_taskq_recycle_arg_t)); 1772 1773 return (ibt_status); 1774 } 1775 1776 static void 1777 ibcm_process_abort_via_taskq(void *args) 1778 { 1779 ibcm_state_data_t *statep = (ibcm_state_data_t *)args; 1780 1781 ibcm_process_abort(statep); 1782 mutex_enter(&statep->state_mutex); 1783 IBCM_REF_CNT_DECR(statep); 1784 mutex_exit(&statep->state_mutex); 1785 } 1786 1787 /* 1788 * Local UD CM Handler's private data, used during ibt_request_ud_dest() in 1789 * Non-Blocking mode operations. 1790 */ 1791 typedef struct ibcm_local_handler_s { 1792 ibt_cm_ud_handler_t actual_cm_handler; 1793 void *actual_cm_private; 1794 ibt_ud_dest_t *dest_hdl; 1795 } ibcm_local_handler_t; 1796 1797 _NOTE(READ_ONLY_DATA(ibcm_local_handler_s)) 1798 1799 /* 1800 * Local UD CM Handler, used when ibt_alloc_ud_dest() is issued in 1801 * NON-Blocking mode. 1802 * 1803 * Out here, we update the UD Destination handle with 1804 * the obtained DQPN and QKey (from SIDR REP) and invokes actual client 1805 * handler that was specified by the client. 1806 */ 1807 static ibt_cm_status_t 1808 ibcm_local_cm_handler(void *priv, ibt_cm_ud_event_t *event, 1809 ibt_cm_ud_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 1810 { 1811 ibcm_local_handler_t *handler_priv = (ibcm_local_handler_t *)priv; 1812 1813 IBTF_DPRINTF_L4(cmlog, "ibcm_local_cm_handler: event %d", 1814 event->cm_type); 1815 1816 ASSERT(handler_priv != NULL); 1817 1818 switch (event->cm_type) { 1819 case IBT_CM_UD_EVENT_SIDR_REP: 1820 /* Update QPN & QKey from event into destination handle. */ 1821 if (handler_priv->dest_hdl != NULL) { 1822 handler_priv->dest_hdl->ud_dst_qpn = 1823 event->cm_event.sidr_rep.srep_remote_qpn; 1824 handler_priv->dest_hdl->ud_qkey = 1825 event->cm_event.sidr_rep.srep_remote_qkey; 1826 } 1827 1828 /* Invoke the client handler - inform only, so ignore retval */ 1829 (void) handler_priv->actual_cm_handler( 1830 handler_priv->actual_cm_private, event, ret_args, priv_data, 1831 len); 1832 1833 /* Free memory allocated for local handler's private data. */ 1834 if (handler_priv != NULL) 1835 kmem_free(handler_priv, sizeof (*handler_priv)); 1836 1837 break; 1838 default: 1839 IBTF_DPRINTF_L2(cmlog, "ibcm_local_cm_handler: ERROR"); 1840 break; 1841 } 1842 1843 return (IBT_CM_ACCEPT); 1844 } 1845 1846 1847 /* Validate the input UD destination attributes. */ 1848 static ibt_status_t 1849 ibcm_validate_dqpn_data(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1850 ibt_ud_returns_t *ret_args) 1851 { 1852 /* cm handler must always be specified */ 1853 if (mode == IBT_NONBLOCKING && attr->ud_cm_handler == NULL) { 1854 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1855 "CM handler is not specified "); 1856 return (IBT_INVALID_PARAM); 1857 } 1858 1859 if (mode == IBT_NONBLOCKING) { 1860 if (ret_args != NULL) { 1861 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1862 "ret_args should be NULL when called in " 1863 "non-blocking mode"); 1864 return (IBT_INVALID_PARAM); 1865 } 1866 } else if (mode == IBT_BLOCKING) { 1867 if (ret_args == NULL) { 1868 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1869 "ret_args should be Non-NULL when called in " 1870 "blocking mode"); 1871 return (IBT_INVALID_PARAM); 1872 } 1873 } else { 1874 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1875 "invalid mode %x specified ", mode); 1876 return (IBT_INVALID_PARAM); 1877 } 1878 1879 if (attr->ud_sid == 0) { 1880 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1881 "ServiceID must be specified. "); 1882 return (IBT_INVALID_PARAM); 1883 } 1884 1885 if (attr->ud_addr == NULL) { 1886 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: " 1887 "Address Info NULL"); 1888 return (IBT_INVALID_PARAM); 1889 } 1890 1891 /* Validate SGID */ 1892 if ((attr->ud_addr->av_sgid.gid_prefix == 0) || 1893 (attr->ud_addr->av_sgid.gid_guid == 0)) { 1894 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid SGID"); 1895 return (IBT_INVALID_PARAM); 1896 } 1897 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: SGID<%llX:%llX>", 1898 attr->ud_addr->av_sgid.gid_prefix, 1899 attr->ud_addr->av_sgid.gid_guid); 1900 1901 /* Validate DGID */ 1902 if ((attr->ud_addr->av_dgid.gid_prefix == 0) || 1903 (attr->ud_addr->av_dgid.gid_guid == 0)) { 1904 IBTF_DPRINTF_L2(cmlog, "ibcm_validate_dqpn_data: Invalid DGID"); 1905 return (IBT_INVALID_PARAM); 1906 } 1907 IBTF_DPRINTF_L3(cmlog, "ibcm_validate_dqpn_data: DGID<%llX:%llX>", 1908 attr->ud_addr->av_dgid.gid_prefix, 1909 attr->ud_addr->av_dgid.gid_guid); 1910 1911 return (IBT_SUCCESS); 1912 } 1913 1914 1915 /* Perform SIDR to retrieve DQPN and QKey. */ 1916 static ibt_status_t 1917 ibcm_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 1918 ibt_ud_returns_t *ret_args) 1919 { 1920 ibt_status_t retval; 1921 ib_pkey_t ud_pkey; 1922 ibmf_handle_t ibmf_hdl; 1923 ibmf_msg_t *ibmf_msg; 1924 ibcm_hca_info_t *hcap; 1925 ibcm_sidr_req_msg_t *sidr_req_msgp; 1926 ibcm_ud_state_data_t *ud_statep; 1927 ibtl_cm_hca_port_t port; 1928 ibcm_sidr_srch_t sidr_entry; 1929 ibcm_qp_list_t *cm_qp_entry; 1930 1931 /* Retrieve HCA GUID value from the available SGID info. */ 1932 retval = ibtl_cm_get_hca_port(attr->ud_addr->av_sgid, 0, &port); 1933 if ((retval != IBT_SUCCESS) || (port.hp_port == 0)) { 1934 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1935 "ibtl_cm_get_hca_port failed: %d", retval); 1936 return (retval); 1937 } 1938 1939 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: " 1940 "HCA GUID:%llX, port_num:%d", port.hp_hca_guid, port.hp_port); 1941 1942 /* Lookup the HCA info for this GUID */ 1943 if ((hcap = ibcm_find_hca_entry(port.hp_hca_guid)) == NULL) { 1944 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: hcap is NULL"); 1945 return (IBT_HCA_INVALID); 1946 } 1947 1948 /* Return failure if the HCA device or Port is not operational */ 1949 1950 if ((retval = ibt_get_port_state_byguid(port.hp_hca_guid, port.hp_port, 1951 NULL, NULL)) != IBT_SUCCESS) { 1952 /* Device Port is not in good state, don't use it. */ 1953 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: Invalid " 1954 "port specified or port not active"); 1955 ibcm_dec_hca_acc_cnt(hcap); 1956 return (retval); 1957 } 1958 1959 retval = ibt_index2pkey_byguid(port.hp_hca_guid, port.hp_port, 1960 attr->ud_pkey_ix, &ud_pkey); 1961 if (retval != IBT_SUCCESS) { 1962 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1963 "Failed to convert index2pkey: %d", retval); 1964 ibcm_dec_hca_acc_cnt(hcap); 1965 return (retval); 1966 } 1967 1968 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(sidr_entry)) 1969 1970 /* Allocate a new request id */ 1971 if (ibcm_alloc_reqid(hcap, &sidr_entry.srch_req_id) == IBCM_FAILURE) { 1972 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1973 "no req id available"); 1974 ibcm_dec_hca_acc_cnt(hcap); 1975 return (IBT_INSUFF_KERNEL_RESOURCE); 1976 } 1977 1978 if ((hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl == NULL) && 1979 ((retval = ibcm_hca_reinit_port(hcap, port.hp_port - 1)) 1980 != IBT_SUCCESS)) { 1981 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: " 1982 "ibmf reg or callback setup failed during re-initialize"); 1983 return (retval); 1984 } 1985 1986 ibmf_hdl = hcap->hca_port_info[port.hp_port - 1].port_ibmf_hdl; 1987 1988 /* find the ibmf QP to post the SIDR REQ */ 1989 if ((cm_qp_entry = ibcm_find_qp(hcap, port.hp_port, ud_pkey)) == 1990 NULL) { 1991 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF QP allocation" 1992 " failed"); 1993 ibcm_dec_hca_acc_cnt(hcap); 1994 return (IBT_INSUFF_RESOURCE); 1995 } 1996 1997 if ((retval = ibcm_alloc_out_msg(ibmf_hdl, &ibmf_msg, MAD_METHOD_SEND)) 1998 != IBT_SUCCESS) { 1999 IBTF_DPRINTF_L2(cmlog, "ibcm_ud_get_dqpn: IBMF MSG allocation" 2000 " failed"); 2001 ibcm_release_qp(cm_qp_entry); 2002 ibcm_dec_hca_acc_cnt(hcap); 2003 return (retval); 2004 } 2005 2006 sidr_entry.srch_lid = port.hp_base_lid; 2007 sidr_entry.srch_gid = attr->ud_addr->av_sgid; 2008 sidr_entry.srch_grh_exists = attr->ud_addr->av_send_grh; 2009 sidr_entry.srch_mode = IBCM_ACTIVE_MODE; 2010 2011 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(sidr_entry)) 2012 2013 /* do various allocations needed here */ 2014 rw_enter(&hcap->hca_sidr_list_lock, RW_WRITER); 2015 2016 (void) ibcm_find_sidr_entry(&sidr_entry, hcap, &ud_statep, 2017 IBCM_FLAG_ADD); 2018 rw_exit(&hcap->hca_sidr_list_lock); 2019 2020 /* Increment hca's resource count */ 2021 ibcm_inc_hca_res_cnt(hcap); 2022 2023 /* After a resource created on hca, no need to hold the acc cnt */ 2024 ibcm_dec_hca_acc_cnt(hcap); 2025 2026 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ud_statep)) 2027 2028 /* Initialize some ud_statep fields */ 2029 ud_statep->ud_stored_msg = ibmf_msg; 2030 ud_statep->ud_svc_id = attr->ud_sid; 2031 ud_statep->ud_pkt_life_time = 2032 ibt_ib2usec(attr->ud_pkt_lt); 2033 ud_statep->ud_stored_reply_addr.cm_qp_entry = cm_qp_entry; 2034 2035 /* set remaining retry cnt */ 2036 ud_statep->ud_remaining_retry_cnt = ud_statep->ud_max_cm_retries; 2037 2038 /* 2039 * Get UD handler and corresponding args which is pass it back 2040 * as first argument for the handler. 2041 */ 2042 ud_statep->ud_state_cm_private = attr->ud_cm_private; 2043 2044 if (mode == IBT_BLOCKING) 2045 ud_statep->ud_return_data = ret_args; 2046 else 2047 ud_statep->ud_cm_handler = attr->ud_cm_handler; 2048 2049 /* Initialize the fields of ud_statep->ud_stored_reply_addr */ 2050 ud_statep->ud_stored_reply_addr.grh_exists = attr->ud_addr->av_send_grh; 2051 ud_statep->ud_stored_reply_addr.ibmf_hdl = ibmf_hdl; 2052 ud_statep->ud_stored_reply_addr.grh_hdr.ig_hop_limit = 2053 attr->ud_addr->av_hop; 2054 ud_statep->ud_stored_reply_addr.grh_hdr.ig_sender_gid = 2055 attr->ud_addr->av_sgid; 2056 ud_statep->ud_stored_reply_addr.grh_hdr.ig_recver_gid = 2057 attr->ud_addr->av_dgid; 2058 ud_statep->ud_stored_reply_addr.grh_hdr.ig_tclass = 2059 attr->ud_addr->av_tclass; 2060 ud_statep->ud_stored_reply_addr.grh_hdr.ig_flow_label = 2061 attr->ud_addr->av_flow & IB_GRH_FLOW_LABEL_MASK; 2062 2063 /* needs to be derived based on the base LID and path bits */ 2064 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_local_lid = 2065 port.hp_base_lid; 2066 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_lid = 2067 attr->ud_addr->av_dlid; 2068 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_p_key = ud_pkey; 2069 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_q_key = IB_GSI_QKEY; 2070 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_service_level = 2071 attr->ud_addr->av_srvl; 2072 2073 /* 2074 * This may be enchanced later, to use a remote qno based on past 2075 * redirect rej mad responses. This would be the place to specify 2076 * appropriate remote qno 2077 */ 2078 ud_statep->ud_stored_reply_addr.rcvd_addr.ia_remote_qno = 1; 2079 2080 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2081 2082 /* Initialize the SIDR REQ message fields */ 2083 sidr_req_msgp = 2084 (ibcm_sidr_req_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg); 2085 2086 sidr_req_msgp->sidr_req_request_id = h2b32(ud_statep->ud_req_id); 2087 sidr_req_msgp->sidr_req_service_id = h2b64(attr->ud_sid); 2088 sidr_req_msgp->sidr_req_pkey = h2b16(ud_pkey); 2089 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->AttributeID = 2090 h2b16(IBCM_INCOMING_SIDR_REQ + IBCM_ATTR_BASE_ID); 2091 2092 if ((attr->ud_priv_data != NULL) && (attr->ud_priv_data_len > 0)) { 2093 bcopy(attr->ud_priv_data, sidr_req_msgp->sidr_req_private_data, 2094 min(attr->ud_priv_data_len, IBT_SIDR_REQ_PRIV_DATA_SZ)); 2095 } 2096 2097 /* Send out the SIDR REQ message */ 2098 ud_statep->ud_state = IBCM_STATE_SIDR_REQ_SENT; 2099 ud_statep->ud_timer_stored_state = IBCM_STATE_SIDR_REQ_SENT; 2100 IBCM_UD_REF_CNT_INCR(ud_statep); /* for non-blocking SIDR REQ post */ 2101 ud_statep->ud_timer_value = ibt_ib2usec(ibcm_max_sidr_rep_proctime) + 2102 (ud_statep->ud_pkt_life_time * 2); 2103 2104 IBCM_OUT_HDRP(ud_statep->ud_stored_msg)->TransactionID = 2105 h2b64(ibcm_generate_tranid(IBCM_INCOMING_SIDR_REQ, 2106 ud_statep->ud_req_id, 0)); 2107 2108 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: timer_value in HZ = %x", 2109 ud_statep->ud_timer_value); 2110 2111 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*ud_statep)) 2112 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sidr_req_msgp)) 2113 2114 ibcm_post_ud_mad(ud_statep, ud_statep->ud_stored_msg, 2115 ibcm_post_sidr_req_complete, ud_statep); 2116 2117 mutex_enter(&ud_statep->ud_state_mutex); 2118 2119 /* Wait for SIDR_REP */ 2120 if (mode == IBT_BLOCKING) { 2121 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: blocking"); 2122 2123 while (ud_statep->ud_blocking_done != B_TRUE) { 2124 cv_wait(&ud_statep->ud_block_client_cv, 2125 &ud_statep->ud_state_mutex); 2126 } 2127 2128 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: finished blocking"); 2129 2130 if (ret_args->ud_status == IBT_CM_SREP_QPN_VALID) { 2131 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: DQPN = %x, " 2132 "status = %x, QKey = %x", ret_args->ud_dqpn, 2133 ret_args->ud_status, ret_args->ud_qkey); 2134 2135 } else { 2136 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: Status<%x>", 2137 ret_args->ud_status); 2138 retval = IBT_CM_FAILURE; 2139 } 2140 } 2141 2142 IBCM_UD_REF_CNT_DECR(ud_statep); 2143 mutex_exit(&ud_statep->ud_state_mutex); 2144 2145 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_get_dqpn: done"); 2146 2147 return (retval); 2148 } 2149 2150 2151 /* 2152 * Function: 2153 * ibt_request_ud_dest 2154 * Input: 2155 * ud_dest A previously allocated UD destination handle. 2156 * mode This function can execute in blocking or non blocking 2157 * modes. 2158 * attr UD destination attributes to be modified. 2159 * Output: 2160 * ud_ret_args If the function is called in blocking mode, ud_ret_args 2161 * should be a pointer to an ibt_ud_returns_t struct. 2162 * Returns: 2163 * IBT_SUCCESS 2164 * Description: 2165 * Modify a previously allocated UD destination handle based on the 2166 * results of doing the SIDR protocol. 2167 */ 2168 ibt_status_t 2169 ibt_request_ud_dest(ibt_ud_dest_hdl_t ud_dest, ibt_execution_mode_t mode, 2170 ibt_ud_dest_attr_t *attr, ibt_ud_returns_t *ud_ret_args) 2171 { 2172 ibt_status_t retval; 2173 ibt_ud_dest_t *ud_destp; 2174 ibcm_local_handler_t *local_handler_priv = NULL; 2175 2176 IBTF_DPRINTF_L3(cmlog, "ibt_request_ud_dest(%p, %x, %p, %p)", 2177 ud_dest, mode, attr, ud_ret_args); 2178 2179 retval = ibcm_validate_dqpn_data(attr, mode, ud_ret_args); 2180 if (retval != IBT_SUCCESS) { 2181 return (retval); 2182 } 2183 2184 ud_destp = ud_dest; 2185 2186 /* Allocate an Address handle. */ 2187 retval = ibt_modify_ah(ud_destp->ud_dest_hca, ud_destp->ud_ah, 2188 attr->ud_addr); 2189 if (retval != IBT_SUCCESS) { 2190 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2191 "Address Handle Modification failed: %d", retval); 2192 return (retval); 2193 } 2194 2195 if (mode == IBT_NONBLOCKING) { 2196 /* 2197 * In NON-BLOCKING mode, and we need to update the destination 2198 * handle with the DQPN and QKey that are obtained from 2199 * SIDR REP, hook-up our own handler, so that we can catch 2200 * the event, and we ourselves call the actual client's 2201 * ud_cm_handler, in our handler. 2202 */ 2203 2204 /* Allocate memory for local handler's private data. */ 2205 local_handler_priv = 2206 kmem_alloc(sizeof (*local_handler_priv), KM_SLEEP); 2207 2208 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2209 2210 local_handler_priv->actual_cm_handler = attr->ud_cm_handler; 2211 local_handler_priv->actual_cm_private = attr->ud_cm_private; 2212 local_handler_priv->dest_hdl = ud_destp; 2213 2214 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*local_handler_priv)) 2215 2216 attr->ud_cm_handler = ibcm_local_cm_handler; 2217 attr->ud_cm_private = local_handler_priv; 2218 } 2219 2220 /* In order to get DQPN and Destination QKey, perform SIDR */ 2221 retval = ibcm_ud_get_dqpn(attr, mode, ud_ret_args); 2222 if (retval != IBT_SUCCESS) { 2223 IBTF_DPRINTF_L2(cmlog, "ibt_request_ud_dest: " 2224 "Failed to get DQPN: %d", retval); 2225 2226 /* Free memory allocated for local handler's private data. */ 2227 if (local_handler_priv != NULL) 2228 kmem_free(local_handler_priv, 2229 sizeof (*local_handler_priv)); 2230 return (retval); 2231 } 2232 2233 /* 2234 * Fill in the dqpn and dqkey as obtained from ud_ret_args, 2235 * values will be valid only on BLOCKING mode. 2236 */ 2237 if (mode == IBT_BLOCKING) { 2238 ud_destp->ud_dst_qpn = ud_ret_args->ud_dqpn; 2239 ud_destp->ud_qkey = ud_ret_args->ud_qkey; 2240 } 2241 2242 return (retval); 2243 } 2244 2245 /* 2246 * Function: 2247 * ibt_ud_get_dqpn 2248 * Input: 2249 * attr A pointer to an ibt_ud_dest_attr_t struct that are 2250 * required for SIDR REQ message. Not specified attributes 2251 * should be set to "NULL" or "0". 2252 * ud_sid, ud_addr and ud_pkt_lt must be specified. 2253 * mode This function can execute in blocking or non blocking 2254 * modes. 2255 * Output: 2256 * returns If the function is called in blocking mode, returns 2257 * should be a pointer to an ibt_ud_returns_t struct. 2258 * Return: 2259 * IBT_SUCCESS on success or respective failure on error. 2260 * Description: 2261 * Finds the destination QPN at the specified destination that the 2262 * specified service can be reached on. The IBTF CM initiates the 2263 * service ID resolution protocol (SIDR) to determine a destination QPN. 2264 * 2265 * NOTE: SIDR_REQ is initiated from active side. 2266 */ 2267 ibt_status_t 2268 ibt_ud_get_dqpn(ibt_ud_dest_attr_t *attr, ibt_execution_mode_t mode, 2269 ibt_ud_returns_t *returns) 2270 { 2271 ibt_status_t retval; 2272 2273 IBTF_DPRINTF_L3(cmlog, "ibt_ud_get_dqpn(%p, %x, %p)", 2274 attr, mode, returns); 2275 2276 retval = ibcm_validate_dqpn_data(attr, mode, returns); 2277 if (retval != IBT_SUCCESS) { 2278 return (retval); 2279 } 2280 2281 return (ibcm_ud_get_dqpn(attr, mode, returns)); 2282 } 2283 2284 2285 /* 2286 * ibt_cm_delay: 2287 * A client CM handler function can call this function 2288 * to extend its response time to a CM event. 2289 * INPUTS: 2290 * flags Indicates what CM message processing is being delayed 2291 * by the CM handler, valid values are: 2292 * IBT_CM_DELAY_REQ 2293 * IBT_CM_DELAY_REP 2294 * IBT_CM_DELAY_LAP 2295 * cm_session_id The session ID that was passed to client srv_handler 2296 * by the CM 2297 * service_time The extended service time 2298 * priv_data Vendor specific data to be sent in the CM generated 2299 * MRA message. Should be NULL if not specified. 2300 * len The number of bytes of data specified by priv_data. 2301 * 2302 * RETURN VALUES: 2303 * IBT_SUCCESS on success (or respective failure on error) 2304 */ 2305 ibt_status_t 2306 ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id, 2307 clock_t service_time, void *priv_data, ibt_priv_data_len_t len) 2308 { 2309 uint8_t msg_typ = 0; 2310 ibcm_mra_msg_t *mra_msgp; 2311 ibcm_state_data_t *statep; 2312 ibt_status_t status; 2313 2314 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay(0x%x, %p, 0x%x)", 2315 flags, cm_session_id, service_time); 2316 2317 /* 2318 * Make sure channel is associated with a statep 2319 */ 2320 statep = (ibcm_state_data_t *)cm_session_id; 2321 2322 if (statep == NULL) { 2323 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: statep NULL"); 2324 return (IBT_INVALID_PARAM); 2325 } 2326 2327 IBTF_DPRINTF_L4(cmlog, "ibt_cm_delay: statep %p", statep); 2328 2329 /* Allocate an ibmf msg for mra, if not allocated yet */ 2330 if (statep->mra_msg == NULL) { 2331 if ((status = ibcm_alloc_out_msg( 2332 statep->stored_reply_addr.ibmf_hdl, &statep->mra_msg, 2333 MAD_METHOD_SEND)) != IBT_SUCCESS) { 2334 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: chan 0x%p" 2335 "IBMF MSG allocation failed", statep->channel); 2336 return (status); 2337 } 2338 } 2339 2340 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2341 2342 mra_msgp = (ibcm_mra_msg_t *)IBCM_OUT_MSGP(statep->mra_msg); 2343 mra_msgp->mra_local_comm_id = h2b32(statep->local_comid); 2344 mra_msgp->mra_remote_comm_id = h2b32(statep->remote_comid); 2345 2346 /* fill in rest of MRA's fields - Message MRAed and Service Timeout */ 2347 if (flags == IBT_CM_DELAY_REQ) { 2348 msg_typ = IBT_CM_MRA_TYPE_REQ; 2349 } else if (flags == IBT_CM_DELAY_REP) { 2350 msg_typ = IBT_CM_MRA_TYPE_REP; 2351 } else if (flags == IBT_CM_DELAY_LAP) { 2352 msg_typ = IBT_CM_MRA_TYPE_LAP; 2353 } 2354 2355 mra_msgp->mra_message_type_plus = msg_typ << 6; 2356 mra_msgp->mra_service_timeout_plus = ibt_usec2ib(service_time) << 3; 2357 2358 len = min(len, IBT_MRA_PRIV_DATA_SZ); 2359 if (priv_data && (len > 0)) 2360 bcopy(priv_data, mra_msgp->mra_private_data, len); 2361 2362 IBCM_OUT_HDRP(statep->mra_msg)->AttributeID = 2363 h2b16(IBCM_INCOMING_MRA + IBCM_ATTR_BASE_ID); 2364 2365 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mra_msgp)) 2366 2367 mutex_enter(&statep->state_mutex); 2368 2369 if ((statep->mode == IBCM_ACTIVE_MODE) && 2370 (statep->state == IBCM_STATE_REP_RCVD)) { 2371 statep->state = IBCM_STATE_MRA_REP_SENT; 2372 } else if (statep->mode == IBCM_PASSIVE_MODE) { 2373 if (statep->state == IBCM_STATE_REQ_RCVD) { 2374 statep->state = IBCM_STATE_MRA_SENT; 2375 } else if (statep->ap_state == IBCM_AP_STATE_LAP_RCVD) { 2376 statep->ap_state = IBCM_AP_STATE_MRA_LAP_RCVD; 2377 } else { 2378 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2379 "/ap_state/mode %x, %x, %x", statep->state, 2380 statep->ap_state, statep->mode); 2381 mutex_exit(&statep->state_mutex); 2382 return (IBT_CHAN_STATE_INVALID); 2383 } 2384 } else { 2385 IBTF_DPRINTF_L2(cmlog, "ibt_cm_delay: invalid state " 2386 "/ap_state/mode %x, %x, %x", statep->state, 2387 statep->ap_state, statep->mode); 2388 mutex_exit(&statep->state_mutex); 2389 2390 return (IBT_CHAN_STATE_INVALID); 2391 } 2392 /* service time is usecs, stale_clock is nsecs */ 2393 statep->stale_clock = gethrtime() + 2394 (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 * 2395 statep->max_cm_retries); 2396 2397 statep->send_mad_flags |= IBCM_MRA_POST_BUSY; 2398 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_mra_complete */ 2399 mutex_exit(&statep->state_mutex); 2400 2401 IBCM_OUT_HDRP(statep->mra_msg)->TransactionID = 2402 IBCM_OUT_HDRP(statep->stored_msg)->TransactionID; 2403 2404 /* post the MRA mad in blocking mode, as no timers involved */ 2405 ibcm_post_rc_mad(statep, statep->mra_msg, ibcm_post_mra_complete, 2406 statep); 2407 ibcm_insert_trace(statep, IBCM_TRACE_OUTGOING_MRA); 2408 /* If this message isn't seen then ibt_cm_delay failed */ 2409 IBTF_DPRINTF_L3(cmlog, "ibt_cm_delay: done !!"); 2410 2411 return (IBT_SUCCESS); 2412 } 2413 2414 2415 /* 2416 * ibt_register_service() 2417 * Register a service with the IBCM 2418 * 2419 * INPUTS: 2420 * ibt_hdl The IBT client handle returned to the client 2421 * on an ibt_attach() call. 2422 * 2423 * srv The address of a ibt_srv_desc_t that describes 2424 * the service, containing the following: 2425 * 2426 * sd_ud_handler The Service CM UD event Handler. 2427 * sd_handler The Service CM RC/UC/RD event Handler. 2428 * sd_flags Service flags (peer-to-peer, or not). 2429 * 2430 * sid This tells CM if the service is local (sid is 0) or 2431 * wellknown (sid is the starting service id of the range). 2432 * 2433 * num_sids The number of contiguous service-ids to reserve. 2434 * 2435 * srv_hdl The address of a service identification handle, used 2436 * to deregister a service, and to bind GIDs to. 2437 * 2438 * ret_sid The address to store the Service ID return value. 2439 * If num_sids > 1, ret_sid is the first Service ID 2440 * in the range. 2441 * 2442 * ibt_register_service() returns: 2443 * IBT_SUCCESS - added a service successfully. 2444 * IBT_INVALID_PARAM - invalid input parameter. 2445 * IBT_CM_FAILURE - failed to add the service. 2446 * IBT_CM_SERVICE_EXISTS - service already exists. 2447 * IBT_INSUFF_KERNEL_RESOURCE - ran out of local service ids (should 2448 * never happen). 2449 */ 2450 ibt_status_t 2451 ibt_register_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_desc_t *srv, 2452 ib_svc_id_t sid, int num_sids, ibt_srv_hdl_t *srv_hdl, ib_svc_id_t *ret_sid) 2453 { 2454 ibcm_svc_info_t *svcinfop; 2455 2456 IBTF_DPRINTF_L2(cmlog, "ibt_register_service(%p, %p, %llx, %d)", 2457 ibt_hdl, srv, (longlong_t)sid, num_sids); 2458 2459 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*svcinfop)) 2460 2461 *srv_hdl = NULL; 2462 2463 if (num_sids <= 0) { 2464 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2465 "Invalid number of service-ids specified (%d)", num_sids); 2466 return (IBT_INVALID_PARAM); 2467 } 2468 2469 if (sid == 0) { 2470 if (ret_sid == NULL) 2471 return (IBT_INVALID_PARAM); 2472 sid = ibcm_alloc_local_sids(num_sids); 2473 if (sid == 0) 2474 return (IBT_INSUFF_KERNEL_RESOURCE); 2475 2476 /* Make sure that the ServiceId specified is not of LOCAL AGN type. */ 2477 } else if ((sid & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) { 2478 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2479 "Invalid non-LOCAL SID specified: 0x%llX", 2480 (longlong_t)sid); 2481 return (IBT_INVALID_PARAM); 2482 } 2483 2484 svcinfop = ibcm_create_svc_entry(sid, num_sids); 2485 2486 if (svcinfop == NULL) { 2487 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: " 2488 "Service-ID 0x%llx already registered", (longlong_t)sid); 2489 return (IBT_CM_SERVICE_EXISTS); 2490 } 2491 2492 /* 2493 * 'sid' and 'num_sids' are filled in ibcm_create_svc_entry() 2494 */ 2495 svcinfop->svc_flags = srv->sd_flags; 2496 svcinfop->svc_rc_handler = srv->sd_handler; 2497 svcinfop->svc_ud_handler = srv->sd_ud_handler; 2498 2499 if (ret_sid != NULL) 2500 *ret_sid = sid; 2501 2502 *srv_hdl = svcinfop; 2503 2504 ibtl_cm_change_service_cnt(ibt_hdl, num_sids); 2505 2506 /* If this message isn't seen, then ibt_register_service failed. */ 2507 IBTF_DPRINTF_L2(cmlog, "ibt_register_service: done (%p, %llX)", 2508 svcinfop, sid); 2509 2510 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*svcinfop)) 2511 2512 return (IBT_SUCCESS); 2513 } 2514 2515 2516 static ibt_status_t 2517 ibcm_write_service_record(ibmf_saa_handle_t saa_handle, 2518 sa_service_record_t *srv_recp, ibmf_saa_access_type_t saa_type) 2519 { 2520 int rval; 2521 int retry; 2522 2523 ibcm_sa_access_enter(); 2524 for (retry = 0; retry < ibcm_max_sa_retries; retry++) { 2525 rval = ibmf_saa_update_service_record( 2526 saa_handle, srv_recp, saa_type, 0); 2527 if (rval != IBMF_TRANS_TIMEOUT) { 2528 break; 2529 } 2530 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2531 "ibmf_saa_update_service_record timed out" 2532 " SID = %llX, rval = %d, saa_type = %d", 2533 (longlong_t)srv_recp->ServiceID, rval, saa_type); 2534 delay(ibcm_sa_timeout_delay); 2535 } 2536 ibcm_sa_access_exit(); 2537 2538 if (rval != IBMF_SUCCESS) { 2539 IBTF_DPRINTF_L2(cmlog, "ibcm_write_service_record: " 2540 "ibmf_saa_update_service_record() : Failed - %d", rval); 2541 return (ibcm_ibmf_analyze_error(rval)); 2542 } else 2543 return (IBT_SUCCESS); 2544 } 2545 2546 2547 static void 2548 ibcm_rem_stale_srec(ibmf_saa_handle_t saa_handle, sa_service_record_t *srec) 2549 { 2550 ibt_status_t retval; 2551 uint_t num_found; 2552 size_t length; 2553 sa_service_record_t *srv_resp; 2554 void *results_p; 2555 uint_t i; 2556 uint64_t component_mask; 2557 ibmf_saa_access_args_t access_args; 2558 2559 component_mask = 2560 SA_SR_COMPMASK_PKEY | SA_SR_COMPMASK_NAME | SA_SR_COMPMASK_GID; 2561 2562 /* Call in SA Access retrieve routine to get Service Records. */ 2563 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 2564 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 2565 access_args.sq_component_mask = component_mask; 2566 access_args.sq_template = srec; 2567 access_args.sq_template_length = sizeof (sa_service_record_t); 2568 access_args.sq_callback = NULL; 2569 access_args.sq_callback_arg = NULL; 2570 2571 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 2572 &results_p); 2573 if (retval != IBT_SUCCESS) { 2574 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2575 "SA Access Failure"); 2576 return; 2577 } 2578 2579 num_found = length / sizeof (sa_service_record_t); 2580 2581 if (num_found) 2582 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2583 "Found %d matching Service Records.", num_found); 2584 2585 /* Validate the returned number of records. */ 2586 if ((results_p != NULL) && (num_found > 0)) { 2587 2588 /* Remove all the records. */ 2589 for (i = 0; i < num_found; i++) { 2590 2591 srv_resp = (sa_service_record_t *) 2592 ((uchar_t *)results_p + 2593 i * sizeof (sa_service_record_t)); 2594 2595 /* 2596 * Found some matching records, but check out whether 2597 * this Record is really stale or just happens to match 2598 * the current session records. If yes, don't remove it. 2599 */ 2600 mutex_enter(&ibcm_svc_info_lock); 2601 if (ibcm_find_svc_entry(srv_resp->ServiceID) != NULL) { 2602 /* This record is NOT STALE. */ 2603 mutex_exit(&ibcm_svc_info_lock); 2604 IBTF_DPRINTF_L3(cmlog, "ibcm_rem_stale_srec: " 2605 "This is not Stale, it's an active record"); 2606 continue; 2607 } 2608 mutex_exit(&ibcm_svc_info_lock); 2609 2610 IBTF_DPRINTF_L2(cmlog, "ibcm_rem_stale_srec: " 2611 "Removing Stale Rec: %s, %llX", 2612 srv_resp->ServiceName, srv_resp->ServiceID); 2613 2614 IBCM_DUMP_SERVICE_REC(srv_resp); 2615 2616 /* 2617 * Remove the Service Record Entry from SA. 2618 * 2619 * Get ServiceID info from Response Buf, other 2620 * attributes are already filled-in. 2621 */ 2622 2623 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2624 2625 srec->ServiceID = srv_resp->ServiceID; 2626 2627 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(srec->ServiceID)) 2628 2629 (void) ibcm_write_service_record(saa_handle, srec, 2630 IBMF_SAA_DELETE); 2631 } 2632 2633 /* Deallocate the memory for results_p. */ 2634 kmem_free(results_p, length); 2635 } 2636 } 2637 2638 2639 2640 /* 2641 * ibt_bind_service() 2642 * Register a service with the IBCM 2643 * 2644 * INPUTS: 2645 * srv_hdl The service id handle returned to the client 2646 * on an ibt_service_register() call. 2647 * 2648 * gid The GID to which to bind the service. 2649 * 2650 * srv_bind The address of a ibt_srv_bind_t that describes 2651 * the service record. This should be NULL if there 2652 * is to be no service record. This contains: 2653 * 2654 * sb_lease Lease period 2655 * sb_pkey Partition 2656 * sb_name pointer to ASCII string Service Name, 2657 * NULL terminated. 2658 * sb_key[] Key to secure the service record. 2659 * sb_data Service Data structure (64-byte) 2660 * 2661 * cm_private First argument of Service handler. 2662 * 2663 * sb_hdl_p The address of a service bind handle, used 2664 * to undo the service binding. 2665 * 2666 * ibt_bind_service() returns: 2667 * IBT_SUCCESS - added a service successfully. 2668 * IBT_INVALID_PARAM - invalid input parameter. 2669 * IBT_CM_FAILURE - failed to add the service. 2670 * IBT_CM_SERVICE_EXISTS - service already exists. 2671 */ 2672 ibt_status_t 2673 ibt_bind_service(ibt_srv_hdl_t srv_hdl, ib_gid_t gid, ibt_srv_bind_t *srv_bind, 2674 void *cm_private, ibt_sbind_hdl_t *sb_hdl_p) 2675 { 2676 ibt_status_t status; 2677 ibtl_cm_hca_port_t port; 2678 ibcm_svc_bind_t *sbindp, *sbp; 2679 ibcm_hca_info_t *hcap; 2680 ib_svc_id_t sid, start_sid, end_sid; 2681 ibmf_saa_handle_t saa_handle; 2682 sa_service_record_t srv_rec; 2683 uint16_t pkey_ix; 2684 2685 if (sb_hdl_p != NULL) 2686 *sb_hdl_p = NULL; /* return value for error cases */ 2687 2688 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: srv_hdl %p, gid (%llX:%llX)", 2689 srv_hdl, (longlong_t)gid.gid_prefix, (longlong_t)gid.gid_guid); 2690 2691 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sbindp)) 2692 2693 /* Call ibtl_cm_get_hca_port to get the port number and the HCA GUID. */ 2694 if ((status = ibtl_cm_get_hca_port(gid, 0, &port)) != IBT_SUCCESS) { 2695 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2696 "ibtl_cm_get_hca_port failed: %d", status); 2697 return (status); 2698 } 2699 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: Port:%d HCA GUID:%llX", 2700 port.hp_port, port.hp_hca_guid); 2701 2702 hcap = ibcm_find_hca_entry(port.hp_hca_guid); 2703 if (hcap == NULL) { 2704 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: NO HCA found"); 2705 return (IBT_HCA_BUSY_DETACHING); 2706 } 2707 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: hcap = %p", hcap); 2708 2709 if (srv_bind != NULL) { 2710 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2711 if (saa_handle == NULL) { 2712 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2713 "saa_handle is NULL"); 2714 ibcm_dec_hca_acc_cnt(hcap); 2715 return (IBT_HCA_PORT_NOT_ACTIVE); 2716 } 2717 if (srv_bind->sb_pkey == 0) { 2718 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2719 "P_Key must not be 0"); 2720 ibcm_dec_hca_acc_cnt(hcap); 2721 return (IBT_INVALID_PARAM); 2722 } 2723 if (strlen(srv_bind->sb_name) >= IB_SVC_NAME_LEN) { 2724 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2725 "Service Name is too long"); 2726 ibcm_dec_hca_acc_cnt(hcap); 2727 return (IBT_INVALID_PARAM); 2728 } else 2729 IBTF_DPRINTF_L3(cmlog, "ibt_bind_service: " 2730 "Service Name='%s'", srv_bind->sb_name); 2731 status = ibt_pkey2index_byguid(port.hp_hca_guid, 2732 port.hp_port, srv_bind->sb_pkey, &pkey_ix); 2733 if (status != IBT_SUCCESS) { 2734 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2735 "P_Key 0x%x not found in P_Key_Table", 2736 srv_bind->sb_pkey); 2737 ibcm_dec_hca_acc_cnt(hcap); 2738 return (status); 2739 } 2740 } 2741 2742 /* assume success - allocate before locking */ 2743 sbindp = kmem_zalloc(sizeof (*sbindp), KM_SLEEP); 2744 sbindp->sbind_cm_private = cm_private; 2745 sbindp->sbind_gid = gid; 2746 sbindp->sbind_hcaguid = port.hp_hca_guid; 2747 sbindp->sbind_port = port.hp_port; 2748 2749 mutex_enter(&ibcm_svc_info_lock); 2750 2751 sbp = srv_hdl->svc_bind_list; 2752 while (sbp != NULL) { 2753 if (sbp->sbind_gid.gid_guid == gid.gid_guid && 2754 sbp->sbind_gid.gid_prefix == gid.gid_prefix) { 2755 if (srv_bind == NULL || 2756 srv_bind->sb_pkey == sbp->sbind_pkey) { 2757 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: " 2758 "failed: GID %llX:%llX and PKEY %x is " 2759 "already bound", gid.gid_guid, 2760 gid.gid_prefix, sbp->sbind_pkey); 2761 mutex_exit(&ibcm_svc_info_lock); 2762 ibcm_dec_hca_acc_cnt(hcap); 2763 kmem_free(sbindp, sizeof (*sbindp)); 2764 return (IBT_CM_SERVICE_EXISTS); 2765 } 2766 } 2767 sbp = sbp->sbind_link; 2768 } 2769 /* no entry found */ 2770 2771 sbindp->sbind_link = srv_hdl->svc_bind_list; 2772 srv_hdl->svc_bind_list = sbindp; 2773 2774 mutex_exit(&ibcm_svc_info_lock); 2775 2776 if (srv_bind != NULL) { 2777 bzero(&srv_rec, sizeof (srv_rec)); 2778 2779 srv_rec.ServiceLease = 2780 sbindp->sbind_lease = srv_bind->sb_lease; 2781 srv_rec.ServiceP_Key = 2782 sbindp->sbind_pkey = srv_bind->sb_pkey; 2783 srv_rec.ServiceKey_hi = 2784 sbindp->sbind_key[0] = srv_bind->sb_key[0]; 2785 srv_rec.ServiceKey_lo = 2786 sbindp->sbind_key[1] = srv_bind->sb_key[1]; 2787 (void) strcpy(sbindp->sbind_name, srv_bind->sb_name); 2788 (void) strcpy((char *)srv_rec.ServiceName, srv_bind->sb_name); 2789 srv_rec.ServiceGID = gid; 2790 2791 /* 2792 * Find out whether we have any stale Local Service records 2793 * matching the current attributes. If yes, we shall try to 2794 * remove them from SA using the current request's ServiceKey. 2795 * 2796 * We will perform this operation only for Local Services, as 2797 * it is handled by SA automatically for WellKnown Services. 2798 * 2799 * Ofcourse, clients can specify NOT to do this clean-up by 2800 * setting IBT_SBIND_NO_CLEANUP flag (srv_bind->sb_flag). 2801 */ 2802 if ((srv_hdl->svc_id & IB_SID_AGN_LOCAL) && 2803 (!(srv_bind->sb_flag & IBT_SBIND_NO_CLEANUP))) { 2804 ibcm_rem_stale_srec(saa_handle, &srv_rec); 2805 } 2806 2807 /* Handle endianess for service data. */ 2808 ibcm_swizzle_from_srv(&srv_bind->sb_data, sbindp->sbind_data); 2809 2810 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 2811 2812 /* insert srv record into the SA */ 2813 start_sid = srv_hdl->svc_id; 2814 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 2815 for (sid = start_sid; sid <= end_sid; sid++) { 2816 2817 srv_rec.ServiceID = sid; 2818 2819 IBCM_DUMP_SERVICE_REC(&srv_rec); 2820 2821 IBTF_DPRINTF_L4(cmlog, "ibt_bind_service: " 2822 "ibmf_saa_write_service_record, SvcId = %llX", 2823 (longlong_t)sid); 2824 2825 status = ibcm_write_service_record(saa_handle, &srv_rec, 2826 IBMF_SAA_UPDATE); 2827 if (status != IBT_SUCCESS) { 2828 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service:" 2829 " ibcm_write_service_record fails %d, " 2830 "sid %llX", status, (longlong_t)sid); 2831 2832 if (sid != start_sid) { 2833 /* 2834 * Bind failed while bind SID other than 2835 * first in the sid_range. So we need 2836 * to unbind those, which are passed. 2837 * 2838 * Need to increment svc count to 2839 * compensate for ibt_unbind_service(). 2840 */ 2841 ibcm_inc_hca_svc_cnt(hcap); 2842 ibcm_dec_hca_acc_cnt(hcap); 2843 2844 (void) ibt_unbind_service(srv_hdl, 2845 sbindp); 2846 } else { 2847 ibcm_svc_bind_t **sbpp; 2848 2849 /* 2850 * Bind failed for the first SID or the 2851 * only SID in question, then no need 2852 * to unbind, just free memory and 2853 * return error. 2854 */ 2855 mutex_enter(&ibcm_svc_info_lock); 2856 2857 sbpp = &srv_hdl->svc_bind_list; 2858 sbp = *sbpp; 2859 while (sbp != NULL) { 2860 if (sbp == sbindp) { 2861 *sbpp = sbp->sbind_link; 2862 break; 2863 } 2864 sbpp = &sbp->sbind_link; 2865 sbp = *sbpp; 2866 } 2867 mutex_exit(&ibcm_svc_info_lock); 2868 ibcm_dec_hca_acc_cnt(hcap); 2869 2870 kmem_free(sbindp, sizeof (*sbindp)); 2871 } 2872 return (status); 2873 } 2874 } 2875 } 2876 ibcm_inc_hca_svc_cnt(hcap); 2877 ibcm_dec_hca_acc_cnt(hcap); 2878 2879 /* If this message isn't seen then ibt_bind_service failed */ 2880 IBTF_DPRINTF_L2(cmlog, "ibt_bind_service: DONE (%p, %llX:%llX)", 2881 srv_hdl, gid.gid_prefix, gid.gid_guid); 2882 2883 if (sb_hdl_p != NULL) 2884 *sb_hdl_p = sbindp; 2885 2886 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*sbindp)) 2887 2888 return (IBT_SUCCESS); 2889 } 2890 2891 ibt_status_t 2892 ibt_unbind_service(ibt_srv_hdl_t srv_hdl, ibt_sbind_hdl_t sbindp) 2893 { 2894 ib_svc_id_t sid, end_sid; 2895 ibt_status_t rval; 2896 ibcm_hca_info_t *hcap; 2897 ibcm_svc_bind_t *sbp, **sbpp; 2898 2899 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service(%p, %p)", 2900 srv_hdl, sbindp); 2901 2902 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 2903 2904 /* If there is a service on hca, respective hcap cannot go away */ 2905 ASSERT(hcap != NULL); 2906 2907 mutex_enter(&ibcm_svc_info_lock); 2908 2909 sbpp = &srv_hdl->svc_bind_list; 2910 sbp = *sbpp; 2911 while (sbp != NULL) { 2912 if (sbp == sbindp) { 2913 *sbpp = sbp->sbind_link; 2914 break; 2915 } 2916 sbpp = &sbp->sbind_link; 2917 sbp = *sbpp; 2918 } 2919 sid = srv_hdl->svc_id; 2920 end_sid = srv_hdl->svc_id + srv_hdl->svc_num_sids - 1; 2921 if (sbp != NULL) 2922 while (sbp->sbind_rewrite_state == IBCM_REWRITE_BUSY) 2923 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 2924 mutex_exit(&ibcm_svc_info_lock); 2925 2926 if (sbp == NULL) { 2927 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2928 "service binding not found: srv_hdl %p, srv_bind %p", 2929 srv_hdl, sbindp); 2930 ibcm_dec_hca_acc_cnt(hcap); 2931 return (IBT_INVALID_PARAM); 2932 } 2933 2934 if (sbindp->sbind_pkey != 0) { /* Are there service records? */ 2935 ibtl_cm_hca_port_t port; 2936 sa_service_record_t srv_rec; 2937 ibmf_saa_handle_t saa_handle; 2938 ibt_status_t status; 2939 2940 /* get the default SGID of the port */ 2941 if ((status = ibtl_cm_get_hca_port(sbindp->sbind_gid, 0, &port)) 2942 != IBT_SUCCESS) { 2943 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2944 "ibtl_cm_get_hca_port failed: %d", status); 2945 /* we're done, but there may be stale service records */ 2946 goto done; 2947 } 2948 2949 saa_handle = ibcm_get_saa_handle(hcap, port.hp_port); 2950 if (saa_handle == NULL) { 2951 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2952 "saa_handle is NULL"); 2953 /* we're done, but there may be stale service records */ 2954 goto done; 2955 } 2956 2957 /* Fill in fields of srv_rec */ 2958 bzero(&srv_rec, sizeof (srv_rec)); 2959 2960 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 2961 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 2962 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 2963 srv_rec.ServiceGID = sbindp->sbind_gid; 2964 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 2965 2966 while (sid <= end_sid) { 2967 2968 srv_rec.ServiceID = sid; 2969 IBCM_DUMP_SERVICE_REC(&srv_rec); 2970 2971 rval = ibcm_write_service_record(saa_handle, &srv_rec, 2972 IBMF_SAA_DELETE); 2973 2974 IBTF_DPRINTF_L4(cmlog, "ibt_unbind_service: " 2975 "ibcm_write_service_record rval = %d, SID %llx", 2976 rval, sid); 2977 if (rval != IBT_SUCCESS) { 2978 /* this is not considered a reason to fail */ 2979 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: " 2980 "ibcm_write_service_record fails %d, " 2981 "sid %llx", rval, sid); 2982 } 2983 sid++; 2984 } 2985 } 2986 done: 2987 ibcm_dec_hca_svc_cnt(hcap); 2988 ibcm_dec_hca_acc_cnt(hcap); 2989 kmem_free(sbindp, sizeof (*sbindp)); 2990 2991 /* If this message isn't seen then ibt_unbind_service failed */ 2992 IBTF_DPRINTF_L2(cmlog, "ibt_unbind_service: done !!"); 2993 2994 return (IBT_SUCCESS); 2995 } 2996 2997 /* 2998 * Simply pull off each binding from the list and unbind it. 2999 * If any of the unbind calls fail, we fail. 3000 */ 3001 ibt_status_t 3002 ibt_unbind_all_services(ibt_srv_hdl_t srv_hdl) 3003 { 3004 ibt_status_t status; 3005 ibcm_svc_bind_t *sbp; 3006 3007 mutex_enter(&ibcm_svc_info_lock); 3008 sbp = NULL; 3009 3010 /* this compare keeps the loop from being infinite */ 3011 while (sbp != srv_hdl->svc_bind_list) { 3012 sbp = srv_hdl->svc_bind_list; 3013 mutex_exit(&ibcm_svc_info_lock); 3014 status = ibt_unbind_service(srv_hdl, sbp); 3015 if (status != IBT_SUCCESS) 3016 return (status); 3017 mutex_enter(&ibcm_svc_info_lock); 3018 if (srv_hdl->svc_bind_list == NULL) 3019 break; 3020 } 3021 mutex_exit(&ibcm_svc_info_lock); 3022 return (IBT_SUCCESS); 3023 } 3024 3025 /* 3026 * ibt_deregister_service() 3027 * Deregister a service with the IBCM 3028 * 3029 * INPUTS: 3030 * ibt_hdl The IBT client handle returned to the client 3031 * on an ibt_attach() call. 3032 * 3033 * srv_hdl The address of a service identification handle, used 3034 * to de-register a service. 3035 * RETURN VALUES: 3036 * IBT_SUCCESS on success (or respective failure on error) 3037 */ 3038 ibt_status_t 3039 ibt_deregister_service(ibt_clnt_hdl_t ibt_hdl, ibt_srv_hdl_t srv_hdl) 3040 { 3041 ibcm_svc_info_t *svcp; 3042 ibcm_svc_lookup_t svc; 3043 3044 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(%p, %p)", 3045 ibt_hdl, srv_hdl); 3046 3047 mutex_enter(&ibcm_svc_info_lock); 3048 3049 if (srv_hdl->svc_bind_list != NULL) { 3050 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service:" 3051 " srv_hdl %p still has bindings", srv_hdl); 3052 mutex_exit(&ibcm_svc_info_lock); 3053 return (IBT_CM_SERVICE_BUSY); 3054 } 3055 svc.sid = srv_hdl->svc_id; 3056 svc.num_sids = 1; 3057 IBTF_DPRINTF_L3(cmlog, "ibt_deregister_service: SID 0x%llX, numsids %d", 3058 srv_hdl->svc_id, srv_hdl->svc_num_sids); 3059 3060 #ifdef __lock_lint 3061 ibcm_svc_compare(NULL, NULL); 3062 #endif 3063 svcp = avl_find(&ibcm_svc_avl_tree, &svc, NULL); 3064 if (svcp != srv_hdl) { 3065 mutex_exit(&ibcm_svc_info_lock); 3066 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service(): " 3067 "srv_hdl %p not found", srv_hdl); 3068 return (IBT_INVALID_PARAM); 3069 } 3070 avl_remove(&ibcm_svc_avl_tree, svcp); 3071 3072 /* wait for active REQ/SREQ handling to be done */ 3073 svcp->svc_to_delete = 1; 3074 while (svcp->svc_ref_cnt != 0) 3075 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3076 3077 mutex_exit(&ibcm_svc_info_lock); 3078 3079 if ((srv_hdl->svc_id & IB_SID_AGN_MASK) == IB_SID_AGN_LOCAL) 3080 ibcm_free_local_sids(srv_hdl->svc_id, srv_hdl->svc_num_sids); 3081 3082 ibtl_cm_change_service_cnt(ibt_hdl, -srv_hdl->svc_num_sids); 3083 kmem_free(srv_hdl, sizeof (*srv_hdl)); 3084 3085 /* If this message isn't seen then ibt_deregister_service failed */ 3086 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_service: done !!"); 3087 3088 return (IBT_SUCCESS); 3089 } 3090 3091 ibcm_status_t 3092 ibcm_ar_init(void) 3093 { 3094 ib_svc_id_t sid = IBCM_DAPL_ATS_SID; 3095 ibcm_svc_info_t *tmp_svcp; 3096 3097 IBTF_DPRINTF_L3(cmlog, "ibcm_ar_init()"); 3098 3099 /* remove this special SID from the pool of available SIDs */ 3100 if ((tmp_svcp = ibcm_create_svc_entry(sid, 1)) == NULL) { 3101 IBTF_DPRINTF_L2(cmlog, "ibcm_ar_init: " 3102 "DAPL ATS SID 0x%llx already registered", (longlong_t)sid); 3103 return (IBCM_FAILURE); 3104 } 3105 mutex_enter(&ibcm_svc_info_lock); 3106 ibcm_ar_svcinfop = tmp_svcp; 3107 ibcm_ar_list = NULL; /* no address records registered yet */ 3108 mutex_exit(&ibcm_svc_info_lock); 3109 return (IBCM_SUCCESS); 3110 } 3111 3112 ibcm_status_t 3113 ibcm_ar_fini(void) 3114 { 3115 ibcm_ar_t *ar_list; 3116 ibcm_svc_info_t *tmp_svcp; 3117 3118 mutex_enter(&ibcm_svc_info_lock); 3119 ar_list = ibcm_ar_list; 3120 3121 if (ar_list == NULL && 3122 avl_numnodes(&ibcm_svc_avl_tree) == 1 && 3123 avl_first(&ibcm_svc_avl_tree) == ibcm_ar_svcinfop) { 3124 avl_remove(&ibcm_svc_avl_tree, ibcm_ar_svcinfop); 3125 tmp_svcp = ibcm_ar_svcinfop; 3126 mutex_exit(&ibcm_svc_info_lock); 3127 kmem_free(tmp_svcp, sizeof (*ibcm_ar_svcinfop)); 3128 return (IBCM_SUCCESS); 3129 } 3130 mutex_exit(&ibcm_svc_info_lock); 3131 return (IBCM_FAILURE); 3132 } 3133 3134 3135 /* 3136 * Return to the caller: 3137 * IBT_SUCCESS Found a perfect match. 3138 * *arpp is set to the record. 3139 * IBT_INCONSISTENT_AR Found a record that's inconsistent. 3140 * IBT_AR_NOT_REGISTERED Found no record with same GID/pkey and 3141 * found no record with same data. 3142 */ 3143 static ibt_status_t 3144 ibcm_search_ar(ibt_ar_t *arp, ibcm_ar_t **arpp) 3145 { 3146 ibcm_ar_t *tmp; 3147 int i; 3148 3149 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3150 tmp = ibcm_ar_list; 3151 while (tmp != NULL) { 3152 if (tmp->ar.ar_gid.gid_prefix == arp->ar_gid.gid_prefix && 3153 tmp->ar.ar_gid.gid_guid == arp->ar_gid.gid_guid && 3154 tmp->ar.ar_pkey == arp->ar_pkey) { 3155 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3156 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3157 return (IBT_INCONSISTENT_AR); 3158 *arpp = tmp; 3159 return (IBT_SUCCESS); 3160 } else { 3161 /* if all the data bytes match, we have inconsistency */ 3162 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) 3163 if (tmp->ar.ar_data[i] != arp->ar_data[i]) 3164 break; 3165 if (i == IBCM_DAPL_ATS_NBYTES) 3166 return (IBT_INCONSISTENT_AR); 3167 /* try next address record */ 3168 } 3169 tmp = tmp->ar_link; 3170 } 3171 return (IBT_AR_NOT_REGISTERED); 3172 } 3173 3174 ibt_status_t 3175 ibt_register_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3176 { 3177 ibcm_ar_t *found; 3178 ibcm_ar_t *tmp; 3179 ibt_status_t status; 3180 ibt_status_t s1, s2; 3181 char *s; 3182 ibcm_ar_ref_t *hdlp; 3183 ibcm_ar_t *new; 3184 ibcm_ar_t **linkp; 3185 ibtl_cm_hca_port_t cm_port; 3186 uint16_t pkey_ix; 3187 ibcm_hca_info_t *hcap; 3188 ibmf_saa_handle_t saa_handle; 3189 sa_service_record_t *srv_recp; 3190 uint64_t gid_ored; 3191 3192 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: PKey 0x%X GID %llX:%llX", 3193 arp->ar_pkey, (longlong_t)arp->ar_gid.gid_prefix, 3194 (longlong_t)arp->ar_gid.gid_guid); 3195 3196 /* 3197 * If P_Key is 0, but GID is not, this query is invalid. 3198 * If GID is 0, but P_Key is not, this query is invalid. 3199 */ 3200 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3201 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3202 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3203 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3204 "GID/P_Key is not valid"); 3205 return (IBT_INVALID_PARAM); 3206 } 3207 3208 /* assume success, so these might be needed */ 3209 hdlp = kmem_alloc(sizeof (*hdlp), KM_SLEEP); 3210 new = kmem_zalloc(sizeof (*new), KM_SLEEP); 3211 3212 mutex_enter(&ibcm_svc_info_lock); 3213 /* search for existing GID/pkey (there can be at most 1) */ 3214 status = ibcm_search_ar(arp, &found); 3215 if (status == IBT_INCONSISTENT_AR) { 3216 mutex_exit(&ibcm_svc_info_lock); 3217 kmem_free(new, sizeof (*new)); 3218 kmem_free(hdlp, sizeof (*hdlp)); 3219 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3220 "address record is inconsistent with a known one"); 3221 return (IBT_INCONSISTENT_AR); 3222 } else if (status == IBT_SUCCESS) { 3223 if (found->ar_flags == IBCM_AR_INITING) { 3224 found->ar_waiters++; 3225 cv_wait(&found->ar_cv, &ibcm_svc_info_lock); 3226 found->ar_waiters--; 3227 } 3228 if (found->ar_flags == IBCM_AR_FAILED) { 3229 if (found->ar_waiters == 0) { 3230 cv_destroy(&found->ar_cv); 3231 kmem_free(found, sizeof (*found)); 3232 } 3233 mutex_exit(&ibcm_svc_info_lock); 3234 kmem_free(new, sizeof (*new)); 3235 kmem_free(hdlp, sizeof (*hdlp)); 3236 return (ibt_get_module_failure(IBT_FAILURE_IBCM, 0)); 3237 } 3238 hdlp->ar_ibt_hdl = ibt_hdl; 3239 hdlp->ar_ref_link = found->ar_ibt_hdl_list; 3240 found->ar_ibt_hdl_list = hdlp; 3241 mutex_exit(&ibcm_svc_info_lock); 3242 kmem_free(new, sizeof (*new)); 3243 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3244 return (IBT_SUCCESS); 3245 } else { 3246 ASSERT(status == IBT_AR_NOT_REGISTERED); 3247 } 3248 hdlp->ar_ref_link = NULL; 3249 hdlp->ar_ibt_hdl = ibt_hdl; 3250 new->ar_ibt_hdl_list = hdlp; 3251 new->ar = *arp; 3252 new->ar_flags = IBCM_AR_INITING; 3253 new->ar_waiters = 0; 3254 cv_init(&new->ar_cv, NULL, CV_DEFAULT, NULL); 3255 new->ar_link = ibcm_ar_list; 3256 ibcm_ar_list = new; 3257 3258 /* verify GID/pkey is valid for a local port, etc. */ 3259 hcap = NULL; 3260 if ((s1 = ibtl_cm_get_hca_port(arp->ar_gid, 0, &cm_port)) 3261 != IBT_SUCCESS || 3262 (s2 = ibt_pkey2index_byguid(cm_port.hp_hca_guid, cm_port.hp_port, 3263 arp->ar_pkey, &pkey_ix)) != IBT_SUCCESS || 3264 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL) { 3265 cv_destroy(&new->ar_cv); 3266 ibcm_ar_list = new->ar_link; 3267 mutex_exit(&ibcm_svc_info_lock); 3268 kmem_free(new, sizeof (*new)); 3269 kmem_free(hdlp, sizeof (*hdlp)); 3270 status = IBT_INVALID_PARAM; 3271 if (s1 == IBT_HCA_PORT_NOT_ACTIVE) { 3272 s = "PORT DOWN"; 3273 status = IBT_HCA_PORT_NOT_ACTIVE; 3274 } else if (s1 != IBT_SUCCESS) 3275 s = "GID not found"; 3276 else if (s2 != IBT_SUCCESS) 3277 s = "PKEY not found"; 3278 else 3279 s = "CM could not find its HCA entry"; 3280 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: %s, status = %d", 3281 s, status); 3282 return (status); 3283 } 3284 mutex_exit(&ibcm_svc_info_lock); 3285 saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port); 3286 3287 /* create service record */ 3288 srv_recp = kmem_zalloc(sizeof (*srv_recp), KM_SLEEP); 3289 srv_recp->ServiceLease = 0xFFFFFFFF; /* infinite */ 3290 srv_recp->ServiceP_Key = arp->ar_pkey; 3291 srv_recp->ServiceKey_hi = 0xDA410000ULL; /* DAPL */ 3292 srv_recp->ServiceKey_lo = 0xA7500000ULL; /* ATS */ 3293 (void) strcpy((char *)srv_recp->ServiceName, IBCM_DAPL_ATS_NAME); 3294 srv_recp->ServiceGID = arp->ar_gid; 3295 bcopy(arp->ar_data, srv_recp->ServiceData, IBCM_DAPL_ATS_NBYTES); 3296 srv_recp->ServiceID = IBCM_DAPL_ATS_SID; 3297 3298 /* insert service record into the SA */ 3299 3300 IBCM_DUMP_SERVICE_REC(srv_recp); 3301 3302 if (saa_handle != NULL) 3303 status = ibcm_write_service_record(saa_handle, srv_recp, 3304 IBMF_SAA_UPDATE); 3305 else 3306 status = IBT_HCA_PORT_NOT_ACTIVE; 3307 3308 if (status != IBT_SUCCESS) { 3309 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: sa access fails %d, " 3310 "sid %llX", status, (longlong_t)srv_recp->ServiceID); 3311 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: FAILED for gid " 3312 "%llX:%llX pkey 0x%X", (longlong_t)arp->ar_gid.gid_prefix, 3313 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3314 3315 kmem_free(srv_recp, sizeof (*srv_recp)); 3316 kmem_free(hdlp, sizeof (*hdlp)); 3317 3318 mutex_enter(&ibcm_svc_info_lock); 3319 linkp = &ibcm_ar_list; 3320 tmp = *linkp; 3321 while (tmp != NULL) { 3322 if (tmp == new) { 3323 *linkp = new->ar_link; 3324 break; 3325 } 3326 linkp = &tmp->ar_link; 3327 tmp = *linkp; 3328 } 3329 if (new->ar_waiters > 0) { 3330 new->ar_flags = IBCM_AR_FAILED; 3331 cv_broadcast(&new->ar_cv); 3332 mutex_exit(&ibcm_svc_info_lock); 3333 } else { 3334 cv_destroy(&new->ar_cv); 3335 mutex_exit(&ibcm_svc_info_lock); 3336 kmem_free(new, sizeof (*new)); 3337 } 3338 ibcm_dec_hca_acc_cnt(hcap); 3339 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: " 3340 "IBMF_SAA failed to write address record"); 3341 } else { /* SUCCESS */ 3342 uint8_t *b; 3343 3344 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar: SUCCESS for gid " 3345 "%llx:%llx pkey %x", (longlong_t)arp->ar_gid.gid_prefix, 3346 (longlong_t)arp->ar_gid.gid_guid, arp->ar_pkey); 3347 b = arp->ar_data; 3348 3349 IBTF_DPRINTF_L2(cmlog, "ibt_register_ar:" 3350 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3351 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3352 b[10], b[11], b[12], b[13], b[14], b[15]); 3353 mutex_enter(&ibcm_svc_info_lock); 3354 new->ar_srv_recp = srv_recp; 3355 new->ar_saa_handle = saa_handle; 3356 new->ar_port = cm_port.hp_port; 3357 new->ar_hcap = hcap; 3358 new->ar_flags = IBCM_AR_SUCCESS; 3359 if (new->ar_waiters > 0) 3360 cv_broadcast(&new->ar_cv); 3361 mutex_exit(&ibcm_svc_info_lock); 3362 ibtl_cm_change_service_cnt(ibt_hdl, 1); 3363 /* do not call ibcm_dec_hca_acc_cnt(hcap) until deregister */ 3364 } 3365 return (status); 3366 } 3367 3368 ibt_status_t 3369 ibt_deregister_ar(ibt_clnt_hdl_t ibt_hdl, ibt_ar_t *arp) 3370 { 3371 ibcm_ar_t *found; 3372 ibcm_ar_t *tmp; 3373 ibcm_ar_t **linkp; 3374 ibcm_ar_ref_t *hdlp; 3375 ibcm_ar_ref_t **hdlpp; 3376 ibt_status_t status; 3377 ibmf_saa_handle_t saa_handle; 3378 sa_service_record_t *srv_recp; 3379 uint64_t gid_ored; 3380 3381 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: pkey %x", arp->ar_pkey); 3382 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: gid %llx:%llx", 3383 (longlong_t)arp->ar_gid.gid_prefix, 3384 (longlong_t)arp->ar_gid.gid_guid); 3385 3386 /* 3387 * If P_Key is 0, but GID is not, this query is invalid. 3388 * If GID is 0, but P_Key is not, this query is invalid. 3389 */ 3390 gid_ored = arp->ar_gid.gid_guid | arp->ar_gid.gid_prefix; 3391 if ((arp->ar_pkey == 0 && gid_ored != 0ULL) || 3392 (arp->ar_pkey != 0 && gid_ored == 0ULL)) { 3393 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3394 "GID/P_Key is not valid"); 3395 return (IBT_INVALID_PARAM); 3396 } 3397 3398 mutex_enter(&ibcm_svc_info_lock); 3399 /* search for existing GID/pkey (there can be at most 1) */ 3400 status = ibcm_search_ar(arp, &found); 3401 if (status == IBT_INCONSISTENT_AR || status == IBT_AR_NOT_REGISTERED) { 3402 mutex_exit(&ibcm_svc_info_lock); 3403 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3404 "address record not found"); 3405 return (IBT_AR_NOT_REGISTERED); 3406 } 3407 ASSERT(status == IBT_SUCCESS); 3408 3409 hdlpp = &found->ar_ibt_hdl_list; 3410 hdlp = *hdlpp; 3411 while (hdlp != NULL) { 3412 if (hdlp->ar_ibt_hdl == ibt_hdl) 3413 break; 3414 hdlpp = &hdlp->ar_ref_link; 3415 hdlp = *hdlpp; 3416 } 3417 if (hdlp == NULL) { /* could not find ibt_hdl on list */ 3418 mutex_exit(&ibcm_svc_info_lock); 3419 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3420 "address record found, but not for this client"); 3421 return (IBT_AR_NOT_REGISTERED); 3422 } 3423 *hdlpp = hdlp->ar_ref_link; /* remove ref for this client */ 3424 if (found->ar_ibt_hdl_list == NULL && found->ar_waiters == 0) { 3425 /* last entry was removed */ 3426 found->ar_flags = IBCM_AR_INITING; /* hold off register_ar */ 3427 saa_handle = found->ar_saa_handle; 3428 srv_recp = found->ar_srv_recp; 3429 3430 /* wait if this service record is being rewritten */ 3431 while (found->ar_rewrite_state == IBCM_REWRITE_BUSY) 3432 cv_wait(&ibcm_svc_info_cv, &ibcm_svc_info_lock); 3433 mutex_exit(&ibcm_svc_info_lock); 3434 3435 /* remove service record */ 3436 status = ibcm_write_service_record(saa_handle, srv_recp, 3437 IBMF_SAA_DELETE); 3438 if (status != IBT_SUCCESS) 3439 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3440 "IBMF_SAA failed to delete address record"); 3441 mutex_enter(&ibcm_svc_info_lock); 3442 if (found->ar_waiters == 0) { /* still no waiters */ 3443 linkp = &ibcm_ar_list; 3444 tmp = *linkp; 3445 while (tmp != found) { 3446 linkp = &tmp->ar_link; 3447 tmp = *linkp; 3448 } 3449 *linkp = tmp->ar_link; 3450 ibcm_dec_hca_acc_cnt(found->ar_hcap); 3451 kmem_free(srv_recp, sizeof (*srv_recp)); 3452 cv_destroy(&found->ar_cv); 3453 kmem_free(found, sizeof (*found)); 3454 } else { 3455 /* add service record back in for the waiters */ 3456 mutex_exit(&ibcm_svc_info_lock); 3457 status = ibcm_write_service_record(saa_handle, srv_recp, 3458 IBMF_SAA_UPDATE); 3459 mutex_enter(&ibcm_svc_info_lock); 3460 if (status == IBT_SUCCESS) 3461 found->ar_flags = IBCM_AR_SUCCESS; 3462 else { 3463 found->ar_flags = IBCM_AR_FAILED; 3464 IBTF_DPRINTF_L2(cmlog, "ibt_deregister_ar: " 3465 "IBMF_SAA failed to write address record"); 3466 } 3467 cv_broadcast(&found->ar_cv); 3468 } 3469 } 3470 mutex_exit(&ibcm_svc_info_lock); 3471 kmem_free(hdlp, sizeof (*hdlp)); 3472 ibtl_cm_change_service_cnt(ibt_hdl, -1); 3473 return (status); 3474 } 3475 3476 ibt_status_t 3477 ibt_query_ar(ib_gid_t *sgid, ibt_ar_t *queryp, ibt_ar_t *resultp) 3478 { 3479 sa_service_record_t svcrec_req; 3480 sa_service_record_t *svcrec_resp; 3481 void *results_p; 3482 uint64_t component_mask = 0; 3483 uint64_t gid_ored; 3484 size_t length; 3485 int num_rec; 3486 int i; 3487 ibmf_saa_access_args_t access_args; 3488 ibt_status_t retval; 3489 ibtl_cm_hca_port_t cm_port; 3490 ibcm_hca_info_t *hcap; 3491 ibmf_saa_handle_t saa_handle; 3492 3493 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar(%p, %p)", queryp, resultp); 3494 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: sgid %llx:%llx", 3495 (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid); 3496 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_pkey %x", queryp->ar_pkey); 3497 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: query_gid %llx:%llx", 3498 (longlong_t)queryp->ar_gid.gid_prefix, 3499 (longlong_t)queryp->ar_gid.gid_guid); 3500 3501 /* 3502 * If P_Key is 0, but GID is not, this query is invalid. 3503 * If GID is 0, but P_Key is not, this query is invalid. 3504 */ 3505 gid_ored = queryp->ar_gid.gid_guid | queryp->ar_gid.gid_prefix; 3506 if ((queryp->ar_pkey == 0 && gid_ored != 0ULL) || 3507 (queryp->ar_pkey != 0 && gid_ored == 0ULL)) { 3508 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: GID/P_Key is not valid"); 3509 return (IBT_INVALID_PARAM); 3510 } 3511 3512 hcap = NULL; 3513 if (ibtl_cm_get_hca_port(*sgid, 0, &cm_port) != IBT_SUCCESS || 3514 (hcap = ibcm_find_hca_entry(cm_port.hp_hca_guid)) == NULL || 3515 (saa_handle = ibcm_get_saa_handle(hcap, cm_port.hp_port)) == NULL) { 3516 if (hcap != NULL) 3517 ibcm_dec_hca_acc_cnt(hcap); 3518 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: sgid is not valid"); 3519 return (IBT_INVALID_PARAM); 3520 } 3521 3522 bzero(&svcrec_req, sizeof (svcrec_req)); 3523 3524 /* Is GID/P_Key Specified. */ 3525 if (queryp->ar_pkey != 0) { /* GID is non-zero from check above */ 3526 svcrec_req.ServiceP_Key = queryp->ar_pkey; 3527 component_mask |= SA_SR_COMPMASK_PKEY; 3528 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: P_Key %X", 3529 queryp->ar_pkey); 3530 svcrec_req.ServiceGID = queryp->ar_gid; 3531 component_mask |= SA_SR_COMPMASK_GID; 3532 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: GID %llX:%llX", 3533 (longlong_t)queryp->ar_gid.gid_prefix, 3534 (longlong_t)queryp->ar_gid.gid_guid); 3535 } 3536 3537 /* Is ServiceData Specified. */ 3538 for (i = 0; i < IBCM_DAPL_ATS_NBYTES; i++) { 3539 if (queryp->ar_data[i] != 0) { 3540 bcopy(queryp->ar_data, svcrec_req.ServiceData, 3541 IBCM_DAPL_ATS_NBYTES); 3542 component_mask |= 0xFFFF << 7; /* all 16 Data8 */ 3543 /* components */ 3544 break; 3545 } 3546 } 3547 3548 /* Service Name */ 3549 (void) strcpy((char *)svcrec_req.ServiceName, IBCM_DAPL_ATS_NAME); 3550 component_mask |= SA_SR_COMPMASK_NAME; 3551 3552 svcrec_req.ServiceID = IBCM_DAPL_ATS_SID; 3553 component_mask |= SA_SR_COMPMASK_ID; 3554 3555 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3556 "Perform SA Access: Mask: 0x%X", component_mask); 3557 3558 /* 3559 * Call in SA Access retrieve routine to get Service Records. 3560 * 3561 * SA Access framework allocated memory for the "results_p". 3562 * Make sure to deallocate once we are done with the results_p. 3563 * The size of the buffer allocated will be as returned in 3564 * "length" field. 3565 */ 3566 access_args.sq_attr_id = SA_SERVICERECORD_ATTRID; 3567 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 3568 access_args.sq_component_mask = component_mask; 3569 access_args.sq_template = &svcrec_req; 3570 access_args.sq_template_length = sizeof (sa_service_record_t); 3571 access_args.sq_callback = NULL; 3572 access_args.sq_callback_arg = NULL; 3573 3574 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 3575 &results_p); 3576 3577 ibcm_dec_hca_acc_cnt(hcap); 3578 if (retval != IBT_SUCCESS) { 3579 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: SA Access Failed"); 3580 return (retval); 3581 } 3582 3583 num_rec = length / sizeof (sa_service_record_t); 3584 3585 IBTF_DPRINTF_L3(cmlog, "ibt_query_ar: " 3586 "Found %d Service Records.", num_rec); 3587 3588 /* Validate the returned number of records. */ 3589 if ((results_p != NULL) && (num_rec > 0)) { 3590 uint8_t *b; 3591 3592 /* Just return info from the first service record. */ 3593 svcrec_resp = (sa_service_record_t *)results_p; 3594 3595 /* The Service GID and Service ID */ 3596 resultp->ar_gid = svcrec_resp->ServiceGID; 3597 resultp->ar_pkey = svcrec_resp->ServiceP_Key; 3598 bcopy(svcrec_resp->ServiceData, 3599 resultp->ar_data, IBCM_DAPL_ATS_NBYTES); 3600 3601 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3602 "Found: pkey %x dgid %llX:%llX", resultp->ar_pkey, 3603 (longlong_t)resultp->ar_gid.gid_prefix, 3604 (longlong_t)resultp->ar_gid.gid_guid); 3605 b = resultp->ar_data; 3606 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar:" 3607 " data %d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d:%d", 3608 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], 3609 b[10], b[11], b[12], b[13], b[14], b[15]); 3610 3611 /* Deallocate the memory for results_p. */ 3612 kmem_free(results_p, length); 3613 if (num_rec > 1) 3614 retval = IBT_MULTIPLE_AR; 3615 else 3616 retval = IBT_SUCCESS; 3617 } else { 3618 IBTF_DPRINTF_L2(cmlog, "ibt_query_ar: " 3619 "ibmf_sa_access found 0 matching records"); 3620 retval = IBT_AR_NOT_REGISTERED; 3621 } 3622 return (retval); 3623 } 3624 3625 /* mark all ATS service records associated with the port */ 3626 static void 3627 ibcm_mark_ar(ib_guid_t hca_guid, uint8_t port) 3628 { 3629 ibcm_ar_t *tmp; 3630 3631 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3632 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3633 if (tmp->ar_hcap == NULL) 3634 continue; 3635 if (tmp->ar_hcap->hca_guid == hca_guid && 3636 tmp->ar_port == port) { 3637 /* even if it's busy, we mark it for rewrite */ 3638 tmp->ar_rewrite_state = IBCM_REWRITE_NEEDED; 3639 } 3640 } 3641 } 3642 3643 /* rewrite all ATS service records */ 3644 static int 3645 ibcm_rewrite_ar(void) 3646 { 3647 ibcm_ar_t *tmp; 3648 ibmf_saa_handle_t saa_handle; 3649 sa_service_record_t *srv_recp; 3650 ibt_status_t rval; 3651 int did_something = 0; 3652 3653 ASSERT(MUTEX_HELD(&ibcm_svc_info_lock)); 3654 check_for_work: 3655 for (tmp = ibcm_ar_list; tmp != NULL; tmp = tmp->ar_link) { 3656 if (tmp->ar_rewrite_state == IBCM_REWRITE_NEEDED) { 3657 tmp->ar_rewrite_state = IBCM_REWRITE_BUSY; 3658 saa_handle = tmp->ar_saa_handle; 3659 srv_recp = tmp->ar_srv_recp; 3660 mutex_exit(&ibcm_svc_info_lock); 3661 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_ar: " 3662 "rewriting ar @ %p", tmp); 3663 did_something = 1; 3664 rval = ibcm_write_service_record(saa_handle, srv_recp, 3665 IBMF_SAA_UPDATE); 3666 if (rval != IBT_SUCCESS) 3667 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_ar: " 3668 "ibcm_write_service_record failed: " 3669 "status = %d", rval); 3670 mutex_enter(&ibcm_svc_info_lock); 3671 /* if it got marked again, then we want to rewrite */ 3672 if (tmp->ar_rewrite_state == IBCM_REWRITE_BUSY) 3673 tmp->ar_rewrite_state = IBCM_REWRITE_IDLE; 3674 /* in case there was a waiter... */ 3675 cv_broadcast(&ibcm_svc_info_cv); 3676 goto check_for_work; 3677 } 3678 } 3679 return (did_something); 3680 } 3681 3682 static void 3683 ibcm_rewrite_svc_record(ibcm_svc_info_t *srv_hdl, ibcm_svc_bind_t *sbindp) 3684 { 3685 ibcm_hca_info_t *hcap; 3686 ib_svc_id_t sid, start_sid, end_sid; 3687 ibmf_saa_handle_t saa_handle; 3688 sa_service_record_t srv_rec; 3689 ibt_status_t rval; 3690 3691 hcap = ibcm_find_hca_entry(sbindp->sbind_hcaguid); 3692 if (hcap == NULL) { 3693 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3694 "NO HCA found for HCA GUID %llX", sbindp->sbind_hcaguid); 3695 return; 3696 } 3697 3698 saa_handle = ibcm_get_saa_handle(hcap, sbindp->sbind_port); 3699 if (saa_handle == NULL) { 3700 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record: " 3701 "saa_handle is NULL"); 3702 ibcm_dec_hca_acc_cnt(hcap); 3703 return; 3704 } 3705 3706 IBTF_DPRINTF_L3(cmlog, "ibcm_rewrite_svc_record: " 3707 "rewriting svc '%s', port_guid = %llX", sbindp->sbind_name, 3708 sbindp->sbind_gid.gid_guid); 3709 3710 bzero(&srv_rec, sizeof (srv_rec)); 3711 3712 srv_rec.ServiceLease = sbindp->sbind_lease; 3713 srv_rec.ServiceP_Key = sbindp->sbind_pkey; 3714 srv_rec.ServiceKey_hi = sbindp->sbind_key[0]; 3715 srv_rec.ServiceKey_lo = sbindp->sbind_key[1]; 3716 (void) strcpy((char *)srv_rec.ServiceName, sbindp->sbind_name); 3717 srv_rec.ServiceGID = sbindp->sbind_gid; 3718 3719 bcopy(sbindp->sbind_data, srv_rec.ServiceData, IB_SVC_DATA_LEN); 3720 3721 /* insert srv record into the SA */ 3722 start_sid = srv_hdl->svc_id; 3723 end_sid = start_sid + srv_hdl->svc_num_sids - 1; 3724 for (sid = start_sid; sid <= end_sid; sid++) { 3725 srv_rec.ServiceID = sid; 3726 3727 rval = ibcm_write_service_record(saa_handle, &srv_rec, 3728 IBMF_SAA_UPDATE); 3729 3730 IBTF_DPRINTF_L4(cmlog, "ibcm_rewrite_svc_record: " 3731 "ibcm_write_service_record, SvcId = %llX, " 3732 "rval = %d", (longlong_t)sid, rval); 3733 if (rval != IBT_SUCCESS) { 3734 IBTF_DPRINTF_L2(cmlog, "ibcm_rewrite_svc_record:" 3735 " ibcm_write_service_record fails %d sid %llX", 3736 rval, (longlong_t)sid); 3737 } 3738 } 3739 ibcm_dec_hca_acc_cnt(hcap); 3740 } 3741 3742 /* 3743 * Task to mark all service records as needing to be rewritten to the SM/SA. 3744 * This task does not return until all of them have been rewritten. 3745 */ 3746 void 3747 ibcm_service_record_rewrite_task(void *arg) 3748 { 3749 ibcm_port_up_t *pup = (ibcm_port_up_t *)arg; 3750 ib_guid_t hca_guid = pup->pup_hca_guid; 3751 uint8_t port = pup->pup_port; 3752 ibcm_svc_info_t *svcp; 3753 ibcm_svc_bind_t *sbp; 3754 avl_tree_t *avl_tree = &ibcm_svc_avl_tree; 3755 static int task_is_running = 0; 3756 3757 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task STARTED " 3758 "for hca_guid %llX, port %d", hca_guid, port); 3759 3760 mutex_enter(&ibcm_svc_info_lock); 3761 ibcm_mark_ar(hca_guid, port); 3762 for (svcp = avl_first(avl_tree); svcp != NULL; 3763 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3764 sbp = svcp->svc_bind_list; 3765 while (sbp != NULL) { 3766 if (sbp->sbind_pkey != 0 && 3767 sbp->sbind_port == port && 3768 sbp->sbind_hcaguid == hca_guid) { 3769 /* even if it's busy, we mark it for rewrite */ 3770 sbp->sbind_rewrite_state = IBCM_REWRITE_NEEDED; 3771 } 3772 sbp = sbp->sbind_link; 3773 } 3774 } 3775 if (task_is_running) { 3776 /* let the other task thread finish the work */ 3777 mutex_exit(&ibcm_svc_info_lock); 3778 return; 3779 } 3780 task_is_running = 1; 3781 3782 (void) ibcm_rewrite_ar(); 3783 3784 check_for_work: 3785 for (svcp = avl_first(avl_tree); svcp != NULL; 3786 svcp = avl_walk(avl_tree, svcp, AVL_AFTER)) { 3787 sbp = svcp->svc_bind_list; 3788 while (sbp != NULL) { 3789 if (sbp->sbind_rewrite_state == IBCM_REWRITE_NEEDED) { 3790 sbp->sbind_rewrite_state = IBCM_REWRITE_BUSY; 3791 mutex_exit(&ibcm_svc_info_lock); 3792 ibcm_rewrite_svc_record(svcp, sbp); 3793 mutex_enter(&ibcm_svc_info_lock); 3794 /* if it got marked again, we want to rewrite */ 3795 if (sbp->sbind_rewrite_state == 3796 IBCM_REWRITE_BUSY) 3797 sbp->sbind_rewrite_state = 3798 IBCM_REWRITE_IDLE; 3799 /* in case there was a waiter... */ 3800 cv_broadcast(&ibcm_svc_info_cv); 3801 goto check_for_work; 3802 } 3803 sbp = sbp->sbind_link; 3804 } 3805 } 3806 /* 3807 * If there were no service records to write, and we failed to 3808 * have to rewrite any more ATS service records, then we're done. 3809 */ 3810 if (ibcm_rewrite_ar() != 0) 3811 goto check_for_work; 3812 task_is_running = 0; 3813 mutex_exit(&ibcm_svc_info_lock); 3814 3815 IBTF_DPRINTF_L2(cmlog, "ibcm_service_record_rewrite_task DONE"); 3816 kmem_free(pup, sizeof (ibcm_port_up_t)); 3817 } 3818 3819 3820 /* 3821 * Function: 3822 * ibt_cm_proceed 3823 * 3824 * Verifies the arguments and dispatches the cm state machine processing 3825 * via taskq 3826 */ 3827 3828 ibt_status_t 3829 ibt_cm_proceed(ibt_cm_event_type_t event, void *session_id, 3830 ibt_cm_status_t status, ibt_cm_proceed_reply_t *cm_event_data, 3831 void *priv_data, ibt_priv_data_len_t priv_data_len) 3832 { 3833 ibcm_state_data_t *statep = (ibcm_state_data_t *)session_id; 3834 ibcm_proceed_targs_t *proceed_targs; 3835 ibcm_proceed_error_t proceed_error; 3836 3837 IBTF_DPRINTF_L3(cmlog, "ibt_cm_proceed chan 0x%p event %x status %x " 3838 "session_id %p", statep->channel, event, status, session_id); 3839 3840 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed chan 0x%p cm_event_data %p, " 3841 "priv_data %p priv_data_len %x", statep->channel, cm_event_data, 3842 priv_data, priv_data_len); 3843 3844 /* validate session_id and status */ 3845 if ((statep == NULL) || (status == IBT_CM_DEFER)) { 3846 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : Invalid Args"); 3847 return (IBT_INVALID_PARAM); 3848 } 3849 3850 /* If priv data len specified, then priv_data cannot be NULL */ 3851 if ((priv_data_len > 0) && (priv_data == NULL)) 3852 return (IBT_INVALID_PARAM); 3853 3854 proceed_error = IBCM_PROCEED_INVALID_NONE; 3855 3856 mutex_enter(&statep->state_mutex); 3857 if (event == IBT_CM_EVENT_REQ_RCV) { 3858 3859 if ((statep->state != IBCM_STATE_REQ_RCVD) && 3860 (statep->state != IBCM_STATE_MRA_SENT)) 3861 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3862 else if (priv_data_len > IBT_REP_PRIV_DATA_SZ) 3863 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3864 3865 } else if (event == IBT_CM_EVENT_REP_RCV) { 3866 if ((statep->state != IBCM_STATE_REP_RCVD) && 3867 (statep->state != IBCM_STATE_MRA_REP_SENT)) 3868 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3869 else if (priv_data_len > IBT_RTU_PRIV_DATA_SZ) 3870 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3871 } else if (event == IBT_CM_EVENT_LAP_RCV) { 3872 if ((statep->ap_state != IBCM_AP_STATE_LAP_RCVD) && 3873 (statep->ap_state != IBCM_AP_STATE_MRA_LAP_SENT)) 3874 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3875 else if (priv_data_len > IBT_APR_PRIV_DATA_SZ) 3876 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3877 } else if (event == IBT_CM_EVENT_CONN_CLOSED) { 3878 if (statep->state != IBCM_STATE_DREQ_RCVD) 3879 proceed_error = IBCM_PROCEED_INVALID_EVENT_STATE; 3880 else if (priv_data_len > IBT_DREP_PRIV_DATA_SZ) 3881 proceed_error = IBCM_PROCEED_INVALID_PRIV_SZ; 3882 } else { 3883 proceed_error = IBCM_PROCEED_INVALID_EVENT; 3884 } 3885 3886 /* if there is an error, print an error message and return */ 3887 if (proceed_error != IBCM_PROCEED_INVALID_NONE) { 3888 mutex_exit(&statep->state_mutex); 3889 if (proceed_error == IBCM_PROCEED_INVALID_EVENT_STATE) { 3890 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3891 "Invalid Event/State combination specified", 3892 statep->channel); 3893 return (IBT_INVALID_PARAM); 3894 } else if (proceed_error == IBCM_PROCEED_INVALID_PRIV_SZ) { 3895 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3896 "Invalid Event/priv len combination specified", 3897 statep->channel); 3898 return (IBT_INVALID_PARAM); 3899 } else if (proceed_error == IBCM_PROCEED_INVALID_EVENT) { 3900 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3901 "Invalid Event specified", statep->channel); 3902 return (IBT_INVALID_PARAM); 3903 } else { 3904 ASSERT(proceed_error == IBCM_PROCEED_INVALID_LAP); 3905 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p" 3906 "IBT_CM_EVENT_LAP_RCV not supported", 3907 statep->channel); 3908 /* UNTIL HCA DRIVER ENABLES AP SUPPORT, FAIL THE CALL */ 3909 return (IBT_APM_NOT_SUPPORTED); 3910 } 3911 } 3912 3913 3914 /* wait until client's CM handler returns DEFER status back to CM */ 3915 3916 while (statep->clnt_proceed == IBCM_BLOCK) { 3917 IBTF_DPRINTF_L5(cmlog, "ibt_cm_proceed : chan 0x%p blocked for " 3918 "return of client's cm handler", statep->channel); 3919 cv_wait(&statep->block_client_cv, &statep->state_mutex); 3920 } 3921 3922 if (statep->clnt_proceed == IBCM_FAIL) { 3923 mutex_exit(&statep->state_mutex); 3924 IBTF_DPRINTF_L2(cmlog, "ibt_cm_proceed : chan 0x%p Failed as " 3925 "client returned non-DEFER status from cm handler", 3926 statep->channel); 3927 return (IBT_CHAN_STATE_INVALID); 3928 } 3929 3930 ASSERT(statep->clnt_proceed == IBCM_UNBLOCK); 3931 statep->clnt_proceed = IBCM_FAIL; 3932 mutex_exit(&statep->state_mutex); 3933 3934 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3935 3936 /* the state machine processing is done in a separate thread */ 3937 3938 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 3939 proceed_targs = kmem_alloc(sizeof (ibcm_proceed_targs_t), 3940 KM_SLEEP); 3941 3942 proceed_targs->event = event; 3943 proceed_targs->status = status; 3944 proceed_targs->priv_data_len = priv_data_len; 3945 3946 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 3947 3948 proceed_targs->tst.rc.statep = statep; 3949 bcopy(cm_event_data, &proceed_targs->tst.rc.rc_cm_event_data, 3950 sizeof (ibt_cm_proceed_reply_t)); 3951 3952 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 3953 3954 (void) taskq_dispatch(ibcm_taskq, ibcm_proceed_via_taskq, 3955 proceed_targs, TQ_SLEEP); 3956 3957 return (IBT_SUCCESS); 3958 } 3959 3960 /* 3961 * Function: 3962 * ibcm_proceed_via_taskq 3963 * 3964 * Called from taskq, dispatched by ibt_cm_proceed 3965 * Completes the cm state processing for ibt_cm_proceed 3966 */ 3967 void 3968 ibcm_proceed_via_taskq(void *targs) 3969 { 3970 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 3971 ibcm_state_data_t *statep = proceed_targs->tst.rc.statep; 3972 ibt_cm_reason_t reject_reason; 3973 uint8_t arej_len; 3974 ibcm_status_t response; 3975 ibcm_clnt_reply_info_t clnt_info; 3976 3977 clnt_info.reply_event = &proceed_targs->tst.rc.rc_cm_event_data; 3978 clnt_info.priv_data = proceed_targs->priv_data; 3979 clnt_info.priv_data_len = proceed_targs->priv_data_len; 3980 3981 IBTF_DPRINTF_L4(cmlog, "ibcm_proceed_via_taskq chan 0x%p targs %x", 3982 statep->channel, targs); 3983 3984 if (proceed_targs->event == IBT_CM_EVENT_REQ_RCV) { 3985 response = 3986 ibcm_process_cep_req_cm_hdlr(statep, proceed_targs->status, 3987 &clnt_info, &reject_reason, &arej_len, 3988 (ibcm_req_msg_t *)statep->defer_cm_msg); 3989 3990 ibcm_handle_cep_req_response(statep, response, reject_reason, 3991 arej_len); 3992 3993 } else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) { 3994 response = 3995 ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status, 3996 &clnt_info, &reject_reason, &arej_len, 3997 (ibcm_rep_msg_t *)statep->defer_cm_msg); 3998 3999 ibcm_handle_cep_rep_response(statep, response, reject_reason, 4000 arej_len, (ibcm_rep_msg_t *)statep->defer_cm_msg); 4001 4002 } else if (proceed_targs->event == IBT_CM_EVENT_LAP_RCV) { 4003 ibcm_process_cep_lap_cm_hdlr(statep, proceed_targs->status, 4004 &clnt_info, (ibcm_lap_msg_t *)statep->defer_cm_msg, 4005 (ibcm_apr_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg)); 4006 4007 ibcm_post_apr_mad(statep); 4008 4009 } else { 4010 ASSERT(proceed_targs->event == IBT_CM_EVENT_CONN_CLOSED); 4011 ibcm_handle_cep_dreq_response(statep, proceed_targs->priv_data, 4012 proceed_targs->priv_data_len); 4013 } 4014 4015 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4016 } 4017 4018 /* 4019 * Function: 4020 * ibt_cm_ud_proceed 4021 * 4022 * Verifies the arguments and dispatches the cm state machine processing 4023 * via taskq 4024 */ 4025 ibt_status_t 4026 ibt_cm_ud_proceed(void *session_id, ibt_channel_hdl_t ud_channel, 4027 ibt_cm_status_t status, ibt_redirect_info_t *redirect_infop, 4028 void *priv_data, ibt_priv_data_len_t priv_data_len) 4029 { 4030 ibcm_ud_state_data_t *ud_statep = (ibcm_ud_state_data_t *)session_id; 4031 ibcm_proceed_targs_t *proceed_targs; 4032 ibt_qp_query_attr_t qp_attr; 4033 ibt_status_t retval; 4034 4035 IBTF_DPRINTF_L3(cmlog, "ibt_cm_ud_proceed session_id %p " 4036 "ud_channel %p ", session_id, ud_channel); 4037 4038 IBTF_DPRINTF_L4(cmlog, "ibt_cm_ud_proceed status %x priv_data %p " 4039 "priv_data_len %x", status, priv_data, priv_data_len); 4040 4041 /* validate session_id and status */ 4042 if ((ud_statep == NULL) || (status == IBT_CM_DEFER)) { 4043 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid Args"); 4044 return (IBT_INVALID_PARAM); 4045 } 4046 4047 /* If priv data len specified, then priv_data cannot be NULL */ 4048 if ((priv_data_len > 0) && (priv_data == NULL)) 4049 return (IBT_INVALID_PARAM); 4050 4051 if (priv_data_len > IBT_SIDR_REP_PRIV_DATA_SZ) 4052 return (IBT_INVALID_PARAM); 4053 4054 /* retrieve qpn and qkey from ud channel */ 4055 4056 /* validate event and statep's state */ 4057 4058 if (status == IBT_CM_ACCEPT) { 4059 retval = ibt_query_qp(ud_channel, &qp_attr); 4060 if ((retval != IBT_SUCCESS) || 4061 (qp_attr.qp_info.qp_trans != IBT_UD_SRV)) { 4062 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed: " 4063 "Failed to retrieve QPN from the channel: %d", 4064 retval); 4065 return (IBT_INVALID_PARAM); 4066 } 4067 } 4068 4069 4070 mutex_enter(&ud_statep->ud_state_mutex); 4071 4072 if (ud_statep->ud_state != IBCM_STATE_SIDR_REQ_RCVD) { 4073 mutex_exit(&ud_statep->ud_state_mutex); 4074 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Invalid State " 4075 "specified"); 4076 return (IBT_INVALID_PARAM); 4077 } 4078 4079 /* wait until client's CM handler returns DEFER status back to CM */ 4080 4081 while (ud_statep->ud_clnt_proceed == IBCM_BLOCK) { 4082 IBTF_DPRINTF_L5(cmlog, "ibt_cm_ud_proceed : Blocked for return" 4083 " of client's ud cm handler"); 4084 cv_wait(&ud_statep->ud_block_client_cv, 4085 &ud_statep->ud_state_mutex); 4086 } 4087 4088 if (ud_statep->ud_clnt_proceed == IBCM_FAIL) { 4089 mutex_exit(&ud_statep->ud_state_mutex); 4090 IBTF_DPRINTF_L2(cmlog, "ibt_cm_ud_proceed : Failed as client " 4091 "returned non-DEFER status from cm handler"); 4092 return (IBT_INVALID_PARAM); 4093 } 4094 4095 ASSERT(ud_statep->ud_clnt_proceed == IBCM_UNBLOCK); 4096 ud_statep->ud_clnt_proceed = IBCM_FAIL; 4097 mutex_exit(&ud_statep->ud_state_mutex); 4098 4099 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4100 4101 /* the state machine processing is done in a separate thread */ 4102 4103 /* proceed_targs is freed in ibcm_proceed_via_taskq */ 4104 proceed_targs = kmem_zalloc(sizeof (ibcm_proceed_targs_t), 4105 KM_SLEEP); 4106 4107 proceed_targs->status = status; 4108 proceed_targs->priv_data_len = priv_data_len; 4109 4110 bcopy(priv_data, proceed_targs->priv_data, priv_data_len); 4111 4112 if (status == IBT_CM_ACCEPT) { 4113 proceed_targs->tst.ud.ud_qkey = 4114 qp_attr.qp_info.qp_transport.ud.ud_qkey; 4115 proceed_targs->tst.ud.ud_qpn = qp_attr.qp_qpn; 4116 } 4117 4118 proceed_targs->tst.ud.ud_statep = ud_statep; 4119 4120 /* copy redirect info based on status */ 4121 if (status == IBT_CM_REDIRECT) 4122 bcopy(redirect_infop, &proceed_targs->tst.ud.ud_redirect_info, 4123 sizeof (ibt_redirect_info_t)); 4124 4125 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*proceed_targs)) 4126 4127 (void) taskq_dispatch(ibcm_taskq, ibcm_ud_proceed_via_taskq, 4128 proceed_targs, TQ_SLEEP); 4129 4130 return (IBT_SUCCESS); 4131 } 4132 4133 /* 4134 * Function: 4135 * ibcm_ud_proceed_via_taskq 4136 * 4137 * Called from taskq, dispatched by ibt_cm_ud_proceed 4138 * Completes the cm state processing for ibt_cm_ud_proceed 4139 */ 4140 void 4141 ibcm_ud_proceed_via_taskq(void *targs) 4142 { 4143 ibcm_proceed_targs_t *proceed_targs = (ibcm_proceed_targs_t *)targs; 4144 ibcm_ud_state_data_t *ud_statep = proceed_targs->tst.ud.ud_statep; 4145 ibcm_ud_clnt_reply_info_t ud_clnt_info; 4146 ibt_sidr_status_t sidr_status; 4147 4148 IBTF_DPRINTF_L4(cmlog, "ibcm_ud_proceed_via_taskq(%p)", targs); 4149 4150 ud_clnt_info.ud_qpn = proceed_targs->tst.ud.ud_qpn; 4151 ud_clnt_info.ud_qkey = proceed_targs->tst.ud.ud_qkey; 4152 ud_clnt_info.priv_data = proceed_targs->priv_data; 4153 ud_clnt_info.priv_data_len = proceed_targs->priv_data_len; 4154 ud_clnt_info.redirect_infop = &proceed_targs->tst.ud.ud_redirect_info; 4155 4156 /* validate event and statep's state */ 4157 ibcm_process_sidr_req_cm_hdlr(ud_statep, proceed_targs->status, 4158 &ud_clnt_info, &sidr_status, 4159 (ibcm_sidr_rep_msg_t *)IBCM_OUT_MSGP(ud_statep->ud_stored_msg)); 4160 4161 ibcm_post_sidr_rep_mad(ud_statep, sidr_status); 4162 4163 /* decr the statep ref cnt incremented in ibcm_process_sidr_req_msg */ 4164 mutex_enter(&ud_statep->ud_state_mutex); 4165 IBCM_UD_REF_CNT_DECR(ud_statep); 4166 mutex_exit(&ud_statep->ud_state_mutex); 4167 4168 kmem_free(targs, sizeof (ibcm_proceed_targs_t)); 4169 } 4170 4171 /* 4172 * Function: 4173 * ibt_set_alt_path 4174 * Input: 4175 * channel Channel handle returned from ibt_alloc_rc_channel(9F). 4176 * 4177 * mode Execute in blocking or non blocking mode. 4178 * 4179 * alt_path A pointer to an ibt_alt_path_info_t as returned from an 4180 * ibt_get_alt_path(9F) call that specifies the new 4181 * alternate path. 4182 * 4183 * priv_data A pointer to a buffer specified by caller for the 4184 * private data in the outgoing CM Load Alternate Path 4185 * (LAP) message sent to the remote host. This can be NULL 4186 * if no private data is available to communicate to the 4187 * remote node. 4188 * 4189 * priv_data_len Length of valid data in priv_data, this should be less 4190 * than or equal to IBT_LAP_PRIV_DATA_SZ. 4191 * 4192 * Output: 4193 * ret_args If called in blocking mode, points to a return argument 4194 * structure of type ibt_ap_returns_t. 4195 * 4196 * Returns: 4197 * IBT_SUCCESS on Success else appropriate error. 4198 * Description: 4199 * Load the specified alternate path. Causes the CM to send an LAP message 4200 * to the remote node. 4201 * Can only be called on a previously opened RC channel. 4202 */ 4203 ibt_status_t 4204 ibt_set_alt_path(ibt_channel_hdl_t channel, ibt_execution_mode_t mode, 4205 ibt_alt_path_info_t *alt_path, void *priv_data, 4206 ibt_priv_data_len_t priv_data_len, ibt_ap_returns_t *ret_args) 4207 { 4208 ibmf_handle_t ibmf_hdl; 4209 ibt_status_t status = IBT_SUCCESS; 4210 ibcm_lap_msg_t *lap_msgp; 4211 ibcm_hca_info_t *hcap; 4212 ibcm_state_data_t *statep; 4213 uint8_t port_no; 4214 ib_lid_t alternate_slid; 4215 ibt_priv_data_len_t len; 4216 ib_lid_t base_lid; 4217 boolean_t alt_grh; 4218 4219 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path(%p, %x, %p, %p, %x, %p)", 4220 channel, mode, alt_path, priv_data, priv_data_len, ret_args); 4221 4222 /* validate channel */ 4223 if (IBCM_INVALID_CHANNEL(channel)) { 4224 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: invalid channel"); 4225 return (IBT_CHAN_HDL_INVALID); 4226 } 4227 4228 if (ibtl_cm_get_chan_type(channel) != IBT_RC_SRV) { 4229 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4230 "Invalid Channel type: Applicable only to RC Channel"); 4231 return (IBT_CHAN_SRV_TYPE_INVALID); 4232 } 4233 4234 if (mode == IBT_NONBLOCKING) { 4235 if (ret_args != NULL) { 4236 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4237 "ret_args should be NULL when called in " 4238 "non-blocking mode"); 4239 return (IBT_INVALID_PARAM); 4240 } 4241 } else if (mode == IBT_BLOCKING) { 4242 if (ret_args == NULL) { 4243 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4244 "ret_args should be Non-NULL when called in " 4245 "blocking mode"); 4246 return (IBT_INVALID_PARAM); 4247 } 4248 if (ret_args->ap_priv_data_len > IBT_APR_PRIV_DATA_SZ) { 4249 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4250 "expected private data length is too large"); 4251 return (IBT_INVALID_PARAM); 4252 } 4253 if ((ret_args->ap_priv_data_len > 0) && 4254 (ret_args->ap_priv_data == NULL)) { 4255 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4256 "apr_priv_data_len > 0, but apr_priv_data NULL"); 4257 return (IBT_INVALID_PARAM); 4258 } 4259 } else { /* any other mode is not valid for ibt_set_alt_path */ 4260 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4261 "invalid mode %x specified", mode); 4262 return (IBT_INVALID_PARAM); 4263 } 4264 4265 if ((port_no = alt_path->ap_alt_cep_path.cep_hca_port_num) == 0) 4266 return (IBT_INVALID_PARAM); 4267 4268 /* get the statep */ 4269 IBCM_GET_CHAN_PRIVATE(channel, statep); 4270 if (statep == NULL) { 4271 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: statep NULL"); 4272 return (IBT_CM_FAILURE); 4273 } 4274 4275 mutex_enter(&statep->state_mutex); 4276 IBCM_RELEASE_CHAN_PRIVATE(channel); 4277 IBCM_REF_CNT_INCR(statep); 4278 mutex_exit(&statep->state_mutex); 4279 4280 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: statep %p", statep); 4281 4282 hcap = statep->hcap; 4283 4284 /* HCA must have been in active state. If not, it's a client bug */ 4285 if (!IBCM_ACCESS_HCA_OK(hcap)) 4286 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: hca in error state"); 4287 4288 ASSERT(statep->cm_handler != NULL); 4289 4290 /* Check Alternate port */ 4291 status = ibt_get_port_state_byguid(hcap->hca_guid, port_no, NULL, 4292 &base_lid); 4293 if (status != IBT_SUCCESS) { 4294 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4295 "ibt_get_port_state_byguid status %d ", status); 4296 mutex_enter(&statep->state_mutex); 4297 IBCM_REF_CNT_DECR(statep); 4298 mutex_exit(&statep->state_mutex); 4299 return (status); 4300 } 4301 4302 if ((hcap->hca_port_info[port_no - 1].port_ibmf_hdl == NULL) && 4303 ((status = ibcm_hca_reinit_port(hcap, port_no - 1)) 4304 != IBT_SUCCESS)) { 4305 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4306 "ibmf reg or callback setup failed during re-initialize"); 4307 mutex_enter(&statep->state_mutex); 4308 IBCM_REF_CNT_DECR(statep); 4309 mutex_exit(&statep->state_mutex); 4310 return (status); 4311 } 4312 4313 ibmf_hdl = statep->stored_reply_addr.ibmf_hdl; 4314 4315 alternate_slid = base_lid + 4316 alt_path->ap_alt_cep_path.cep_adds_vect.av_src_path; 4317 4318 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: alternate SLID = %x", 4319 h2b16(alternate_slid)); 4320 4321 ibcm_lapr_enter(); /* limit how many run simultaneously */ 4322 4323 /* Allocate MAD for LAP */ 4324 if (statep->lapr_msg == NULL) 4325 if ((status = ibcm_alloc_out_msg(ibmf_hdl, &statep->lapr_msg, 4326 MAD_METHOD_SEND)) != IBT_SUCCESS) { 4327 ibcm_lapr_exit(); 4328 IBTF_DPRINTF_L2(cmlog, "ibt_set_alt_path: " 4329 "chan 0x%p ibcm_alloc_out_msg failed", channel); 4330 mutex_enter(&statep->state_mutex); 4331 IBCM_REF_CNT_DECR(statep); 4332 mutex_exit(&statep->state_mutex); 4333 return (status); 4334 } 4335 4336 mutex_enter(&statep->state_mutex); 4337 4338 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: connection state is" 4339 " %x", statep->state); 4340 4341 /* Check state */ 4342 if ((statep->state != IBCM_STATE_ESTABLISHED) || 4343 (statep->ap_state != IBCM_AP_STATE_IDLE)) { 4344 IBCM_REF_CNT_DECR(statep); 4345 mutex_exit(&statep->state_mutex); 4346 (void) ibcm_free_out_msg(ibmf_hdl, &statep->lapr_msg); 4347 ibcm_lapr_exit(); 4348 return (IBT_CHAN_STATE_INVALID); 4349 } else { 4350 /* Set to LAP Sent state */ 4351 statep->ap_state = IBCM_AP_STATE_LAP_SENT; 4352 statep->ap_done = B_FALSE; 4353 statep->remaining_retry_cnt = statep->max_cm_retries; 4354 statep->timer_stored_state = statep->state; 4355 statep->timer_stored_ap_state = statep->ap_state; 4356 IBCM_REF_CNT_INCR(statep); /* for ibcm_post_lap_complete */ 4357 } 4358 4359 mutex_exit(&statep->state_mutex); 4360 4361 /* No more failure returns below */ 4362 4363 /* Allocate MAD for LAP */ 4364 IBTF_DPRINTF_L5(cmlog, "ibt_set_alt_path:" 4365 " statep's mad addr = 0x%p", IBCM_OUT_HDRP(statep->lapr_msg)); 4366 4367 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4368 4369 lap_msgp = (ibcm_lap_msg_t *)IBCM_OUT_MSGP(statep->lapr_msg); 4370 4371 lap_msgp->lap_alt_l_port_lid = h2b16(alternate_slid); 4372 lap_msgp->lap_alt_r_port_lid = 4373 h2b16(alt_path->ap_alt_cep_path.cep_adds_vect.av_dlid); 4374 4375 /* Fill in remote port gid */ 4376 lap_msgp->lap_alt_r_port_gid.gid_prefix = 4377 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_prefix); 4378 lap_msgp->lap_alt_r_port_gid.gid_guid = 4379 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_dgid.gid_guid); 4380 4381 /* Fill in local port gid */ 4382 lap_msgp->lap_alt_l_port_gid.gid_prefix = 4383 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_prefix); 4384 lap_msgp->lap_alt_l_port_gid.gid_guid = 4385 h2b64(alt_path->ap_alt_cep_path.cep_adds_vect.av_sgid.gid_guid); 4386 4387 alt_grh = alt_path->ap_alt_cep_path.cep_adds_vect.av_send_grh; 4388 4389 /* alternate_flow_label, and alternate srate, alternate traffic class */ 4390 lap_msgp->lap_alt_srate_plus = 4391 alt_path->ap_alt_cep_path.cep_adds_vect.av_srate & 0x3f; 4392 lap_msgp->lap_alt_flow_label_plus = h2b32(((alt_grh == B_TRUE) ? 4393 (alt_path->ap_alt_cep_path.cep_adds_vect.av_flow << 12) : 0) | 4394 alt_path->ap_alt_cep_path.cep_adds_vect.av_tclass); 4395 4396 /* Alternate hop limit, service level */ 4397 lap_msgp->lap_alt_hop_limit = (alt_grh == B_TRUE) ? 4398 alt_path->ap_alt_cep_path.cep_adds_vect.av_hop : 1; 4399 lap_msgp->lap_alt_sl_plus = 4400 alt_path->ap_alt_cep_path.cep_adds_vect.av_srvl << 4 | 4401 ((alt_grh == B_FALSE) ? 0x8 : 0); 4402 4403 lap_msgp->lap_alt_local_acktime_plus = ibt_usec2ib( 4404 (2 * statep->rc_alt_pkt_lt) + 4405 ibt_ib2usec(hcap->hca_ack_delay)) << 3; 4406 4407 lap_msgp->lap_local_comm_id = h2b32(statep->local_comid); 4408 lap_msgp->lap_remote_comm_id = h2b32(statep->remote_comid); 4409 4410 lap_msgp->lap_remote_qpn_eecn_plus = 4411 h2b32((statep->remote_qpn << 8) | 4412 ibt_usec2ib(ibcm_remote_response_time) << 3); 4413 4414 len = min(priv_data_len, IBT_LAP_PRIV_DATA_SZ); 4415 if ((len > 0) && priv_data) { 4416 bcopy(priv_data, lap_msgp->lap_private_data, len); 4417 } 4418 4419 /* only rc_alt_pkt_lt and ap_return_data fields are initialized */ 4420 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*statep)) 4421 4422 statep->rc_alt_pkt_lt = ibt_ib2usec(alt_path->ap_alt_pkt_lt); 4423 4424 /* return_data is filled up in the state machine code */ 4425 statep->ap_return_data = ret_args; 4426 4427 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep)) 4428 4429 IBCM_OUT_HDRP(statep->lapr_msg)->AttributeID = 4430 h2b16(IBCM_INCOMING_LAP + IBCM_ATTR_BASE_ID); 4431 4432 IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID = 4433 h2b64(ibcm_generate_tranid(IBCM_INCOMING_LAP, statep->local_comid, 4434 0)); 4435 IBTF_DPRINTF_L3(cmlog, "ibt_set_alt_path: statep %p, tid %llx", 4436 statep, IBCM_OUT_HDRP(statep->lapr_msg)->TransactionID); 4437 4438 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*lap_msgp)) 4439 4440 /* Send LAP */ 4441 ibcm_post_rc_mad(statep, statep->lapr_msg, ibcm_post_lap_complete, 4442 statep); 4443 4444 mutex_enter(&statep->state_mutex); 4445 4446 if (mode == IBT_BLOCKING) { 4447 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: blocking"); 4448 4449 /* wait for APR */ 4450 while (statep->ap_done != B_TRUE) { 4451 cv_wait(&statep->block_client_cv, 4452 &statep->state_mutex); 4453 } 4454 4455 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done blocking"); 4456 4457 /* 4458 * In the case that ibt_set_alt_path fails, 4459 * change retval to IBT_CM_FAILURE 4460 */ 4461 if (statep->ap_return_data->ap_status != IBT_CM_AP_LOADED) 4462 status = IBT_CM_FAILURE; 4463 4464 } 4465 4466 /* decrement the ref-count before leaving here */ 4467 IBCM_REF_CNT_DECR(statep); 4468 4469 mutex_exit(&statep->state_mutex); 4470 4471 ibcm_lapr_exit(); 4472 4473 /* If this message isn't seen then ibt_set_alt_path failed */ 4474 IBTF_DPRINTF_L4(cmlog, "ibt_set_alt_path: done"); 4475 4476 return (status); 4477 } 4478 4479 4480 #ifdef DEBUG 4481 4482 /* 4483 * ibcm_query_classport_info: 4484 * Query classportinfo 4485 * 4486 * INPUTS: 4487 * channel - Channel that is associated with a statep 4488 * 4489 * RETURN VALUE: NONE 4490 * This function is currently used to generate a valid get method classport 4491 * info, and test CM functionality. There is no ibtl client interface to 4492 * generate a classportinfo. It is possible that CM may use classportinfo 4493 * from other nodes in the future, and most of the code below could be re-used. 4494 */ 4495 void 4496 ibcm_query_classport_info(ibt_channel_hdl_t channel) 4497 { 4498 ibcm_state_data_t *statep; 4499 ibmf_msg_t *msgp; 4500 4501 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p)", channel); 4502 4503 /* validate channel, first */ 4504 if (IBCM_INVALID_CHANNEL(channel)) { 4505 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4506 "invalid channel (%p)", channel); 4507 return; 4508 } 4509 4510 /* get the statep */ 4511 IBCM_GET_CHAN_PRIVATE(channel, statep); 4512 4513 /* 4514 * This can happen, if the statep is already gone by a DREQ from 4515 * the remote side 4516 */ 4517 if (statep == NULL) { 4518 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info: " 4519 "statep NULL"); 4520 return; 4521 } 4522 4523 mutex_enter(&statep->state_mutex); 4524 IBCM_RELEASE_CHAN_PRIVATE(channel); 4525 IBCM_REF_CNT_INCR(statep); 4526 mutex_exit(&statep->state_mutex); 4527 4528 /* Debug/test code, so don't care about return status */ 4529 (void) ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp, 4530 MAD_METHOD_GET); 4531 4532 IBCM_OUT_HDRP(msgp)->TransactionID = h2b64(ibcm_generate_tranid( 4533 MAD_ATTR_ID_CLASSPORTINFO, statep->local_comid, 0)); 4534 IBCM_OUT_HDRP(msgp)->AttributeID = h2b16(MAD_ATTR_ID_CLASSPORTINFO); 4535 4536 (void) ibcm_post_mad(msgp, &statep->stored_reply_addr, NULL, NULL); 4537 4538 IBTF_DPRINTF_L2(cmlog, "ibcm_query_classport_info(%p) " 4539 "Get method MAD posted ", channel); 4540 4541 (void) ibcm_free_out_msg(statep->stored_reply_addr.ibmf_hdl, &msgp); 4542 4543 mutex_enter(&statep->state_mutex); 4544 IBCM_REF_CNT_DECR(statep); 4545 mutex_exit(&statep->state_mutex); 4546 } 4547 4548 static void 4549 ibcm_print_reply_addr(ibt_channel_hdl_t channel, ibcm_mad_addr_t *cm_reply_addr) 4550 { 4551 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: chan 0x%p, SLID %x, " 4552 "DLID %x", channel, cm_reply_addr->rcvd_addr.ia_local_lid, 4553 cm_reply_addr->rcvd_addr.ia_remote_lid); 4554 4555 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: QKEY %x, PKEY %x, " 4556 "RQPN %x SL %x", cm_reply_addr->rcvd_addr.ia_q_key, 4557 cm_reply_addr->rcvd_addr.ia_p_key, 4558 cm_reply_addr->rcvd_addr.ia_remote_qno, 4559 cm_reply_addr->rcvd_addr.ia_service_level); 4560 4561 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM SGID %llX:%llX ", 4562 cm_reply_addr->grh_hdr.ig_sender_gid.gid_prefix, 4563 cm_reply_addr->grh_hdr.ig_sender_gid.gid_guid); 4564 4565 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM DGID %llX:%llX", 4566 cm_reply_addr->grh_hdr.ig_recver_gid.gid_prefix, 4567 cm_reply_addr->grh_hdr.ig_recver_gid.gid_guid); 4568 4569 IBTF_DPRINTF_L4(cmlog, "ibcm_print_reply_addr: CM FL %x TC %x HL %x", 4570 cm_reply_addr->grh_hdr.ig_flow_label, 4571 cm_reply_addr->grh_hdr.ig_tclass, 4572 cm_reply_addr->grh_hdr.ig_hop_limit); 4573 } 4574 4575 #endif 4576 4577 typedef struct ibcm_join_mcg_tqarg_s { 4578 ib_gid_t rgid; 4579 ibt_mcg_attr_t mcg_attr; 4580 ibt_mcg_info_t *mcg_infop; 4581 ibt_mcg_handler_t func; 4582 void *arg; 4583 } ibcm_join_mcg_tqarg_t; 4584 4585 _NOTE(READ_ONLY_DATA(ibcm_join_mcg_tqarg_s)) 4586 4587 /* 4588 * Function: 4589 * ibt_join_mcg 4590 * Input: 4591 * rgid The request GID that defines the HCA port from which a 4592 * contact to SA Access is performed to add the specified 4593 * endport GID ((mcg_attr->mc_pgid) to a multicast group. 4594 * If mcg_attr->mc_pgid is null, then this (rgid) will be 4595 * treated as endport GID that is to be added to the 4596 * multicast group. 4597 * 4598 * mcg_attr A pointer to an ibt_mcg_attr_t structure that defines 4599 * the attributes of the desired multicast group to be 4600 * created or joined. 4601 * 4602 * func NULL or a pointer to a function to call when 4603 * ibt_join_mcg() completes. If 'func' is not NULL then 4604 * ibt_join_mcg() will return as soon as possible after 4605 * initiating the multicast group join/create process. 4606 * 'func' is then called when the process completes. 4607 * 4608 * arg Argument to the 'func'. 4609 * 4610 * Output: 4611 * mcg_info_p A pointer to the ibt_mcg_info_t structure, allocated 4612 * by the caller, where the attributes of the created or 4613 * joined multicast group are copied. 4614 * Returns: 4615 * IBT_SUCCESS 4616 * IBT_INVALID_PARAM 4617 * IBT_MCG_RECORDS_NOT_FOUND 4618 * IBT_INSUFF_RESOURCE 4619 * Description: 4620 * Join a multicast group. The first full member "join" causes the MCG 4621 * to be created. 4622 */ 4623 ibt_status_t 4624 ibt_join_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, 4625 ibt_mcg_info_t *mcg_info_p, ibt_mcg_handler_t func, void *arg) 4626 { 4627 ibcm_join_mcg_tqarg_t *mcg_tq; 4628 int flag = ((func == NULL) ? KM_SLEEP : KM_NOSLEEP); 4629 4630 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg(%llX:%llX, %p)", rgid.gid_prefix, 4631 rgid.gid_guid, mcg_attr); 4632 4633 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4634 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Request GID is required"); 4635 return (IBT_INVALID_PARAM); 4636 } 4637 4638 if ((mcg_attr->mc_pkey == IB_PKEY_INVALID_LIMITED) || 4639 (mcg_attr->mc_pkey == IB_PKEY_INVALID_FULL)) { 4640 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Invalid P_Key specified"); 4641 return (IBT_INVALID_PARAM); 4642 } 4643 4644 if (mcg_attr->mc_join_state == 0) { 4645 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: JoinState not specified"); 4646 return (IBT_INVALID_PARAM); 4647 } 4648 4649 if (mcg_info_p == NULL) { 4650 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: mcg_info_p is NULL"); 4651 return (IBT_INVALID_PARAM); 4652 } 4653 4654 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4655 4656 mcg_tq = kmem_alloc(sizeof (ibcm_join_mcg_tqarg_t), flag); 4657 if (mcg_tq == NULL) { 4658 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: " 4659 "Unable to allocate memory for local usage."); 4660 return (IBT_INSUFF_KERNEL_RESOURCE); 4661 } 4662 4663 mcg_tq->rgid = rgid; 4664 bcopy(mcg_attr, &mcg_tq->mcg_attr, sizeof (ibt_mcg_attr_t)); 4665 mcg_tq->mcg_infop = mcg_info_p; 4666 mcg_tq->func = func; 4667 mcg_tq->arg = arg; 4668 4669 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*mcg_tq)) 4670 4671 if (func != NULL) { /* Non-Blocking */ 4672 IBTF_DPRINTF_L3(cmlog, "ibt_join_mcg: Non-Blocking Call"); 4673 if (taskq_dispatch(ibcm_taskq, ibcm_process_async_join_mcg, 4674 mcg_tq, TQ_NOSLEEP) == 0) { 4675 IBTF_DPRINTF_L2(cmlog, "ibt_join_mcg: Failed to " 4676 "Dispatch the TaskQ"); 4677 kmem_free(mcg_tq, sizeof (ibcm_join_mcg_tqarg_t)); 4678 return (IBT_INSUFF_KERNEL_RESOURCE); 4679 } else 4680 return (IBT_SUCCESS); 4681 } else { /* Blocking */ 4682 return (ibcm_process_join_mcg(mcg_tq)); 4683 } 4684 } 4685 4686 static void 4687 ibcm_process_async_join_mcg(void *tq_arg) 4688 { 4689 (void) ibcm_process_join_mcg(tq_arg); 4690 } 4691 4692 static ibt_status_t 4693 ibcm_process_join_mcg(void *taskq_arg) 4694 { 4695 sa_mcmember_record_t mcg_req; 4696 sa_mcmember_record_t *mcg_resp; 4697 ibmf_saa_access_args_t access_args; 4698 ibmf_saa_handle_t saa_handle; 4699 uint64_t component_mask = 0; 4700 ibt_status_t retval; 4701 ibtl_cm_hca_port_t hca_port; 4702 uint_t num_records; 4703 size_t length; 4704 ibcm_hca_info_t *hcap; 4705 ibcm_join_mcg_tqarg_t *mcg_arg = (ibcm_join_mcg_tqarg_t *)taskq_arg; 4706 ibt_mcg_info_t *mcg_info_p = mcg_arg->mcg_infop; 4707 4708 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg(%p)", mcg_arg); 4709 4710 retval = ibtl_cm_get_hca_port(mcg_arg->rgid, 0, &hca_port); 4711 if (retval != IBT_SUCCESS) { 4712 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed to get " 4713 "port info from specified RGID: status = %d", retval); 4714 goto ibcm_join_mcg_exit1; 4715 } 4716 4717 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4718 4719 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix == 0) || 4720 (mcg_arg->mcg_attr.mc_pgid.gid_guid == 0)) { 4721 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4722 "Request GID is Port GID"); 4723 mcg_req.PortGID = mcg_arg->rgid; 4724 } else { 4725 mcg_req.PortGID = mcg_arg->mcg_attr.mc_pgid; 4726 } 4727 component_mask |= SA_MC_COMPMASK_PORTGID; 4728 4729 mcg_req.Q_Key = mcg_arg->mcg_attr.mc_qkey; 4730 mcg_req.P_Key = mcg_arg->mcg_attr.mc_pkey; 4731 mcg_req.JoinState = mcg_arg->mcg_attr.mc_join_state; 4732 mcg_req.TClass = mcg_arg->mcg_attr.mc_tclass; 4733 mcg_req.FlowLabel = mcg_arg->mcg_attr.mc_flow; 4734 mcg_req.SL = mcg_arg->mcg_attr.mc_sl; 4735 4736 component_mask |= SA_MC_COMPMASK_QKEY | SA_MC_COMPMASK_PKEY | 4737 SA_MC_COMPMASK_JOINSTATE | SA_MC_COMPMASK_TCLASS | 4738 SA_MC_COMPMASK_FLOWLABEL | SA_MC_COMPMASK_SL; 4739 4740 /* If client has specified MGID, use it else SA will assign one. */ 4741 if ((mcg_arg->mcg_attr.mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4742 mcg_req.MGID = mcg_arg->mcg_attr.mc_mgid; 4743 component_mask |= SA_MC_COMPMASK_MGID; 4744 } 4745 4746 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: PGID=%llX:%llX, " 4747 "MGID=%llX:%llX", mcg_req.PortGID.gid_prefix, 4748 mcg_req.PortGID.gid_guid, mcg_req.MGID.gid_prefix, 4749 mcg_req.MGID.gid_guid); 4750 4751 /* Is MTU specified. */ 4752 if (mcg_arg->mcg_attr.mc_mtu_req.r_mtu) { 4753 mcg_req.MTU = mcg_arg->mcg_attr.mc_mtu_req.r_mtu; 4754 mcg_req.MTUSelector = mcg_arg->mcg_attr.mc_mtu_req.r_selector; 4755 4756 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 4757 SA_MC_COMPMASK_MTU; 4758 } 4759 4760 /* Is RATE specified. */ 4761 if (mcg_arg->mcg_attr.mc_rate_req.r_srate) { 4762 mcg_req.Rate = mcg_arg->mcg_attr.mc_rate_req.r_srate; 4763 mcg_req.RateSelector = 4764 mcg_arg->mcg_attr.mc_rate_req.r_selector; 4765 4766 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 4767 SA_MC_COMPMASK_RATE; 4768 } 4769 4770 /* Is Packet Life Time specified. */ 4771 if (mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt) { 4772 mcg_req.Rate = mcg_arg->mcg_attr.mc_pkt_lt_req.p_pkt_lt; 4773 mcg_req.RateSelector = 4774 mcg_arg->mcg_attr.mc_pkt_lt_req.p_selector; 4775 4776 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 4777 SA_MC_COMPMASK_PKTLT; 4778 } 4779 4780 if (mcg_arg->mcg_attr.mc_hop) { 4781 mcg_req.HopLimit = mcg_arg->mcg_attr.mc_hop; 4782 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 4783 } 4784 4785 if (mcg_arg->mcg_attr.mc_scope) { 4786 mcg_req.Scope = mcg_arg->mcg_attr.mc_scope; 4787 component_mask |= SA_MC_COMPMASK_SCOPE; 4788 } 4789 4790 if (mcg_arg->mcg_attr.mc_mlid) { 4791 mcg_req.MLID = mcg_arg->mcg_attr.mc_mlid; 4792 component_mask |= SA_MC_COMPMASK_MLID; 4793 } 4794 4795 /* Get SA Access Handle. */ 4796 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 4797 if (hcap == NULL) { 4798 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: NO HCA found"); 4799 4800 retval = IBT_HCA_BUSY_DETACHING; 4801 goto ibcm_join_mcg_exit1; 4802 } 4803 4804 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 4805 if (saa_handle == NULL) { 4806 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: SA Handle NULL"); 4807 4808 retval = IBT_HCA_PORT_NOT_ACTIVE; 4809 goto ibcm_join_mcg_exit; 4810 } 4811 4812 if ((mcg_arg->mcg_attr.mc_pgid.gid_prefix != 0) && 4813 (mcg_arg->mcg_attr.mc_pgid.gid_guid != 0)) { 4814 retval = ibtl_cm_get_hca_port(mcg_arg->mcg_attr.mc_pgid, 0, 4815 &hca_port); 4816 if (retval != IBT_SUCCESS) { 4817 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: Failed " 4818 "to get PortInfo of specified PGID: status = %d", 4819 retval); 4820 goto ibcm_join_mcg_exit1; 4821 } 4822 } 4823 4824 /* Contact SA Access */ 4825 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 4826 access_args.sq_access_type = IBMF_SAA_UPDATE; 4827 access_args.sq_component_mask = component_mask; 4828 access_args.sq_template = &mcg_req; 4829 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 4830 access_args.sq_callback = NULL; 4831 access_args.sq_callback_arg = NULL; 4832 4833 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 4834 (void **)&mcg_resp); 4835 if (retval != IBT_SUCCESS) { 4836 IBTF_DPRINTF_L2(cmlog, "ibcm_process_join_mcg: " 4837 "SA Access Failed"); 4838 goto ibcm_join_mcg_exit; 4839 } 4840 4841 num_records = length/sizeof (sa_mcmember_record_t); 4842 4843 IBTF_DPRINTF_L4(cmlog, "ibcm_process_join_mcg: " 4844 "Found %d MCMember Records", num_records); 4845 4846 /* Validate the returned number of records. */ 4847 if ((mcg_resp != NULL) && (num_records > 0)) { 4848 /* Update the return values. */ 4849 mcg_info_p->mc_adds_vect.av_dgid = mcg_resp->MGID; 4850 mcg_info_p->mc_adds_vect.av_sgid = mcg_resp->PortGID; 4851 mcg_info_p->mc_adds_vect.av_srate = mcg_resp->Rate; 4852 mcg_info_p->mc_adds_vect.av_srvl = mcg_resp->SL; 4853 mcg_info_p->mc_adds_vect.av_flow = mcg_resp->FlowLabel; 4854 mcg_info_p->mc_adds_vect.av_tclass = mcg_resp->TClass; 4855 mcg_info_p->mc_adds_vect.av_hop = mcg_resp->HopLimit; 4856 mcg_info_p->mc_adds_vect.av_send_grh = B_TRUE; 4857 mcg_info_p->mc_adds_vect.av_dlid = mcg_resp->MLID; 4858 mcg_info_p->mc_mtu = mcg_resp->MTU; 4859 mcg_info_p->mc_qkey = mcg_resp->Q_Key; 4860 4861 retval = ibt_pkey2index_byguid(hca_port.hp_hca_guid, 4862 hca_port.hp_port, mcg_resp->P_Key, &mcg_info_p->mc_pkey_ix); 4863 if (retval != IBT_SUCCESS) { 4864 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4865 "Pkey2Index Conversion failed<%d>", retval); 4866 mcg_info_p->mc_pkey_ix = 0; 4867 } 4868 4869 mcg_info_p->mc_scope = mcg_resp->Scope; 4870 mcg_info_p->mc_pkt_lt = mcg_resp->PacketLifeTime; 4871 4872 mcg_info_p->mc_adds_vect.av_port_num = hca_port.hp_port; 4873 mcg_info_p->mc_adds_vect.av_sgid_ix = hca_port.hp_sgid_ix; 4874 mcg_info_p->mc_adds_vect.av_src_path = 0; 4875 4876 /* Deallocate the memory allocated by SA for mcg_resp. */ 4877 kmem_free(mcg_resp, length); 4878 retval = IBT_SUCCESS; 4879 } else { 4880 retval = IBT_MCG_RECORDS_NOT_FOUND; 4881 IBTF_DPRINTF_L3(cmlog, "ibcm_process_join_mcg: " 4882 "MCG RECORDS NOT FOUND"); 4883 } 4884 4885 ibcm_join_mcg_exit: 4886 ibcm_dec_hca_acc_cnt(hcap); 4887 4888 ibcm_join_mcg_exit1: 4889 if (mcg_arg->func) 4890 (*(mcg_arg->func))(mcg_arg->arg, retval, mcg_info_p); 4891 4892 kmem_free(mcg_arg, sizeof (ibcm_join_mcg_tqarg_t)); 4893 4894 return (retval); 4895 } 4896 4897 4898 /* 4899 * Function: 4900 * ibt_leave_mcg 4901 * Input: 4902 * rgid The request GID that defines the HCA port upon which 4903 * to send the request to the Subnet Administrator, to 4904 * remove the specified port (port_gid) from the multicast 4905 * group. If 'port_gid' is the Reserved GID (i.e. 4906 * port_gid.gid_prefix = 0 and port_gid.gid_guid = 0), 4907 * then the end-port associated with 'rgid' is removed 4908 * from the multicast group. 4909 * 4910 * mc_gid A multicast group GID as returned from ibt_join_mcg() 4911 * call. This is optional, if not specified (i.e. 4912 * mc_gid.gid_prefix has 0xFF in its upper 8 bits to 4913 * identify this as being a multicast GID), then the 4914 * port is removed from all the multicast groups of 4915 * which it is a member. 4916 * 4917 * port_gid This is optional, if not the Reserved GID (gid_prefix 4918 * and gid_guid not equal to 0), then this specifies the 4919 * endport GID of the multicast group member being deleted 4920 * from the group. If it is the Reserved GID (gid_prefix 4921 * and gid_guid equal to 0) then the member endport GID is 4922 * determined from 'rgid'. 4923 * 4924 * mc_join_state The Join State attribute used when the group was joined 4925 * using ibt_join_mcg(). This Join State component must 4926 * contains at least one bit set to 1 in the same position 4927 * as that used during ibt_join_mcg(). i.e. the logical 4928 * AND of the two JoinState components is not all zeros. 4929 * This Join State component must not have some bits set 4930 * which are not set using ibt_join_mcg(). 4931 * Output: 4932 * None. 4933 * Returns: 4934 * IBT_SUCCESS 4935 * IBT_INVALID_PARAM 4936 * IBT_MC_GROUP_INVALID 4937 * IBT_INSUFF_RESOURCE 4938 * Description: 4939 * The port associated with the port GID shall be removed from the 4940 * multicast group specified by MGID (mc_gid) or from all the multicast 4941 * groups of which it is a member if the MGID (mc_gid) is not specified. 4942 * 4943 * The last full member to leave causes the destruction of the Multicast 4944 * Group. 4945 */ 4946 ibt_status_t 4947 ibt_leave_mcg(ib_gid_t rgid, ib_gid_t mc_gid, ib_gid_t port_gid, 4948 uint8_t mc_join_state) 4949 { 4950 sa_mcmember_record_t mcg_req; 4951 ibmf_saa_access_args_t access_args; 4952 ibmf_saa_handle_t saa_handle; 4953 uint64_t component_mask = 0; 4954 int sa_retval; 4955 ibt_status_t retval; 4956 ibtl_cm_hca_port_t hca_port; 4957 size_t length; 4958 void *results_p; 4959 ibcm_hca_info_t *hcap; 4960 4961 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, %llX:%llX)", 4962 rgid.gid_prefix, rgid.gid_guid, mc_gid.gid_prefix, mc_gid.gid_guid); 4963 4964 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg(%llX:%llX, 0x%X)", 4965 port_gid.gid_prefix, port_gid.gid_guid, mc_join_state); 4966 4967 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 4968 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: RequestGID is required"); 4969 return (IBT_INVALID_PARAM); 4970 } 4971 4972 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 4973 4974 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: MGID: %llX%llX", 4975 mc_gid.gid_prefix, mc_gid.gid_guid); 4976 4977 /* Validate MGID */ 4978 if ((mc_gid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 4979 mcg_req.MGID = mc_gid; 4980 component_mask |= SA_MC_COMPMASK_MGID; 4981 } else if ((mc_gid.gid_prefix != 0) || (mc_gid.gid_guid != 0)) { 4982 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Invalid MGID specified"); 4983 return (IBT_MC_MGID_INVALID); 4984 } 4985 4986 if ((port_gid.gid_prefix == 0) || (port_gid.gid_guid == 0)) { 4987 mcg_req.PortGID = rgid; 4988 } else { 4989 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Performing PROXY Leave"); 4990 mcg_req.PortGID = port_gid; 4991 } 4992 component_mask |= SA_MC_COMPMASK_PORTGID; 4993 4994 IBTF_DPRINTF_L3(cmlog, "ibt_leave_mcg: Port GID <%llX:%llX>", 4995 mcg_req.PortGID.gid_prefix, mcg_req.PortGID.gid_guid); 4996 4997 /* Join State */ 4998 mcg_req.JoinState = mc_join_state; 4999 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5000 5001 retval = ibtl_cm_get_hca_port(rgid, 0, &hca_port); 5002 if (retval != IBT_SUCCESS) { 5003 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: Failed to get port info " 5004 "from specified RGID : status = %d", retval); 5005 return (retval); 5006 } 5007 5008 /* Get SA Access Handle. */ 5009 hcap = ibcm_find_hca_entry(hca_port.hp_hca_guid); 5010 if (hcap == NULL) { 5011 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: " 5012 "NO HCA found"); 5013 return (IBT_HCA_BUSY_DETACHING); 5014 } 5015 5016 saa_handle = ibcm_get_saa_handle(hcap, hca_port.hp_port); 5017 if (saa_handle == NULL) { 5018 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: saa_handle is NULL"); 5019 ibcm_dec_hca_acc_cnt(hcap); 5020 return (IBT_HCA_PORT_NOT_ACTIVE); 5021 } 5022 5023 /* Contact SA Access */ 5024 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5025 access_args.sq_access_type = IBMF_SAA_DELETE; 5026 access_args.sq_component_mask = component_mask; 5027 access_args.sq_template = &mcg_req; 5028 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5029 access_args.sq_callback = NULL; 5030 access_args.sq_callback_arg = NULL; 5031 5032 ibcm_sa_access_enter(); 5033 5034 sa_retval = ibmf_sa_access(saa_handle, &access_args, 0, &length, 5035 &results_p); 5036 if (sa_retval != IBMF_SUCCESS) { 5037 IBTF_DPRINTF_L2(cmlog, "ibt_leave_mcg: SA access Failed: %d", 5038 sa_retval); 5039 (void) ibcm_ibmf_analyze_error(sa_retval); 5040 retval = IBT_MC_GROUP_INVALID; 5041 } 5042 5043 ibcm_sa_access_exit(); 5044 5045 ibcm_dec_hca_acc_cnt(hcap); 5046 5047 return (retval); 5048 } 5049 5050 5051 /* 5052 * Function: 5053 * ibt_query_mcg 5054 * Input: 5055 * rgid The request GID that defines the HCA port upon which 5056 * to send the request to the Subnet Administrator, to 5057 * retrieve Multicast Records matching attributes as 5058 * specified through 'mcg_attr' argument. 5059 * 5060 * mcg_attr NULL or a pointer to an ibt_mcg_attr_t structure that 5061 * specifies MCG attributes that are to be matched. 5062 * Attributes that are not required can be wild carded 5063 * by specifying as '0'. 5064 * 5065 * mcgs_max_num The maximum number of matching multicast groups to 5066 * return. If zero, then all available matching multicast 5067 * groups are returned. 5068 * Output: 5069 * mcgs_info_p The address of an ibt_mcg_info_t pointer, where 5070 * multicast group information is returned. The actual 5071 * number of entries filled in the array is returned in 5072 * entries_p. 5073 * 5074 * entries_p The number of ibt_mcg_attr_t entries returned. 5075 * Returns: 5076 * IBT_SUCCESS 5077 * IBT_INVALID_PARAM 5078 * IBT_MCG_RECORDS_NOT_FOUND 5079 * Description: 5080 * Request information on multicast groups that match the parameters 5081 * specified in mcg_attr. Information on each multicast group is returned 5082 * to the caller in the form of an array of ibt_mcg_info_t. 5083 * ibt_query_mcg() allocates the memory for this array and returns a 5084 * pointer to the array (mcgs_p) and the number of entries in the array 5085 * (entries_p). This memory should be freed by the client using 5086 * ibt_free_mcg_info(). 5087 */ 5088 ibt_status_t 5089 ibt_query_mcg(ib_gid_t rgid, ibt_mcg_attr_t *mcg_attr, uint_t mcgs_max_num, 5090 ibt_mcg_info_t **mcgs_info_p, uint_t *entries_p) 5091 { 5092 sa_mcmember_record_t mcg_req; 5093 sa_mcmember_record_t *mcg_resp; 5094 ibt_mcg_info_t *mcg_infop; 5095 ibmf_saa_access_args_t access_args; 5096 ibmf_saa_handle_t saa_handle; 5097 uint64_t component_mask = 0; 5098 ibt_status_t retval; 5099 ibtl_cm_hca_port_t hport; 5100 uint_t num_records; 5101 size_t length; 5102 void *results_p; 5103 ib_gid_t port_gid; 5104 ibcm_hca_info_t *hcap; 5105 5106 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg(%p, %d)", mcg_attr, mcgs_max_num); 5107 5108 if ((entries_p == NULL) || (mcgs_info_p == NULL)) { 5109 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5110 "entries_p or mcgs_info_p is NULL"); 5111 return (IBT_INVALID_PARAM); 5112 } 5113 5114 if ((rgid.gid_prefix == 0) || (rgid.gid_guid == 0)) { 5115 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: RequestGID is required"); 5116 return (IBT_INVALID_PARAM); 5117 } 5118 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Request GID <%llX:%llX>", 5119 rgid.gid_prefix, rgid.gid_guid); 5120 5121 bzero(&mcg_req, sizeof (sa_mcmember_record_t)); 5122 port_gid.gid_prefix = port_gid.gid_guid = 0; 5123 5124 if (mcg_attr != NULL) { 5125 port_gid = mcg_attr->mc_pgid; 5126 5127 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5128 mcg_req.PortGID = mcg_attr->mc_pgid; 5129 component_mask |= SA_MC_COMPMASK_PORTGID; 5130 5131 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: PGID %llX:%llX", 5132 port_gid.gid_prefix, port_gid.gid_guid); 5133 } 5134 5135 /* Is Q_Key specified. */ 5136 if (mcg_attr->mc_qkey != 0) { 5137 mcg_req.Q_Key = mcg_attr->mc_qkey; 5138 component_mask |= SA_MC_COMPMASK_QKEY; 5139 } 5140 5141 /* Is P_Key specified. */ 5142 if (mcg_attr->mc_pkey != 0) { 5143 mcg_req.P_Key = mcg_attr->mc_pkey; 5144 component_mask |= SA_MC_COMPMASK_PKEY; 5145 } 5146 5147 /* Is MGID specified. */ 5148 if ((mcg_attr->mc_mgid.gid_prefix >> 56ULL & 0xFF) == 0xFF) { 5149 mcg_req.MGID = mcg_attr->mc_mgid; 5150 component_mask |= SA_MC_COMPMASK_MGID; 5151 } 5152 5153 /* Is MTU specified. */ 5154 if (mcg_attr->mc_mtu_req.r_mtu) { 5155 mcg_req.MTU = mcg_attr->mc_mtu_req.r_mtu; 5156 mcg_req.MTUSelector = mcg_attr->mc_mtu_req.r_selector; 5157 5158 component_mask |= SA_MC_COMPMASK_MTUSELECTOR | 5159 SA_MC_COMPMASK_MTU; 5160 } 5161 5162 if (mcg_attr->mc_tclass) { 5163 mcg_req.TClass = mcg_attr->mc_tclass; 5164 component_mask |= SA_MC_COMPMASK_TCLASS; 5165 } 5166 5167 /* Is RATE specified. */ 5168 if (mcg_attr->mc_rate_req.r_srate) { 5169 mcg_req.Rate = mcg_attr->mc_rate_req.r_srate; 5170 mcg_req.RateSelector = mcg_attr->mc_rate_req.r_selector; 5171 5172 component_mask |= SA_MC_COMPMASK_RATESELECTOR | 5173 SA_MC_COMPMASK_RATE; 5174 } 5175 5176 /* Is Packet Life Time specified. */ 5177 if (mcg_attr->mc_pkt_lt_req.p_pkt_lt) { 5178 mcg_req.Rate = mcg_attr->mc_pkt_lt_req.p_pkt_lt; 5179 mcg_req.RateSelector = 5180 mcg_attr->mc_pkt_lt_req.p_selector; 5181 5182 component_mask |= SA_MC_COMPMASK_PKTLTSELECTOR | 5183 SA_MC_COMPMASK_PKTLT; 5184 } 5185 5186 if (mcg_attr->mc_hop) { 5187 mcg_req.HopLimit = mcg_attr->mc_hop; 5188 component_mask |= SA_MC_COMPMASK_HOPLIMIT; 5189 } 5190 5191 if (mcg_attr->mc_flow) { 5192 mcg_req.FlowLabel = mcg_attr->mc_flow; 5193 component_mask |= SA_MC_COMPMASK_FLOWLABEL; 5194 } 5195 5196 if (mcg_attr->mc_sl) { 5197 mcg_req.SL = mcg_attr->mc_sl; 5198 component_mask |= SA_MC_COMPMASK_SL; 5199 } 5200 5201 if (mcg_attr->mc_scope) { 5202 mcg_req.Scope = mcg_attr->mc_scope; 5203 component_mask |= SA_MC_COMPMASK_SCOPE; 5204 } 5205 5206 if (mcg_attr->mc_join_state) { 5207 mcg_req.JoinState = mcg_attr->mc_join_state; 5208 component_mask |= SA_MC_COMPMASK_JOINSTATE; 5209 } 5210 5211 if (mcg_attr->mc_mlid) { 5212 mcg_req.MLID = mcg_attr->mc_mlid; 5213 component_mask |= SA_MC_COMPMASK_MLID; 5214 } 5215 } 5216 5217 retval = ibtl_cm_get_hca_port(rgid, 0, &hport); 5218 if (retval != IBT_SUCCESS) { 5219 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: Failed to get port info " 5220 "from specified RGID : status = %d", retval); 5221 return (retval); 5222 } 5223 5224 /* Get SA Access Handle. */ 5225 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5226 if (hcap == NULL) { 5227 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: NO HCA found"); 5228 return (IBT_HCA_BUSY_DETACHING); 5229 } 5230 5231 saa_handle = ibcm_get_saa_handle(hcap, hport.hp_port); 5232 if (saa_handle == NULL) { 5233 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: saa_handle is NULL"); 5234 ibcm_dec_hca_acc_cnt(hcap); 5235 return (IBT_HCA_PORT_NOT_ACTIVE); 5236 } 5237 5238 /* Contact SA Access */ 5239 access_args.sq_attr_id = SA_MCMEMBERRECORD_ATTRID; 5240 access_args.sq_access_type = IBMF_SAA_RETRIEVE; 5241 access_args.sq_component_mask = component_mask; 5242 access_args.sq_template = &mcg_req; 5243 access_args.sq_template_length = sizeof (sa_mcmember_record_t); 5244 access_args.sq_callback = NULL; 5245 access_args.sq_callback_arg = NULL; 5246 5247 retval = ibcm_contact_sa_access(saa_handle, &access_args, &length, 5248 &results_p); 5249 if (retval != IBT_SUCCESS) { 5250 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: SA access Failed"); 5251 ibcm_dec_hca_acc_cnt(hcap); 5252 return (retval); 5253 } 5254 5255 num_records = length/sizeof (sa_mcmember_record_t); 5256 5257 IBTF_DPRINTF_L4(cmlog, "ibt_query_mcg: Found %d MCMember Records", 5258 num_records); 5259 5260 /* Validate the returned number of records. */ 5261 if ((results_p != NULL) && (num_records > 0)) { 5262 uint_t i; 5263 5264 /* 5265 * If mcgs_max_num is zero, then return all records else 5266 * return only requested number of records 5267 */ 5268 if ((mcgs_max_num != 0) && (num_records > mcgs_max_num)) { 5269 /* we are interested in only mcgs_max_num records */ 5270 num_records = mcgs_max_num; 5271 } 5272 5273 /* 5274 * The SGID returned in "mcg_info_p" buffer should be PortGID, 5275 * (mcg_attr->mc_pgid), if 'mcg_attr->mc_pgid' was specified, 5276 * else RequestGID (rgid) should be returned. 5277 */ 5278 if ((port_gid.gid_prefix != 0) && (port_gid.gid_guid != 0)) { 5279 5280 /* Get sgid_ix and port number of 'port_gid' */ 5281 retval = ibtl_cm_get_hca_port(port_gid, 0, &hport); 5282 if (retval != IBT_SUCCESS) { 5283 IBTF_DPRINTF_L2(cmlog, "ibt_query_mcg: " 5284 "Failed to Get Portinfo for PortGID :" 5285 "status = %d", retval); 5286 return (retval); 5287 } 5288 } else { 5289 /* 5290 * The sgid_ix and port number related to RequestGID 5291 * are already obtained at the beginning. 5292 */ 5293 port_gid = rgid; 5294 } 5295 5296 /* 5297 * Allocate memory for return buffer, to be freed in 5298 * ibt_free_mcg_info(). 5299 */ 5300 mcg_infop = kmem_alloc((num_records * sizeof (ibt_mcg_info_t)), 5301 KM_SLEEP); 5302 5303 *mcgs_info_p = mcg_infop; 5304 *entries_p = num_records; 5305 5306 /* Update the return values. */ 5307 for (i = 0; i < num_records; i++) { 5308 5309 mcg_resp = (sa_mcmember_record_t *)((uchar_t *) 5310 results_p + i * sizeof (sa_mcmember_record_t)); 5311 5312 mcg_infop[i].mc_adds_vect.av_dgid = mcg_resp->MGID; 5313 mcg_infop[i].mc_adds_vect.av_sgid = port_gid; 5314 mcg_infop[i].mc_adds_vect.av_srate = mcg_resp->Rate; 5315 mcg_infop[i].mc_adds_vect.av_srvl = mcg_resp->SL; 5316 mcg_infop[i].mc_adds_vect.av_flow = mcg_resp->FlowLabel; 5317 mcg_infop[i].mc_adds_vect.av_tclass = mcg_resp->TClass; 5318 mcg_infop[i].mc_adds_vect.av_hop = mcg_resp->HopLimit; 5319 mcg_infop[i].mc_adds_vect.av_port_num = hport.hp_port; 5320 mcg_infop[i].mc_adds_vect.av_send_grh = B_TRUE; 5321 mcg_infop[i].mc_adds_vect.av_dlid = mcg_resp->MLID; 5322 mcg_infop[i].mc_adds_vect.av_sgid_ix = hport.hp_sgid_ix; 5323 mcg_infop[i].mc_adds_vect.av_src_path = 0; 5324 mcg_infop[i].mc_mtu = mcg_resp->MTU; 5325 mcg_infop[i].mc_qkey = mcg_resp->Q_Key; 5326 mcg_infop[i].mc_scope = mcg_resp->Scope; 5327 mcg_infop[i].mc_pkt_lt = mcg_resp->PacketLifeTime; 5328 5329 if (ibt_pkey2index_byguid(hport.hp_hca_guid, 5330 hport.hp_port, mcg_resp->P_Key, 5331 &mcg_infop[i].mc_pkey_ix) != IBT_SUCCESS) { 5332 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: " 5333 "Pkey2Index Conversion failed"); 5334 mcg_infop[i].mc_pkey_ix = 0; 5335 } 5336 } 5337 5338 /* 5339 * Deallocate the memory allocated by SA for results_p. 5340 */ 5341 kmem_free(results_p, length); 5342 retval = IBT_SUCCESS; 5343 5344 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: returning %d MCGRecords", 5345 num_records); 5346 5347 } else { 5348 retval = IBT_MCG_RECORDS_NOT_FOUND; 5349 *entries_p = 0; 5350 5351 IBTF_DPRINTF_L3(cmlog, "ibt_query_mcg: MCG RECORDS NOT FOUND"); 5352 } 5353 5354 ibcm_dec_hca_acc_cnt(hcap); 5355 5356 return (retval); 5357 } 5358 5359 5360 /* 5361 * ibt_free_mcg_info() 5362 * Free the memory allocated by successful ibt_query_mcg() 5363 * 5364 * mcgs_info Pointer returned by ibt_query_mcg(). 5365 * 5366 * entries The number of ibt_mcg_info_t entries to free. 5367 */ 5368 void 5369 ibt_free_mcg_info(ibt_mcg_info_t *mcgs_info, uint_t entries) 5370 { 5371 IBTF_DPRINTF_L3(cmlog, "ibt_free_mcg_info: " 5372 "Free <%d> entries from 0x%p", entries, mcgs_info); 5373 5374 if ((mcgs_info != NULL) && (entries > 0)) 5375 kmem_free(mcgs_info, entries * sizeof (ibt_mcg_info_t)); 5376 else 5377 IBTF_DPRINTF_L2(cmlog, "ibt_free_mcg_info: " 5378 "ERROR: NULL buf pointer or length specified."); 5379 } 5380 5381 5382 /* 5383 * Function: 5384 * ibt_gid_to_node_info() 5385 * Input: 5386 * gid Identifies the IB Node and port for which to obtain 5387 * Node information. 5388 * Output: 5389 * node_info_p A pointer to an ibt_node_info_t structure (allocated 5390 * by the caller) in which to return the node information. 5391 * Returns: 5392 * IBT_SUCCESS 5393 * IBT_INVALID_PARAM 5394 * IBT_NODE_RECORDS_NOT_FOUND 5395 * IBT_NO_HCAS_AVAILABLE 5396 * Description: 5397 * Retrieve Node Information for the specified GID. 5398 */ 5399 ibt_status_t 5400 ibt_gid_to_node_info(ib_gid_t gid, ibt_node_info_t *node_info_p) 5401 { 5402 sa_node_record_t nr_req, *nr_resp; 5403 ibmf_saa_handle_t saa_handle; 5404 ibt_status_t retval; 5405 ibcm_hca_info_t *hcap; 5406 ibtl_cm_hca_port_t hport; 5407 int i, j; 5408 uint_t num_rec; 5409 ib_guid_t *guid_array = NULL; 5410 sa_path_record_t *path; 5411 size_t len; 5412 uint8_t npaths; 5413 uint32_t num_hcas = 0; 5414 ib_lid_t node_lid; 5415 boolean_t local_node = B_FALSE; 5416 void *res_p; 5417 uint8_t num_ports = 0; 5418 5419 5420 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info(%llX:%llX, %p)", 5421 gid.gid_prefix, gid.gid_guid, node_info_p); 5422 5423 if ((gid.gid_prefix == 0) || (gid.gid_guid == 0)) { 5424 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: GID is required"); 5425 return (IBT_INVALID_PARAM); 5426 } 5427 5428 if (node_info_p == NULL) { 5429 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5430 "Return Buf (node_info_p) is NULL."); 5431 return (IBT_INVALID_PARAM); 5432 } 5433 5434 /* 5435 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5436 * associated port) info via ibtl_cm_get_hca_port() call. 5437 */ 5438 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5439 if (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS) { 5440 5441 hcap = ibcm_find_hca_entry(hport.hp_hca_guid); 5442 if (hcap == NULL) { 5443 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5444 "HCA(%llX) info not found", hport.hp_hca_guid); 5445 return (IBT_NO_HCAS_AVAILABLE); 5446 } 5447 num_ports = 1; 5448 num_hcas = 1; 5449 node_lid = hport.hp_base_lid; 5450 local_node = B_TRUE; 5451 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: Local Node: " 5452 "LID = 0x%X", node_lid); 5453 } else { 5454 /* Get the number of HCAs and their GUIDs */ 5455 num_hcas = ibt_get_hca_list(&guid_array); 5456 IBTF_DPRINTF_L4(cmlog, "ibt_gid_to_node_info: ibt_get_hca_list " 5457 "returned %d hcas", num_hcas); 5458 5459 if (num_hcas == 0) { 5460 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5461 "NO HCA's Found on this system"); 5462 return (IBT_NO_HCAS_AVAILABLE); 5463 } 5464 } 5465 5466 for (i = 0; i < num_hcas; i++) { 5467 if (local_node == B_FALSE) { 5468 hcap = ibcm_find_hca_entry(guid_array[i]); 5469 if (hcap == NULL) { 5470 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5471 "HCA(%llX) info not found", guid_array[i]); 5472 retval = IBT_NO_HCAS_AVAILABLE; 5473 continue; 5474 } 5475 num_ports = hcap->hca_num_ports; 5476 } 5477 5478 for (j = 0; j < num_ports; j++) { 5479 uint8_t port = 0; 5480 5481 if (local_node == B_TRUE) 5482 port = hport.hp_port; 5483 else 5484 port = j + 1; 5485 5486 /* Get SA Access Handle. */ 5487 saa_handle = ibcm_get_saa_handle(hcap, port); 5488 if (saa_handle == NULL) { 5489 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5490 "Port %d of HCA (%llX) is NOT ACTIVE", 5491 port, hport.hp_hca_guid); 5492 retval = IBT_NODE_RECORDS_NOT_FOUND; 5493 continue; 5494 } 5495 5496 if (local_node == B_FALSE) { 5497 ib_gid_t sgid; 5498 int sa_ret; 5499 5500 /* 5501 * Check whether 'gid' and this port has same 5502 * subnet prefix. If not, then there is no use 5503 * in searching from this port. 5504 */ 5505 sgid = hcap->hca_port_info[j].port_sgid0; 5506 if (gid.gid_prefix != sgid.gid_prefix) { 5507 IBTF_DPRINTF_L3(cmlog, 5508 "ibt_gid_to_node_info:Sn_Prefix of " 5509 "GID(%llX) and Port's(%llX) differ", 5510 gid.gid_prefix, sgid.gid_prefix); 5511 retval = IBT_NODE_RECORDS_NOT_FOUND; 5512 continue; 5513 } 5514 5515 /* 5516 * First Get Path Records for the specified DGID 5517 * from this port (SGID). From Path Records, 5518 * note down DLID, then use this DLID as Input 5519 * attribute to get NodeRecords from SA Access. 5520 */ 5521 npaths = 1; 5522 path = NULL; 5523 5524 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5525 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5526 &path); 5527 if (sa_ret != IBMF_SUCCESS) { 5528 IBTF_DPRINTF_L2(cmlog, 5529 "ibt_gid_to_node_info: " 5530 "ibmf_saa_gid_to_pathrecords() " 5531 "returned error: %d ", sa_ret); 5532 retval = 5533 ibcm_ibmf_analyze_error(sa_ret); 5534 continue; 5535 } else if ((npaths == 0) || (path == NULL)) { 5536 IBTF_DPRINTF_L3(cmlog, 5537 "ibt_gid_to_node_info: failed (%d) " 5538 "to get path records for the DGID " 5539 "0x%llX from SGID 0x%llX", sa_ret, 5540 gid.gid_guid, sgid.gid_guid); 5541 retval = IBT_NODE_RECORDS_NOT_FOUND; 5542 continue; 5543 } 5544 node_lid = path->DLID; /* LID */ 5545 5546 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5547 "Remote Node: LID = 0x%X", node_lid); 5548 5549 /* Free SA_Access memory for path record. */ 5550 kmem_free(path, len); 5551 } 5552 5553 /* Retrieve Node Records from SA Access. */ 5554 bzero(&nr_req, sizeof (sa_node_record_t)); 5555 5556 nr_req.LID = node_lid; /* LID */ 5557 5558 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5559 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5560 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5561 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5562 "failed (%d) to get Node records", retval); 5563 continue; 5564 } else if (retval != IBT_SUCCESS) { 5565 IBTF_DPRINTF_L2(cmlog, "ibt_gid_to_node_info: " 5566 "failed (%d) to get Node records", retval); 5567 ibcm_dec_hca_acc_cnt(hcap); 5568 goto gid_to_ni_exit; 5569 } 5570 5571 num_rec = len/sizeof (sa_node_record_t); 5572 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5573 5574 /* Validate the returned number of records. */ 5575 if ((nr_resp != NULL) && (num_rec > 0)) { 5576 5577 IBCM_DUMP_NODE_REC(nr_resp); 5578 5579 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS( 5580 *node_info_p)) 5581 5582 node_info_p->n_sys_img_guid = 5583 nr_resp->NodeInfo.SystemImageGUID; 5584 node_info_p->n_node_guid = 5585 nr_resp->NodeInfo.NodeGUID; 5586 node_info_p->n_port_guid = 5587 nr_resp->NodeInfo.PortGUID; 5588 node_info_p->n_dev_id = 5589 nr_resp->NodeInfo.DeviceID; 5590 node_info_p->n_revision = 5591 nr_resp->NodeInfo.Revision; 5592 node_info_p->n_vendor_id = 5593 nr_resp->NodeInfo.VendorID; 5594 node_info_p->n_num_ports = 5595 nr_resp->NodeInfo.NumPorts; 5596 node_info_p->n_port_num = 5597 nr_resp->NodeInfo.LocalPortNum; 5598 node_info_p->n_node_type = 5599 nr_resp->NodeInfo.NodeType; 5600 (void) strncpy(node_info_p->n_description, 5601 (char *)&nr_resp->NodeDescription, 64); 5602 5603 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS( 5604 *node_info_p)) 5605 5606 /* 5607 * Deallocate the memory allocated by SA for 5608 * 'nr_resp'. 5609 */ 5610 ibcm_dec_hca_acc_cnt(hcap); 5611 kmem_free(nr_resp, len); 5612 retval = IBT_SUCCESS; 5613 5614 goto gid_to_ni_exit; 5615 } else { 5616 retval = IBT_NODE_RECORDS_NOT_FOUND; 5617 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: " 5618 "Node Records NOT found - PortGUID %016llX", 5619 gid.gid_guid); 5620 } 5621 } 5622 ibcm_dec_hca_acc_cnt(hcap); 5623 5624 if (local_node == B_TRUE) 5625 break; 5626 } 5627 5628 gid_to_ni_exit: 5629 if (guid_array) 5630 ibt_free_hca_list(guid_array, num_hcas); 5631 5632 IBTF_DPRINTF_L3(cmlog, "ibt_gid_to_node_info: done. Status %d", retval); 5633 5634 return (retval); 5635 } 5636 5637 5638 static ibt_status_t 5639 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req, 5640 uint64_t component_mask, void *result_p, size_t *len) 5641 { 5642 ibmf_saa_access_args_t args; 5643 size_t length; 5644 ibt_status_t retval; 5645 5646 args.sq_attr_id = SA_NODERECORD_ATTRID; 5647 args.sq_template = nr_req; 5648 args.sq_access_type = IBMF_SAA_RETRIEVE; 5649 args.sq_template_length = sizeof (sa_node_record_t); 5650 args.sq_component_mask = component_mask; 5651 args.sq_callback = NULL; 5652 args.sq_callback_arg = NULL; 5653 5654 retval = ibcm_contact_sa_access(saa_handle, &args, &length, result_p); 5655 if (retval != IBT_SUCCESS) { 5656 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: SA Call Failed"); 5657 return (retval); 5658 } 5659 5660 *len = length; 5661 5662 /* Validate the returned number of records. */ 5663 if ((result_p != NULL) && (length > 0)) { 5664 IBTF_DPRINTF_L3(cmlog, "ibcm_get_node_rec: Node Records FOUND"); 5665 5666 /* Got it, done!. */ 5667 return (IBT_SUCCESS); 5668 } else { 5669 IBTF_DPRINTF_L2(cmlog, "ibcm_get_node_rec: Node Rec NOT found"); 5670 return (IBT_NODE_RECORDS_NOT_FOUND); 5671 } 5672 } 5673 5674 5675 /* 5676 * Function: 5677 * ibt_get_companion_port_gids() 5678 * Description: 5679 * Get list of GID's available on a companion port(s) of the specified 5680 * GID or list of GIDs available on a specified Node GUID/SystemImage GUID. 5681 */ 5682 ibt_status_t 5683 ibt_get_companion_port_gids(ib_gid_t gid, ib_guid_t hca_guid, 5684 ib_guid_t sysimg_guid, ib_gid_t **gids_p, uint_t *num_gids_p) 5685 { 5686 sa_node_record_t nr_req, *nr_resp; 5687 void *res_p; 5688 ibmf_saa_handle_t saa_handle; 5689 int sa_ret; 5690 ibt_status_t retval = IBT_SUCCESS; 5691 ibcm_hca_info_t *hcap; 5692 ibtl_cm_hca_port_t hport; 5693 int i, j; 5694 uint_t num_rec; 5695 ib_guid_t *guid_array = NULL; 5696 sa_path_record_t *path; 5697 size_t len; 5698 uint8_t npaths; 5699 uint32_t num_hcas = 0; 5700 boolean_t local_node = B_FALSE; 5701 boolean_t local_hca = B_FALSE; 5702 ib_guid_t h_guid = hca_guid; 5703 ib_gid_t *gidp = NULL, *t_gidp = NULL; 5704 int multi_hca_loop = 0; 5705 5706 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids(%llX:%llX, %llX, " 5707 "%llX)", gid.gid_prefix, gid.gid_guid, hca_guid, sysimg_guid); 5708 5709 if (((gid.gid_prefix == 0) || (gid.gid_guid == 0)) && (hca_guid == 0) && 5710 (sysimg_guid == 0)) { 5711 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5712 "Null Input attribute specified."); 5713 return (IBT_INVALID_PARAM); 5714 } 5715 5716 if ((num_gids_p == NULL) || (gids_p == NULL)) { 5717 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5718 "num_gids_p or gids_p is NULL"); 5719 return (IBT_INVALID_PARAM); 5720 } 5721 5722 *num_gids_p = 0; 5723 5724 /* Get the number of HCAs and their GUIDs */ 5725 if ((num_hcas = ibt_get_hca_list(&guid_array)) == 0) { 5726 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5727 "NO HCA's Found on this system"); 5728 return (IBT_NO_HCAS_AVAILABLE); 5729 } 5730 5731 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5732 "ibt_get_hca_list() returned %d hcas", num_hcas); 5733 5734 /* 5735 * If 'gid' is on local node, then get node lid (i.e. base lid of the 5736 * associated port) info via ibtl_cm_get_hca_port() call. 5737 */ 5738 bzero(&hport, sizeof (ibtl_cm_hca_port_t)); 5739 if ((gid.gid_prefix != 0) && (gid.gid_guid != 0) && 5740 (ibtl_cm_get_hca_port(gid, 0, &hport) == IBT_SUCCESS)) { 5741 5742 if ((hca_guid != 0) && (hca_guid != hport.hp_hca_guid)) { 5743 IBTF_DPRINTF_L2(cmlog, "ibt_get_companion_port_gids: " 5744 "Invalid GID<->HCAGUID combination specified."); 5745 retval = IBT_INVALID_PARAM; 5746 goto get_comp_pgid_exit; 5747 } 5748 h_guid = hport.hp_hca_guid; 5749 local_node = B_TRUE; 5750 5751 IBTF_DPRINTF_L4(cmlog, "ibt_get_companion_port_gids: " 5752 "Local Node: HCA (0x%llX)", h_guid); 5753 } else if (h_guid) { /* Is specified HCA GUID - local? */ 5754 for (i = 0; i < num_hcas; i++) { 5755 if (h_guid == guid_array[i]) { 5756 local_hca = B_TRUE; 5757 break; 5758 } 5759 } 5760 } else if (sysimg_guid) { /* Is specified SystemImage GUID - local? */ 5761 for (i = 0; i < num_hcas; i++) { 5762 ibt_status_t ret; 5763 ibt_hca_attr_t hca_attr; 5764 5765 ret = ibt_query_hca_byguid(guid_array[i], &hca_attr); 5766 if (ret != IBT_SUCCESS) { 5767 IBTF_DPRINTF_L2(cmlog, 5768 "ibt_get_companion_port_gids: HCA(%llX) " 5769 "info not found", guid_array[i]); 5770 retval = IBT_NO_HCAS_AVAILABLE; 5771 continue; 5772 } 5773 if (hca_attr.hca_si_guid == sysimg_guid) { 5774 if ((hca_guid != 0) && 5775 (hca_guid != hca_attr.hca_node_guid)) { 5776 IBTF_DPRINTF_L2(cmlog, 5777 "ibt_get_companion_port_gids: " 5778 "Invalid SysImg<->HCA GUID " 5779 "combination specified."); 5780 retval = IBT_INVALID_PARAM; 5781 goto get_comp_pgid_exit; 5782 } 5783 local_hca = B_TRUE; 5784 h_guid = hca_attr.hca_node_guid; 5785 break; 5786 } 5787 } 5788 } 5789 5790 if ((local_node == B_TRUE) || (local_hca == B_TRUE)) { 5791 retval = ibtl_cm_get_local_comp_gids(h_guid, gid, gids_p, 5792 num_gids_p); 5793 goto get_comp_pgid_exit; 5794 } 5795 5796 get_comp_for_multihca: 5797 /* We will be here, if request is for remote node */ 5798 for (i = 0; i < num_hcas; i++) { 5799 int multism; 5800 uint8_t count = 0; 5801 int multi_sm_loop = 0; 5802 uint_t k = 0, l; 5803 5804 hcap = ibcm_find_hca_entry(guid_array[i]); 5805 if (hcap == NULL) { 5806 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5807 "HCA(%llX) info not found", guid_array[i]); 5808 retval = IBT_NO_HCAS_AVAILABLE; 5809 continue; 5810 } 5811 5812 /* 1 - MultiSM, 0 - Single SM */ 5813 multism = ibtl_cm_is_multi_sm(guid_array[i]); 5814 5815 for (j = 0; j < hcap->hca_num_ports; j++) { 5816 ib_gid_t sgid; 5817 uint64_t c_mask = 0; 5818 ib_guid_t pg; 5819 uint_t port = j; 5820 5821 get_comp_for_multism: 5822 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5823 "Port %d, HCA %llX, MultiSM= %d, Loop=%d", 5824 port + 1, h_guid, multism, multi_sm_loop); 5825 5826 /* Get SA Access Handle. */ 5827 saa_handle = ibcm_get_saa_handle(hcap, port + 1); 5828 if (saa_handle == NULL) { 5829 IBTF_DPRINTF_L2(cmlog, 5830 "ibt_get_companion_port_gids: " 5831 "Port (%d) - NOT ACTIVE", port + 1); 5832 retval = IBT_GIDS_NOT_FOUND; 5833 continue; 5834 } 5835 5836 /* 5837 * Check whether 'gid' and this port has same subnet 5838 * prefix. If not, then there is no use in searching 5839 * from this port. 5840 */ 5841 sgid = hcap->hca_port_info[port].port_sgid0; 5842 if ((h_guid == 0) && (gid.gid_prefix != 0) && 5843 (multi_sm_loop == 0) && 5844 (gid.gid_prefix != sgid.gid_prefix)) { 5845 IBTF_DPRINTF_L2(cmlog, 5846 "ibt_get_companion_port_gids: SnPrefix of " 5847 "GID(%llX) and Port SN_Pfx(%llX) differ", 5848 gid.gid_prefix, sgid.gid_prefix); 5849 retval = IBT_GIDS_NOT_FOUND; 5850 continue; 5851 } 5852 5853 /* 5854 * If HCA GUID or System Image GUID is specified, then 5855 * we can achieve our goal sooner!. 5856 */ 5857 if ((h_guid == 0) && (sysimg_guid == 0)) { 5858 /* So only GID info is provided. */ 5859 5860 /* 5861 * First Get Path Records for the specified DGID 5862 * from this port (SGID). From Path Records, 5863 * note down DLID, then use this DLID as Input 5864 * attribute to get NodeRecords. 5865 */ 5866 npaths = 1; 5867 path = NULL; 5868 5869 sa_ret = ibmf_saa_gid_to_pathrecords(saa_handle, 5870 sgid, gid, 0, 0, B_TRUE, &npaths, 0, &len, 5871 &path); 5872 if (sa_ret != IBMF_SUCCESS) { 5873 IBTF_DPRINTF_L2(cmlog, 5874 "ibt_get_companion_port_gids: " 5875 "ibmf_saa_gid_to_pathrecords() " 5876 "returned error: %d ", sa_ret); 5877 retval = 5878 ibcm_ibmf_analyze_error(sa_ret); 5879 ibcm_dec_hca_acc_cnt(hcap); 5880 goto get_comp_pgid_exit; 5881 } else if ((npaths == 0) || (path == NULL)) { 5882 IBTF_DPRINTF_L2(cmlog, 5883 "ibt_get_companion_port_gids: " 5884 "failed (%d) to get path records " 5885 "for the DGID (0x%llX) from SGID " 5886 "(0x%llX)", sa_ret, gid.gid_guid, 5887 sgid.gid_guid); 5888 retval = IBT_GIDS_NOT_FOUND; 5889 continue; 5890 } 5891 5892 bzero(&nr_req, sizeof (sa_node_record_t)); 5893 nr_req.LID = path->DLID; /* LID */ 5894 5895 IBTF_DPRINTF_L3(cmlog, 5896 "ibt_get_companion_port_gids: " 5897 "Remote Node: LID = 0x%X", nr_req.LID); 5898 5899 /* Free SA_Access memory for path record. */ 5900 kmem_free(path, len); 5901 5902 IBTF_DPRINTF_L3(cmlog, 5903 "ibt_get_companion_port_gids: SAA Call: " 5904 "based on LID "); 5905 5906 retval = ibcm_get_node_rec(saa_handle, &nr_req, 5907 SA_NODEINFO_COMPMASK_NODELID, &res_p, &len); 5908 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5909 IBTF_DPRINTF_L2(cmlog, 5910 "ibt_get_companion_port_gids: " 5911 "failed (%d) to get Node records", 5912 retval); 5913 continue; 5914 } else if (retval != IBT_SUCCESS) { 5915 IBTF_DPRINTF_L2(cmlog, 5916 "ibt_get_companion_port_gids: " 5917 "failed (%d) to get Node records", 5918 retval); 5919 ibcm_dec_hca_acc_cnt(hcap); 5920 goto get_comp_pgid_exit; 5921 } 5922 5923 nr_resp = (sa_node_record_t *)(uchar_t *)res_p; 5924 /* Note down HCA GUID info. */ 5925 h_guid = nr_resp->NodeInfo.NodeGUID; 5926 5927 IBTF_DPRINTF_L3(cmlog, 5928 "ibt_get_companion_port_gids: " 5929 "Remote HCA GUID: 0x%llX", h_guid); 5930 5931 IBCM_DUMP_NODE_REC(nr_resp); 5932 5933 kmem_free(res_p, len); 5934 } 5935 5936 bzero(&nr_req, sizeof (sa_node_record_t)); 5937 if (h_guid != 0) { 5938 nr_req.NodeInfo.NodeGUID = h_guid; 5939 c_mask = SA_NODEINFO_COMPMASK_NODEGUID; 5940 } 5941 5942 if (sysimg_guid != 0) { 5943 nr_req.NodeInfo.SystemImageGUID = sysimg_guid; 5944 c_mask |= SA_NODEINFO_COMPMASK_SYSIMAGEGUID; 5945 } 5946 5947 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: " 5948 "SAA Call: CMASK= 0x%llX", c_mask); 5949 5950 retval = ibcm_get_node_rec(saa_handle, &nr_req, c_mask, 5951 &res_p, &len); 5952 if (retval == IBT_NODE_RECORDS_NOT_FOUND) { 5953 IBTF_DPRINTF_L3(cmlog, 5954 "ibt_get_companion_port_gids: " 5955 "failed (%d) to get Node records", retval); 5956 continue; 5957 } else if (retval != IBT_SUCCESS) { 5958 IBTF_DPRINTF_L2(cmlog, 5959 "ibt_get_companion_port_gids: Error: (%d) " 5960 "while getting Node records", retval); 5961 ibcm_dec_hca_acc_cnt(hcap); 5962 goto get_comp_pgid_exit; 5963 } 5964 5965 num_rec = len/sizeof (sa_node_record_t); 5966 5967 /* We will be here, only if we found some NodeRec */ 5968 if (gid.gid_prefix && gid.gid_guid) { 5969 nr_resp = (sa_node_record_t *)res_p; 5970 for (l = 0; l < num_rec; l++, nr_resp++) { 5971 pg = nr_resp->NodeInfo.PortGUID; 5972 if (gid.gid_guid != pg) 5973 count++; 5974 } 5975 } else { 5976 count = num_rec; 5977 } 5978 5979 if (count != 0) { 5980 if (multi_sm_loop == 1) { 5981 count += k; 5982 t_gidp = kmem_zalloc(count * 5983 sizeof (ib_gid_t), KM_SLEEP); 5984 5985 if ((k != 0) && (gidp != NULL)) { 5986 bcopy(gidp, t_gidp, 5987 k * sizeof (ib_gid_t)); 5988 kmem_free(gidp, 5989 k * sizeof (ib_gid_t)); 5990 } 5991 gidp = t_gidp; 5992 } else { 5993 gidp = kmem_zalloc(count * 5994 sizeof (ib_gid_t), KM_SLEEP); 5995 } 5996 *num_gids_p = count; 5997 *gids_p = gidp; 5998 5999 nr_resp = (sa_node_record_t *)res_p; 6000 for (l = 0; l < num_rec; l++, nr_resp++) { 6001 IBCM_DUMP_NODE_REC(nr_resp); 6002 6003 pg = nr_resp->NodeInfo.PortGUID; 6004 IBTF_DPRINTF_L4(cmlog, 6005 "ibt_get_companion_port_gids: " 6006 "PortGID %llX", pg); 6007 6008 if (pg != gid.gid_guid) { 6009 gidp[k].gid_prefix = 6010 sgid.gid_prefix; 6011 gidp[k].gid_guid = pg; 6012 6013 IBTF_DPRINTF_L3(cmlog, 6014 "ibt_get_companion_pgids: " 6015 "GID[%d] = %llX:%llX", k, 6016 gidp[k].gid_prefix, 6017 gidp[k].gid_guid); 6018 6019 k++; 6020 if (k == count) 6021 break; 6022 } 6023 } 6024 retval = IBT_SUCCESS; /* done!. */ 6025 kmem_free(res_p, len); 6026 ibcm_dec_hca_acc_cnt(hcap); 6027 goto get_comp_pgid_exit; 6028 } else { 6029 IBTF_DPRINTF_L2(cmlog, 6030 "ibt_get_companion_port_gids: " 6031 "Companion PortGIDs not available"); 6032 retval = IBT_GIDS_NOT_FOUND; 6033 } 6034 /* Deallocate the memory for 'res_p'. */ 6035 kmem_free(res_p, len); 6036 6037 /* 6038 * If we are on MultiSM setup, then we need to lookout 6039 * from that subnet port too. 6040 */ 6041 if (multism) { 6042 /* break if already searched both the subnet */ 6043 if (multi_sm_loop == 1) 6044 break; 6045 6046 port = (j == 0) ? 1 : 0; 6047 multi_sm_loop = 1; 6048 goto get_comp_for_multism; 6049 } else { 6050 break; 6051 } 6052 } 6053 ibcm_dec_hca_acc_cnt(hcap); 6054 6055 /* 6056 * We may be on dual HCA with dual SM configured system. And 6057 * the input attr GID was visible from second HCA. So in order 6058 * to get the companion portgid we need to re-look from the 6059 * first HCA ports. 6060 */ 6061 if ((num_hcas > 1) && (i > 0) && (h_guid != 0) && 6062 (multi_hca_loop != 1)) { 6063 multi_hca_loop = 1; 6064 goto get_comp_for_multihca; 6065 } 6066 } 6067 if (*num_gids_p == 0) 6068 retval = IBT_GIDS_NOT_FOUND; 6069 6070 get_comp_pgid_exit: 6071 if (guid_array) 6072 ibt_free_hca_list(guid_array, num_hcas); 6073 6074 if ((retval != IBT_SUCCESS) && (*num_gids_p != 0)) { 6075 retval = IBT_SUCCESS; 6076 } 6077 6078 IBTF_DPRINTF_L3(cmlog, "ibt_get_companion_port_gids: done. Status %d, " 6079 "Found %d GIDs", retval, *num_gids_p); 6080 6081 return (retval); 6082 } 6083 6084 /* RDMA IP CM Support routines */ 6085 ibt_status_t 6086 ibt_get_src_ip(ib_gid_t gid, ib_pkey_t pkey, ibt_ip_addr_t *src_ip) 6087 { 6088 ibcm_arp_ip_t *ipp; 6089 ibcm_arp_ibd_insts_t ibds; 6090 int i; 6091 boolean_t found = B_FALSE; 6092 ibt_status_t retval = IBT_SUCCESS; 6093 6094 IBTF_DPRINTF_L4(cmlog, "ibt_get_src_ip(%llX:%llX, %X, %p)", 6095 gid.gid_prefix, gid.gid_guid, pkey, src_ip); 6096 6097 if (gid.gid_prefix == 0 || gid.gid_guid == 0) { 6098 IBTF_DPRINTF_L3(cmlog, "ibt_get_src_ip: Invalid GID."); 6099 return (IBT_INVALID_PARAM); 6100 } 6101 6102 if (src_ip == NULL) { 6103 IBTF_DPRINTF_L3(cmlog, "ibt_get_src_ip: ERROR: src_ip NULL"); 6104 return (IBT_INVALID_PARAM); 6105 } 6106 6107 bzero(&ibds, sizeof (ibcm_arp_ibd_insts_t)); 6108 ibds.ibcm_arp_ibd_alloc = IBCM_ARP_IBD_INSTANCES; 6109 ibds.ibcm_arp_ibd_cnt = 0; 6110 ibds.ibcm_arp_ip = (ibcm_arp_ip_t *)kmem_zalloc( 6111 ibds.ibcm_arp_ibd_alloc * sizeof (ibcm_arp_ip_t), KM_SLEEP); 6112 6113 retval = ibcm_arp_get_ibds(&ibds); 6114 if (retval != IBT_SUCCESS) { 6115 IBTF_DPRINTF_L2(cmlog, "ibt_get_src_ip: ibcm_arp_get_ibds " 6116 "failed to get IBD Instances: ret 0x%x", retval); 6117 goto get_src_ip_end; 6118 } 6119 6120 for (i = 0, ipp = ibds.ibcm_arp_ip; i < ibds.ibcm_arp_ibd_cnt; 6121 i++, ipp++) { 6122 if (ipp->ip_port_gid.gid_prefix == gid.gid_prefix && 6123 ipp->ip_port_gid.gid_guid == gid.gid_guid) { 6124 if (pkey) { 6125 if (ipp->ip_pkey == pkey) { 6126 found = B_TRUE; 6127 break; 6128 } else 6129 continue; 6130 } 6131 found = B_TRUE; 6132 break; 6133 } 6134 } 6135 6136 if (found == B_FALSE) { 6137 retval = IBT_SRC_IP_NOT_FOUND; 6138 } else { 6139 src_ip->family = ipp->ip_inet_family; 6140 if (src_ip->family == AF_INET) { 6141 bcopy(&ipp->ip_cm_sin.sin_addr, &src_ip->un.ip4addr, 6142 sizeof (in_addr_t)); 6143 IBTF_DPRINTF_L4(cmlog, "ibt_get_src_ip: Got %lX", 6144 src_ip->un.ip4addr); 6145 } else if (src_ip->family == AF_INET6) { 6146 bcopy(&ipp->ip_cm_sin6.sin6_addr, &src_ip->un.ip6addr, 6147 sizeof (in6_addr_t)); 6148 } 6149 } 6150 6151 get_src_ip_end: 6152 if (ibds.ibcm_arp_ip) 6153 kmem_free(ibds.ibcm_arp_ip, 6154 ibds.ibcm_arp_ibd_alloc * sizeof (ibcm_arp_ip_t)); 6155 6156 return (retval); 6157 } 6158 6159 ib_svc_id_t 6160 ibt_get_ip_sid(uint8_t protocol_num, in_port_t dst_port) 6161 { 6162 ib_svc_id_t sid; 6163 6164 IBTF_DPRINTF_L4(cmlog, "ibt_get_ip_sid(%X, %lX)", protocol_num, 6165 dst_port); 6166 6167 /* 6168 * If protocol_num is non-zero, then formulate the SID and return it. 6169 * If protocol_num is zero, then we need to assign a locally generated 6170 * IP SID with IB_SID_IPADDR_PREFIX. 6171 */ 6172 if (protocol_num) { 6173 sid = IB_SID_IPADDR_PREFIX | protocol_num << 16 | dst_port; 6174 } else { 6175 sid = ibcm_alloc_ip_sid(); 6176 } 6177 6178 IBTF_DPRINTF_L3(cmlog, "ibt_get_ip_sid: SID: 0x%016llX", sid); 6179 return (sid); 6180 } 6181 6182 ibt_status_t 6183 ibt_release_ip_sid(ib_svc_id_t ip_sid) 6184 { 6185 IBTF_DPRINTF_L4(cmlog, "ibt_release_ip_sid(%llX)", ip_sid); 6186 6187 if (((ip_sid & IB_SID_IPADDR_PREFIX_MASK) != 0) || 6188 (!(ip_sid & IB_SID_IPADDR_PREFIX))) { 6189 IBTF_DPRINTF_L2(cmlog, "ibt_release_ip_sid(0x%016llX): ERROR: " 6190 "Called for Non-RDMA IP SID", ip_sid); 6191 return (IBT_INVALID_PARAM); 6192 } 6193 6194 /* 6195 * If protocol_num in ip_sid are all ZEROs, then this SID is allocated 6196 * by IBTF. If not, then the specified ip_sid is invalid. 6197 */ 6198 if (ip_sid & IB_SID_IPADDR_IPNUM_MASK) { 6199 IBTF_DPRINTF_L2(cmlog, "ibt_release_ip_sid(0x%016llX): ERROR: " 6200 "Called for Non-IBTF assigned RDMA IP SID", ip_sid); 6201 return (IBT_INVALID_PARAM); 6202 } 6203 6204 ibcm_free_ip_sid(ip_sid); 6205 6206 return (IBT_SUCCESS); 6207 } 6208 6209 6210 uint8_t 6211 ibt_get_ip_protocol_num(ib_svc_id_t sid) 6212 { 6213 return ((sid & IB_SID_IPADDR_IPNUM_MASK) >> 16); 6214 } 6215 6216 in_port_t 6217 ibt_get_ip_dst_port(ib_svc_id_t sid) 6218 { 6219 return (sid & IB_SID_IPADDR_PORTNUM_MASK); 6220 } 6221 6222 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", ibt_ip_cm_info_t)) 6223 _NOTE(SCHEME_PROTECTS_DATA("Unshared data", ibcm_ip_pvtdata_t)) 6224 6225 ibt_status_t 6226 ibt_format_ip_private_data(ibt_ip_cm_info_t *ip_cm_info, 6227 ibt_priv_data_len_t priv_data_len, void *priv_data_p) 6228 { 6229 ibcm_ip_pvtdata_t *ip_data; 6230 6231 IBTF_DPRINTF_L4(cmlog, "ibt_format_ip_private_data(%p, %d, %p)", 6232 ip_cm_info, priv_data_len, priv_data_p); 6233 6234 if ((ip_cm_info == NULL) || (priv_data_p == NULL) || 6235 (priv_data_len < IBT_IP_HDR_PRIV_DATA_SZ)) { 6236 IBTF_DPRINTF_L2(cmlog, "ibt_format_ip_private_data: ERROR " 6237 "Invalid Inputs."); 6238 return (IBT_INVALID_PARAM); 6239 } 6240 6241 /* bzero'ing just IP_HDR part */ 6242 bzero(priv_data_p, IBT_IP_HDR_PRIV_DATA_SZ); 6243 ip_data = (ibcm_ip_pvtdata_t *)priv_data_p; 6244 ip_data->ip_srcport = b2h16(ip_cm_info->src_port); /* Source Port */ 6245 6246 /* IPV = 0x4, if IP-Addr are IPv4 format, else 0x6 for IPv6 */ 6247 if (ip_cm_info->src_addr.family == AF_INET) { 6248 ip_data->ip_ipv = IBT_CM_IP_IPV_V4; 6249 ip_data->ip_srcv4 = ntohl(ip_cm_info->src_addr.un.ip4addr); 6250 ip_data->ip_dstv4 = ntohl(ip_cm_info->dst_addr.un.ip4addr); 6251 } else if (ip_cm_info->src_addr.family == AF_INET6) { 6252 ip_data->ip_ipv = IBT_CM_IP_IPV_V6; 6253 bcopy(&ip_cm_info->src_addr.un.ip6addr, 6254 &ip_data->ip_srcv6, sizeof (in6_addr_t)); 6255 bcopy(&ip_cm_info->dst_addr.un.ip6addr, 6256 &ip_data->ip_dstv6, sizeof (in6_addr_t)); 6257 } else { 6258 IBTF_DPRINTF_L2(cmlog, "ibt_format_ip_private_data: ERROR " 6259 "IP Addr needs to be either AF_INET or AF_INET6 family."); 6260 return (IBT_INVALID_PARAM); 6261 } 6262 6263 ip_data->ip_MajV = IBT_CM_IP_MAJ_VER; 6264 ip_data->ip_MinV = IBT_CM_IP_MIN_VER; 6265 6266 return (IBT_SUCCESS); 6267 } 6268 6269 6270 ibt_status_t 6271 ibt_get_ip_data(ibt_priv_data_len_t priv_data_len, void *priv_data, 6272 ibt_ip_cm_info_t *ip_cm_infop) 6273 { 6274 ibcm_ip_pvtdata_t *ip_data; 6275 6276 IBTF_DPRINTF_L4(cmlog, "ibt_get_ip_data(%d, %p, %p)", 6277 priv_data_len, priv_data, ip_cm_infop); 6278 6279 if ((ip_cm_infop == NULL) || (priv_data == NULL) || 6280 (priv_data_len < IBT_IP_HDR_PRIV_DATA_SZ)) { 6281 IBTF_DPRINTF_L2(cmlog, "ibt_get_ip_data: ERROR Invalid Inputs"); 6282 return (IBT_INVALID_PARAM); 6283 } 6284 6285 bzero(ip_cm_infop, sizeof (ibt_ip_cm_info_t)); 6286 6287 ip_data = (ibcm_ip_pvtdata_t *)priv_data; 6288 ip_cm_infop->src_port = b2h16(ip_data->ip_srcport); /* Source Port */ 6289 6290 /* IPV = 0x4, if IP Address are IPv4 format, else 0x6 for IPv6 */ 6291 if (ip_data->ip_ipv == IBT_CM_IP_IPV_V4) { 6292 /* Copy IPv4 Addr */ 6293 ip_cm_infop->src_addr.family = AF_INET; 6294 ip_cm_infop->src_addr.un.ip4addr = ntohl(ip_data->ip_srcv4); 6295 ip_cm_infop->dst_addr.family = AF_INET; 6296 ip_cm_infop->dst_addr.un.ip4addr = ntohl(ip_data->ip_dstv4); 6297 } else if (ip_data->ip_ipv == IBT_CM_IP_IPV_V6) { 6298 /* Copy IPv6 Addr */ 6299 ip_cm_infop->src_addr.family = AF_INET6; 6300 bcopy(&ip_data->ip_srcv6, &ip_cm_infop->src_addr.un.ip6addr, 6301 sizeof (in6_addr_t)); 6302 ip_cm_infop->dst_addr.family = AF_INET6; 6303 bcopy(&ip_data->ip_dstv6, &ip_cm_infop->dst_addr.un.ip6addr, 6304 sizeof (in6_addr_t)); 6305 } else { 6306 IBTF_DPRINTF_L2(cmlog, "ibt_get_ip_data: ERROR: IP Addr needs" 6307 " to be either AF_INET or AF_INET6 family."); 6308 return (IBT_INVALID_PARAM); 6309 } 6310 6311 return (IBT_SUCCESS); 6312 } 6313 6314 6315 /* Routines for warlock */ 6316 6317 /* ARGSUSED */ 6318 static void 6319 ibcm_dummy_mcg_handler(void *arg, ibt_status_t retval, ibt_mcg_info_t *minfo) 6320 { 6321 ibcm_join_mcg_tqarg_t dummy_mcg; 6322 6323 dummy_mcg.func = ibcm_dummy_mcg_handler; 6324 6325 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_mcg_handler: " 6326 "dummy_mcg.func %p", dummy_mcg.func); 6327 } 6328 6329 6330 /* ARGSUSED */ 6331 static void 6332 ibcm_dummy_recycle_rc_handler(ibt_status_t retval, void *arg) 6333 { 6334 ibcm_taskq_recycle_arg_t dummy_rc_recycle; 6335 6336 dummy_rc_recycle.func = ibcm_dummy_recycle_rc_handler; 6337 6338 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_recycle_rc_handler: " 6339 "dummy_rc_recycle.func %p", dummy_rc_recycle.func); 6340 } 6341 6342 6343 /* ARGSUSED */ 6344 static ibt_cm_status_t 6345 ibcm_dummy_ud_handler(void *priv, ibt_cm_ud_event_t *event, 6346 ibt_cm_ud_return_args_t *ret_args, 6347 void *priv_data, ibt_priv_data_len_t len) 6348 { 6349 /* 6350 * Let warlock see that ibcm_local_handler_s::actual_cm_handler 6351 * points to this routine. 6352 */ 6353 ibcm_local_handler_t p; 6354 ibcm_ud_state_data_t dummy_ud; 6355 6356 p.actual_cm_handler = ibcm_dummy_ud_handler; 6357 dummy_ud.ud_cm_handler = ibcm_dummy_ud_handler; 6358 6359 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_ud_handler: p.actual_cm_handler %p" 6360 "dummy_ud.ud_cm_handler %p", p.actual_cm_handler, 6361 dummy_ud.ud_cm_handler); 6362 /* 6363 * Call all routines that the client's callback routine could call. 6364 */ 6365 6366 return (IBT_CM_ACCEPT); 6367 } 6368 6369 /* ARGSUSED */ 6370 static ibt_cm_status_t 6371 ibcm_dummy_rc_handler(void *priv, ibt_cm_event_t *event, 6372 ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len) 6373 { 6374 ibcm_state_data_t dummy_rc; 6375 6376 dummy_rc.cm_handler = ibcm_dummy_rc_handler; 6377 6378 IBTF_DPRINTF_L5(cmlog, "ibcm_dummy_rc_handler: " 6379 "dummy_ud.ud_cm_handler %p", dummy_rc.cm_handler); 6380 /* 6381 * Call all routines that the client's callback routine could call. 6382 */ 6383 6384 return (IBT_CM_ACCEPT); 6385 } 6386