1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * This file implements the MAD send logic in IBMF. 29 */ 30 31 #include <sys/ib/mgt/ibmf/ibmf_impl.h> 32 33 #define IBMF_SEND_WR_ID_TO_ADDR(id, ptr) \ 34 (ptr) = (void *)(uintptr_t)(id) 35 36 extern int ibmf_trace_level; 37 38 static void ibmf_i_do_send_cb(void *taskq_arg); 39 static void ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, 40 ibmf_msg_impl_t *msgimplp, ibmf_send_wqe_t *send_wqep); 41 42 /* 43 * ibmf_i_issue_pkt(): 44 * Post an IB packet on the specified QP's send queue 45 */ 46 int 47 ibmf_i_issue_pkt(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp, 48 ibmf_qp_handle_t ibmf_qp_handle, ibmf_send_wqe_t *send_wqep) 49 { 50 int ret; 51 ibt_status_t status; 52 ibt_wr_ds_t sgl[1]; 53 ibt_qp_hdl_t ibt_qp_handle; 54 55 _NOTE(ASSUMING_PROTECTED(*send_wqep)) 56 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) 57 58 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, 59 ibmf_i_issue_pkt_start, IBMF_TNF_TRACE, "", 60 "ibmf_i_issue_pkt() enter, clientp = %p, msg = %p, " 61 "qp_hdl = %p, swqep = %p\n", tnf_opaque, clientp, clientp, 62 tnf_opaque, msg, msgimplp, tnf_opaque, ibmf_qp_handle, 63 ibmf_qp_handle, tnf_opaque, send_wqep, send_wqep); 64 65 ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); 66 ASSERT(MUTEX_NOT_HELD(&clientp->ic_mutex)); 67 68 /* 69 * if the qp handle provided in ibmf_send_pkt() 70 * is not the default qp handle for this client, 71 * then the wqe must be sent on this qp, 72 * else use the default qp handle set up during ibmf_register() 73 */ 74 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 75 ibt_qp_handle = clientp->ic_qp->iq_qp_handle; 76 } else { 77 ibt_qp_handle = 78 ((ibmf_alt_qp_t *)ibmf_qp_handle)->isq_qp_handle; 79 } 80 81 /* initialize the send WQE */ 82 ibmf_i_init_send_wqe(clientp, msgimplp, sgl, send_wqep, 83 msgimplp->im_ud_dest, ibt_qp_handle, ibmf_qp_handle); 84 85 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) 86 87 /* 88 * Issue the wqe to the transport. 89 * NOTE: ibt_post_send() will not block, so, it is ok 90 * to hold the msgimpl mutex across this call. 91 */ 92 status = ibt_post_send(send_wqep->send_qp_handle, &send_wqep->send_wr, 93 1, NULL); 94 if (status != IBT_SUCCESS) { 95 mutex_enter(&clientp->ic_kstat_mutex); 96 IBMF_ADD32_KSTATS(clientp, send_pkt_failed, 1); 97 mutex_exit(&clientp->ic_kstat_mutex); 98 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 99 ibmf_i_issue_pkt_err, IBMF_TNF_TRACE, "", 100 "ibmf_i_issue_pkt(): %s, status = %d\n", 101 tnf_string, msg, "post send failure", 102 tnf_uint, ibt_status, status); 103 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end, 104 IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt(() exit\n"); 105 return (IBMF_TRANSPORT_FAILURE); 106 } 107 108 ret = IBMF_SUCCESS; 109 110 /* bump the number of active sends */ 111 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 112 mutex_enter(&clientp->ic_mutex); 113 clientp->ic_sends_active++; 114 mutex_exit(&clientp->ic_mutex); 115 mutex_enter(&clientp->ic_kstat_mutex); 116 IBMF_ADD32_KSTATS(clientp, sends_active, 1); 117 mutex_exit(&clientp->ic_kstat_mutex); 118 } else { 119 ibmf_alt_qp_t *qpp = (ibmf_alt_qp_t *)ibmf_qp_handle; 120 mutex_enter(&qpp->isq_mutex); 121 qpp->isq_sends_active++; 122 mutex_exit(&qpp->isq_mutex); 123 mutex_enter(&clientp->ic_kstat_mutex); 124 IBMF_ADD32_KSTATS(clientp, sends_active, 1); 125 mutex_exit(&clientp->ic_kstat_mutex); 126 } 127 128 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_issue_pkt_end, 129 IBMF_TNF_TRACE, "", "ibmf_i_issue_pkt() exit\n"); 130 return (ret); 131 } 132 133 /* 134 * ibmf_i_send_pkt() 135 * Send an IB packet after allocating send resources 136 */ 137 int 138 ibmf_i_send_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle, 139 ibmf_msg_impl_t *msgimplp, int block) 140 { 141 ibmf_send_wqe_t *send_wqep; 142 int status; 143 144 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_start, 145 IBMF_TNF_TRACE, "", 146 "ibmf_i_send_pkt(): clientp = 0x%p, qp_hdl = 0x%p, " 147 "msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp, 148 tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp, 149 tnf_uint, block, block); 150 151 ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); 152 153 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) 154 155 /* 156 * Reset send_done to indicate we have not received the completion 157 * for this send yet. 158 */ 159 msgimplp->im_trans_state_flags &= ~IBMF_TRANS_STATE_FLAG_SEND_DONE; 160 161 /* 162 * Allocate resources needed to send a UD packet including the 163 * send WQE context 164 */ 165 status = ibmf_i_alloc_send_resources(clientp->ic_myci, 166 msgimplp, block, &send_wqep); 167 if (status != IBMF_SUCCESS) { 168 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err, 169 IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n", 170 tnf_string, msg, "unable to allocate send resources", 171 tnf_uint, status, status); 172 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end, 173 IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n"); 174 return (status); 175 } 176 177 /* Set the segment number in the send WQE context */ 178 if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP) 179 send_wqep->send_rmpp_segment = msgimplp->im_rmpp_ctx.rmpp_ns; 180 181 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) 182 183 /* 184 * Increment the count of pending send completions. 185 * Only when this count is zero should the client be notified 186 * of completion of the transaction. 187 */ 188 msgimplp->im_pending_send_compls += 1; 189 190 /* Send the packet */ 191 status = ibmf_i_issue_pkt(clientp, msgimplp, ibmf_qp_handle, send_wqep); 192 if (status != IBMF_SUCCESS) { 193 ibmf_i_free_send_resources(clientp->ic_myci, msgimplp, 194 send_wqep); 195 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, ibmf_i_send_pkt_err, 196 IBMF_TNF_ERROR, "", "ibmf_i_send_pkt(): %s, status = %d\n", 197 tnf_string, msg, "unable to issue packet", 198 tnf_uint, status, status); 199 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end, 200 IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit\n"); 201 return (status); 202 } 203 204 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_pkt_end, 205 IBMF_TNF_TRACE, "", "ibmf_i_send_pkt() exit, status = %d\n", 206 tnf_uint, status, status); 207 208 return (IBMF_SUCCESS); 209 } 210 211 /* 212 * ibmf_i_send_single_pkt(): 213 * Send a single IB packet. Only used to send non-RMPP packets. 214 */ 215 int 216 ibmf_i_send_single_pkt(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle, 217 ibmf_msg_impl_t *msgimplp, int block) 218 { 219 int status; 220 221 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_start, 222 IBMF_TNF_TRACE, "", 223 "ibmf_i_send_single_pkt(): clientp = 0x%p, qp_hdl = 0x%p, " 224 "msgp = 0x%p, block = %d\n", tnf_opaque, clientp, clientp, 225 tnf_opaque, qp_hdl, ibmf_qp_handle, tnf_opaque, msg, msgimplp, 226 tnf_uint, block, block); 227 228 ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); 229 230 status = ibmf_i_send_pkt(clientp, ibmf_qp_handle, msgimplp, block); 231 if (status != IBMF_SUCCESS) { 232 IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1, 233 ibmf_i_send_single_pkt_err, IBMF_TNF_ERROR, "", 234 "ibmf_i_send_single_pkt(): %s, msgp = 0x%p\n", 235 tnf_string, msg, "unable to send packet", 236 tnf_uint, status, status); 237 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 238 ibmf_i_send_single_pkt_end, IBMF_TNF_TRACE, "", 239 "ibmf_i_send_single_pkt() exit\n"); 240 return (status); 241 } 242 243 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_send_single_pkt_end, 244 IBMF_TNF_TRACE, "", "ibmf_i_send_single_pkt() exit\n"); 245 return (IBMF_SUCCESS); 246 } 247 248 /* 249 * ibmf_i_handle_send_completion(): 250 * Process the WQE from the SQ identified in the work completion entry. 251 */ 252 /* ARGSUSED */ 253 void 254 ibmf_i_handle_send_completion(ibmf_ci_t *cip, ibt_wc_t *wcp) 255 { 256 ibmf_client_t *clientp, *cclientp; 257 ibmf_send_wqe_t *send_wqep; 258 ibmf_qp_handle_t ibmf_qp_handle; 259 ibmf_alt_qp_t *qpp; 260 int ret; 261 262 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L4, 263 ibmf_i_handle_send_completion_start, IBMF_TNF_TRACE, "", 264 "ibmf_i_handle_send_completion() enter, cip = %p, wcp = %p\n", 265 tnf_opaque, cip, cip, tnf_opaque, wcp, wcp); 266 267 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*send_wqep)) 268 269 ASSERT(wcp->wc_id != 0); 270 271 ASSERT(IBMF_IS_SEND_WR_ID(wcp->wc_id)); 272 273 /* get the IBMF send WQE context */ 274 IBMF_SEND_WR_ID_TO_ADDR(wcp->wc_id, send_wqep); 275 276 ASSERT(send_wqep != NULL); 277 278 /* get the client context */ 279 cclientp = clientp = send_wqep->send_client; 280 281 /* Check if this is a completion for a BUSY MAD sent by IBMF */ 282 if (clientp == NULL) { 283 ibmf_msg_impl_t *msgimplp; 284 285 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L3, 286 ibmf_i_handle_send_completion, IBMF_TNF_TRACE, "", 287 "ibmf_i_handle_send_completion(): NULL client\n"); 288 289 msgimplp = send_wqep->send_msg; 290 291 /* 292 * Deregister registered memory and free it, and 293 * free up the send WQE context 294 */ 295 (void) ibt_deregister_mr(cip->ci_ci_handle, 296 send_wqep->send_mem_hdl); 297 kmem_free(send_wqep->send_mem, IBMF_MEM_PER_WQE); 298 kmem_free(send_wqep, sizeof (ibmf_send_wqe_t)); 299 300 /* Free up the message context */ 301 ibmf_i_put_ud_dest(cip, msgimplp->im_ibmf_ud_dest); 302 ibmf_i_clean_ud_dest_list(cip, B_FALSE); 303 kmem_free(msgimplp, sizeof (ibmf_msg_impl_t)); 304 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 305 ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", 306 "ibmf_i_handle_send_completion() exit\n"); 307 return; 308 } 309 310 /* get the QP handle */ 311 ibmf_qp_handle = send_wqep->send_ibmf_qp_handle; 312 qpp = (ibmf_alt_qp_t *)ibmf_qp_handle; 313 314 ASSERT(clientp != NULL); 315 316 /* decrement the number of active sends */ 317 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 318 mutex_enter(&clientp->ic_mutex); 319 clientp->ic_sends_active--; 320 mutex_exit(&clientp->ic_mutex); 321 } else { 322 mutex_enter(&qpp->isq_mutex); 323 qpp->isq_sends_active--; 324 mutex_exit(&qpp->isq_mutex); 325 } 326 327 mutex_enter(&clientp->ic_kstat_mutex); 328 IBMF_SUB32_KSTATS(clientp, sends_active, 1); 329 mutex_exit(&clientp->ic_kstat_mutex); 330 331 send_wqep->send_status = ibmf_i_ibt_wc_to_ibmf_status(wcp->wc_status); 332 333 /* 334 * issue the callback using taskq. If no taskq or if the 335 * dispatch fails, we do the send processing in the callback context 336 * which is the interrupt context 337 */ 338 if (cclientp->ic_send_taskq == NULL) { 339 /* Do the processing in callback context */ 340 mutex_enter(&clientp->ic_kstat_mutex); 341 IBMF_ADD32_KSTATS(clientp, send_cb_active, 1); 342 mutex_exit(&clientp->ic_kstat_mutex); 343 ibmf_i_do_send_cb((void *)send_wqep); 344 mutex_enter(&clientp->ic_kstat_mutex); 345 IBMF_SUB32_KSTATS(clientp, send_cb_active, 1); 346 mutex_exit(&clientp->ic_kstat_mutex); 347 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, 348 ibmf_i_handle_send_err, IBMF_TNF_ERROR, "", 349 "ibmf_i_handle_send_completion(): %s\n", 350 tnf_string, msg, "ci_send_taskq == NULL"); 351 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 352 ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", 353 "ibmf_i_handle_send_completion() exit\n"); 354 return; 355 } 356 357 mutex_enter(&clientp->ic_kstat_mutex); 358 IBMF_ADD32_KSTATS(clientp, send_cb_active, 1); 359 mutex_exit(&clientp->ic_kstat_mutex); 360 361 /* Use taskq for processing if the IBMF_REG_FLAG_NO_OFFLOAD isn't set */ 362 if ((clientp->ic_reg_flags & IBMF_REG_FLAG_NO_OFFLOAD) == 0) { 363 ret = taskq_dispatch(cclientp->ic_send_taskq, ibmf_i_do_send_cb, 364 send_wqep, TQ_NOSLEEP); 365 if (ret == TASKQID_INVALID) { 366 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, 367 ibmf_i_handle_send_err, IBMF_TNF_ERROR, "", 368 "ibmf_i_handle_send_completion(): %s\n", 369 tnf_string, msg, "send: dispatch failed"); 370 ibmf_i_do_send_cb((void *)send_wqep); 371 } 372 } else { 373 ibmf_i_do_send_cb((void *)send_wqep); 374 } 375 376 mutex_enter(&clientp->ic_kstat_mutex); 377 IBMF_SUB32_KSTATS(clientp, send_cb_active, 1); 378 mutex_exit(&clientp->ic_kstat_mutex); 379 380 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*send_wqep)) 381 382 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 383 ibmf_i_handle_send_completion_end, IBMF_TNF_TRACE, "", 384 "ibmf_i_handle_send_completion() exit\n"); 385 } 386 387 /* 388 * ibmf_i_do_send_cb(): 389 * Do the send completion processing 390 */ 391 static void 392 ibmf_i_do_send_cb(void *taskq_arg) 393 { 394 ibmf_ci_t *cip; 395 ibmf_msg_impl_t *msgimplp; 396 ibmf_client_t *clientp; 397 ibmf_send_wqe_t *send_wqep; 398 boolean_t found; 399 int msg_trans_state_flags, msg_flags; 400 uint_t ref_cnt; 401 ibmf_qp_handle_t ibmf_qp_handle; 402 struct kmem_cache *kmem_cachep; 403 timeout_id_t msg_rp_unset_id, msg_tr_unset_id; 404 timeout_id_t msg_rp_set_id, msg_tr_set_id; 405 ibmf_alt_qp_t *altqp; 406 boolean_t inc_refcnt; 407 408 send_wqep = taskq_arg; 409 410 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L4, 411 ibmf_i_do_send_cb_start, IBMF_TNF_TRACE, "", 412 "ibmf_i_do_send_cb() enter, send_wqep = %p\n", 413 tnf_opaque, send_wqep, send_wqep); 414 415 clientp = send_wqep->send_client; 416 cip = clientp->ic_myci; 417 msgimplp = send_wqep->send_msg; 418 419 /* get the QP handle */ 420 ibmf_qp_handle = send_wqep->send_ibmf_qp_handle; 421 422 /* Get the WQE kmem cache pointer based on the QP type */ 423 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) 424 kmem_cachep = cip->ci_send_wqes_cache; 425 else { 426 altqp = (ibmf_alt_qp_t *)ibmf_qp_handle; 427 kmem_cachep = altqp->isq_send_wqes_cache; 428 } 429 430 /* Look for a message in the client's message list */ 431 inc_refcnt = B_TRUE; 432 found = ibmf_i_find_msg_client(clientp, msgimplp, inc_refcnt); 433 434 /* 435 * If the message context was not found, then it's likely 436 * been freed up. So, do nothing in this timeout handler 437 */ 438 if (found == B_FALSE) { 439 kmem_cache_free(kmem_cachep, send_wqep); 440 mutex_enter(&cip->ci_mutex); 441 IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); 442 mutex_exit(&cip->ci_mutex); 443 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 444 mutex_enter(&cip->ci_mutex); 445 cip->ci_wqes_alloced--; 446 if (cip->ci_wqes_alloced == 0) 447 cv_signal(&cip->ci_wqes_cv); 448 mutex_exit(&cip->ci_mutex); 449 } else { 450 mutex_enter(&altqp->isq_mutex); 451 altqp->isq_wqes_alloced--; 452 if (altqp->isq_wqes_alloced == 0) 453 cv_signal(&altqp->isq_wqes_cv); 454 mutex_exit(&altqp->isq_mutex); 455 } 456 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 457 ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", 458 "ibmf_i_do_send_cb(): %s\n", tnf_string, msg, 459 "Message not found, return without processing send cb"); 460 return; 461 } 462 463 /* Grab the message context lock */ 464 mutex_enter(&msgimplp->im_mutex); 465 466 /* 467 * Decrement the count of pending send completions for 468 * this transaction 469 */ 470 msgimplp->im_pending_send_compls -= 1; 471 472 /* 473 * If the pending send completions is not zero, then we must 474 * not attempt to notify the client of a transaction completion 475 * in this instance of the send completion handler. Notification 476 * of transaction completion should be provided only by the 477 * last send completion so that all send completions are accounted 478 * for before the client is notified and subsequently attempts to 479 * reuse the message for an other transaction. 480 * If this is not done, the message may be reused while the 481 * send WR from the old transaction is still active in the QP's WQ. 482 * This could result in an attempt to modify the address handle with 483 * information for the new transaction which could be potentially 484 * incompatible, such as an incorrect port number. Such an 485 * incompatible modification of the address handle of the old 486 * transaction could result in a QP error. 487 */ 488 if (msgimplp->im_pending_send_compls != 0) { 489 IBMF_MSG_DECR_REFCNT(msgimplp); 490 mutex_exit(&msgimplp->im_mutex); 491 kmem_cache_free(kmem_cachep, send_wqep); 492 mutex_enter(&cip->ci_mutex); 493 IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); 494 mutex_exit(&cip->ci_mutex); 495 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 496 mutex_enter(&cip->ci_mutex); 497 cip->ci_wqes_alloced--; 498 if (cip->ci_wqes_alloced == 0) 499 cv_signal(&cip->ci_wqes_cv); 500 mutex_exit(&cip->ci_mutex); 501 } else { 502 mutex_enter(&altqp->isq_mutex); 503 altqp->isq_wqes_alloced--; 504 if (altqp->isq_wqes_alloced == 0) 505 cv_signal(&altqp->isq_wqes_cv); 506 mutex_exit(&altqp->isq_mutex); 507 } 508 IBMF_TRACE_1(IBMF_TNF_DEBUG, DPRINT_L3, 509 ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", 510 "ibmf_i_do_send_cb(): %s\n", tnf_string, msg, 511 "Message found with pending send completions, " 512 "return without processing send cb"); 513 return; 514 } 515 516 /* 517 * If the message has been marked unitialized or done 518 * release the message mutex and return 519 */ 520 if ((msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_UNINIT) || 521 (msgimplp->im_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE)) { 522 IBMF_MSG_DECR_REFCNT(msgimplp); 523 msg_trans_state_flags = msgimplp->im_trans_state_flags; 524 msg_flags = msgimplp->im_flags; 525 ref_cnt = msgimplp->im_ref_count; 526 mutex_exit(&msgimplp->im_mutex); 527 /* 528 * This thread may notify the client only if the 529 * transaction is done, the message has been removed 530 * from the client's message list, and the message 531 * reference count is 0. 532 * If the transaction is done, and the message reference 533 * count = 0, there is still a possibility that a 534 * packet could arrive for the message and its reference 535 * count increased if the message is still on the list. 536 * If the message is still on the list, it will be 537 * removed by a call to ibmf_i_client_rem_msg() at 538 * the completion point of the transaction. 539 * So, the reference count should be checked after the 540 * message has been removed. 541 */ 542 if ((msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) && 543 !(msg_flags & IBMF_MSG_FLAGS_ON_LIST) && 544 (ref_cnt == 0)) { 545 546 ibmf_i_notify_sequence(clientp, msgimplp, msg_flags); 547 548 } 549 kmem_cache_free(kmem_cachep, send_wqep); 550 mutex_enter(&cip->ci_mutex); 551 IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); 552 mutex_exit(&cip->ci_mutex); 553 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 554 mutex_enter(&cip->ci_mutex); 555 cip->ci_wqes_alloced--; 556 if (cip->ci_wqes_alloced == 0) 557 cv_signal(&cip->ci_wqes_cv); 558 mutex_exit(&cip->ci_mutex); 559 } else { 560 mutex_enter(&altqp->isq_mutex); 561 altqp->isq_wqes_alloced--; 562 if (altqp->isq_wqes_alloced == 0) 563 cv_signal(&altqp->isq_wqes_cv); 564 mutex_exit(&altqp->isq_mutex); 565 } 566 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 567 ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", 568 "ibmf_i_do_send_cb(): %s, msg = %p\n", tnf_string, msg, 569 "Message marked for removal, return without processing " 570 "send cb", tnf_opaque, msgimplp, msgimplp); 571 return; 572 } 573 574 /* Perform send completion processing of the message context */ 575 ibmf_i_do_send_compl((ibmf_handle_t)clientp, msgimplp, send_wqep); 576 577 msg_rp_unset_id = msg_tr_unset_id = msg_rp_set_id = msg_tr_set_id = 0; 578 579 /* Save the message flags before releasing the mutex */ 580 msg_trans_state_flags = msgimplp->im_trans_state_flags; 581 msg_flags = msgimplp->im_flags; 582 msg_rp_unset_id = msgimplp->im_rp_unset_timeout_id; 583 msg_tr_unset_id = msgimplp->im_tr_unset_timeout_id; 584 msgimplp->im_rp_unset_timeout_id = 0; 585 msgimplp->im_tr_unset_timeout_id = 0; 586 587 /* 588 * Decrement the message reference count 589 * This count was inceremented when the message was found on the 590 * client's message list 591 */ 592 IBMF_MSG_DECR_REFCNT(msgimplp); 593 594 if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) { 595 if (msgimplp->im_rp_timeout_id != 0) { 596 msg_rp_set_id = msgimplp->im_rp_timeout_id; 597 msgimplp->im_rp_timeout_id = 0; 598 } 599 if (msgimplp->im_tr_timeout_id != 0) { 600 msg_tr_set_id = msgimplp->im_tr_timeout_id; 601 msgimplp->im_tr_timeout_id = 0; 602 } 603 } 604 605 mutex_exit(&msgimplp->im_mutex); 606 607 if (msg_rp_unset_id != 0) { 608 (void) untimeout(msg_rp_unset_id); 609 } 610 611 if (msg_tr_unset_id != 0) { 612 (void) untimeout(msg_tr_unset_id); 613 } 614 615 if (msg_rp_set_id != 0) { 616 (void) untimeout(msg_rp_set_id); 617 } 618 619 if (msg_tr_set_id != 0) { 620 (void) untimeout(msg_tr_set_id); 621 } 622 623 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 624 ibmf_i_do_send_cb, IBMF_TNF_TRACE, "", 625 "ibmf_i_do_send_cb(): %s, msg = %p\n", 626 tnf_string, msg, "Send callback done. Dec ref count", 627 tnf_opaque, msgimplp, msgimplp); 628 629 /* 630 * If the transaction is done, signal the block thread if the 631 * transaction is blocking, or call the client's transaction done 632 * notification callback 633 */ 634 if (msg_trans_state_flags & IBMF_TRANS_STATE_FLAG_DONE) { 635 636 /* Remove the message from the client's message list */ 637 ibmf_i_client_rem_msg(clientp, msgimplp, &ref_cnt); 638 639 /* 640 * Notify the client if the message reference count is zero. 641 * At this point, we know that the transaction is done and 642 * the message has been removed from the client's message list. 643 * So, we only need to make sure the reference count is zero 644 * before notifying the client. 645 */ 646 if (ref_cnt == 0) { 647 648 ibmf_i_notify_sequence(clientp, msgimplp, msg_flags); 649 650 } 651 } 652 653 kmem_cache_free(kmem_cachep, send_wqep); 654 mutex_enter(&cip->ci_mutex); 655 IBMF_SUB32_PORT_KSTATS(cip, send_wqes_alloced, 1); 656 mutex_exit(&cip->ci_mutex); 657 if (ibmf_qp_handle == IBMF_QP_HANDLE_DEFAULT) { 658 mutex_enter(&cip->ci_mutex); 659 cip->ci_wqes_alloced--; 660 if (cip->ci_wqes_alloced == 0) 661 cv_signal(&cip->ci_wqes_cv); 662 mutex_exit(&cip->ci_mutex); 663 } else { 664 mutex_enter(&altqp->isq_mutex); 665 altqp->isq_wqes_alloced--; 666 if (altqp->isq_wqes_alloced == 0) 667 cv_signal(&altqp->isq_wqes_cv); 668 mutex_exit(&altqp->isq_mutex); 669 } 670 671 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 672 ibmf_i_do_send_cb_end, IBMF_TNF_TRACE, "", 673 "ibmf_i_do_send_cb() exit\n"); 674 } 675 676 /* 677 * ibmf_i_do_send_compl(): 678 * Determine if the transaction is complete 679 */ 680 /* ARGSUSED */ 681 static void 682 ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp, 683 ibmf_send_wqe_t *send_wqep) 684 { 685 IBMF_TRACE_4(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_start, 686 IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl(): ibmf_hdl = 0x%p " 687 "msgp = %p, send_wqep = 0x%p, msg_flags = 0x%x\n", 688 tnf_opaque, ibmf_hdl, ibmf_handle, tnf_opaque, msgimplp, msgimplp, 689 tnf_opaque, send_wqep, send_wqep, 690 tnf_opaque, msg_flags, msgimplp->im_flags); 691 692 ASSERT(MUTEX_HELD(&msgimplp->im_mutex)); 693 694 /* 695 * For RMPP transactions, we only care about the final packet of the 696 * transaction. For others, the code does not need to wait for the send 697 * completion (although bad things can happen if it never occurs). 698 * The final packets of a transaction are sent when the state is either 699 * ABORT or RECEVR_TERMINATE. 700 * Don't mark the transaction as send_done if there are still more 701 * packets to be sent, including doing the second part of a double-sided 702 * transaction. 703 */ 704 if ((msgimplp->im_flags & IBMF_MSG_FLAGS_RECV_RMPP) || 705 (msgimplp->im_flags & IBMF_MSG_FLAGS_SEND_RMPP)) { 706 707 IBMF_TRACE_3(IBMF_TNF_DEBUG, DPRINT_L3, 708 ibmf_i_do_send_compl, IBMF_TNF_TRACE, "", 709 "ibmf_i_do_send_compl(): %s msgp = %p, rmpp_state = 0x%x\n", 710 tnf_string, msg, "Received send callback for RMPP trans", 711 tnf_opaque, msg, msgimplp, 712 tnf_opaque, rmpp_state, msgimplp->im_rmpp_ctx.rmpp_state); 713 714 /* 715 * For ABORT state, we should not return control to 716 * the client from the send completion handler. 717 * Control should be returned in the error timeout handler. 718 * 719 * The exception is when the IBMF_TRANS_STATE_FLAG_RECV_DONE 720 * flag has already been set. This flag is set when 721 * ibmf_i_terminate_transaction is called from one of the 722 * three timeout handlers. In this case return control from 723 * here. 724 */ 725 if (msgimplp->im_rmpp_ctx.rmpp_state == IBMF_RMPP_STATE_ABORT) { 726 msgimplp->im_trans_state_flags |= 727 IBMF_TRANS_STATE_FLAG_SEND_DONE; 728 if (msgimplp->im_trans_state_flags & 729 IBMF_TRANS_STATE_FLAG_RECV_DONE) { 730 msgimplp->im_trans_state_flags |= 731 IBMF_TRANS_STATE_FLAG_DONE; 732 } 733 } 734 735 if ((msgimplp->im_rmpp_ctx.rmpp_state == 736 IBMF_RMPP_STATE_RECEVR_TERMINATE) || 737 (msgimplp->im_rmpp_ctx.rmpp_state == 738 IBMF_RMPP_STATE_DONE)) { 739 msgimplp->im_trans_state_flags |= 740 IBMF_TRANS_STATE_FLAG_SEND_DONE; 741 if (msgimplp->im_trans_state_flags & 742 IBMF_TRANS_STATE_FLAG_RECV_DONE) { 743 msgimplp->im_trans_state_flags |= 744 IBMF_TRANS_STATE_FLAG_DONE; 745 } 746 } 747 748 /* 749 * If the transaction is a send-only RMPP, then 750 * set the SEND_DONE flag on every send completion 751 * as long as there are no outstanding ones. 752 * This is needed so that the transaction can return 753 * in the receive path, where ibmf_i_terminate_transaction 754 * is called from ibmf_i_rmpp_sender_active_flow, 755 * after checking if the SEND_DONE flag is set. 756 * When a new MAD is sent as part of the RMPP transaction, 757 * the SEND_DONE flag will get reset. 758 * The RECV_DONE indicates that the last ACK was received. 759 */ 760 if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) == 0) { 761 if (msgimplp->im_pending_send_compls == 0) { 762 msgimplp->im_trans_state_flags |= 763 IBMF_TRANS_STATE_FLAG_SEND_DONE; 764 if (msgimplp->im_trans_state_flags & 765 IBMF_TRANS_STATE_FLAG_RECV_DONE) { 766 msgimplp->im_trans_state_flags |= 767 IBMF_TRANS_STATE_FLAG_DONE; 768 } 769 } 770 } 771 772 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 773 ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", 774 "ibmf_i_do_send_compl() exit\n"); 775 return; 776 } 777 778 /* 779 * Only non-RMPP send completion gets here. 780 * If the send is a single-packet send that does not use RMPP, and if 781 * the transaction is not a sequenced transaction, call the transaction 782 * callback handler after flagging the transaction as done. If the 783 * message is sequenced, start a timer to bound the wait for the first 784 * data packet of the response. 785 */ 786 if (msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) { 787 788 IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, 789 ibmf_i_do_send_compl, IBMF_TNF_TRACE, "", 790 "ibmf_i_do_send_compl(): %s msgp = %p\n", tnf_string, msg, 791 "Sequenced transaction, setting response timer", 792 tnf_opaque, msg, msgimplp); 793 794 /* 795 * Check if the send completion already occured, 796 * which could imply that this is a send completion 797 * for some previous transaction that has come in very late. 798 * In this case exit here. 799 */ 800 if (msgimplp->im_trans_state_flags & 801 IBMF_TRANS_STATE_FLAG_SEND_DONE) { 802 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 803 ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", 804 "ibmf_i_do_send_compl() exit, " 805 "Duplicate SEND completion\n"); 806 return; 807 } 808 809 /* mark as send_compl happened */ 810 msgimplp->im_trans_state_flags |= 811 IBMF_TRANS_STATE_FLAG_SEND_DONE; 812 813 if (msgimplp->im_trans_state_flags & 814 IBMF_TRANS_STATE_FLAG_RECV_DONE) { 815 msgimplp->im_trans_state_flags |= 816 IBMF_TRANS_STATE_FLAG_DONE; 817 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, 818 ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "", 819 "ibmf_i_do_send_compl() exit, RECV_DONE\n"); 820 return; 821 } 822 823 /* 824 * check if response was received before send 825 * completion 826 */ 827 if (((msgimplp->im_trans_state_flags & 828 IBMF_TRANS_STATE_FLAG_DONE) == 0) && 829 ((msgimplp->im_trans_state_flags & 830 IBMF_TRANS_STATE_FLAG_RECV_ACTIVE) == 0)) { 831 /* set timer for first packet of response */ 832 ibmf_i_set_timer(ibmf_i_send_timeout, msgimplp, 833 IBMF_RESP_TIMER); 834 } 835 } else { 836 msgimplp->im_msg_status = IBMF_SUCCESS; 837 msgimplp->im_trans_state_flags |= 838 IBMF_TRANS_STATE_FLAG_SEND_DONE; 839 msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE; 840 } 841 842 IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_i_do_send_compl_end, 843 IBMF_TNF_TRACE, "", "ibmf_i_do_send_compl() exit\n"); 844 } 845