1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2017 Joyent, Inc. 25 */ 26 27 /* 28 * sol_uverbs_event.c 29 * 30 * OFED User Verbs Kernel Async Event funtions 31 * 32 */ 33 #include <sys/file.h> 34 #include <sys/fcntl.h> 35 #include <sys/vfs.h> 36 #include <sys/errno.h> 37 #include <sys/cred.h> 38 #include <sys/uio.h> 39 #include <sys/semaphore.h> 40 #include <sys/ddi.h> 41 #include <sys/sunddi.h> 42 43 #include <sys/ib/ibtl/ibvti.h> 44 #include <sys/ib/clients/of/ofa_solaris.h> 45 #include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h> 46 #include <sys/ib/clients/of/ofed_kernel.h> 47 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h> 48 #include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h> 49 50 extern char *sol_uverbs_dbg_str; 51 52 static void 53 uverbs_async_event_common(uverbs_uctxt_uobj_t *, uint64_t, uint32_t, 54 llist_head_t *, uint32_t *); 55 56 /* 57 * Function: 58 * sol_uverbs_event_file_close 59 * Input: 60 * ufile - Pointer to the event ufile 61 * 62 * Output: 63 * None 64 * Returns: 65 * Zero on success, else error code. 66 * Description: 67 * Called when all kernel references to the event file have been 68 * removed and the kernel (asynchronous) or user library (completion) 69 * have closed the file. 70 */ 71 void 72 sol_uverbs_event_file_close(uverbs_ufile_uobj_t *ufile) 73 { 74 if (!ufile) { 75 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 76 "UFILE CLOSE: Ufile NULL\n"); 77 return; 78 } 79 80 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 81 "UFILE CLOSE: Is async? %s", 82 ufile->is_async ? "yes" : "no"); 83 84 /* 85 * Remove the user file from the user object table and 86 * releases appropriate references. The object resources 87 * are freed when it is no longer referenced. 88 * 89 * If sol_ofs_uobj_remove() returns NULL then the obj was already 90 * removed. 91 */ 92 rw_enter(&(ufile->uobj.uo_lock), RW_WRITER); 93 if (sol_ofs_uobj_remove(&uverbs_ufile_uo_tbl, &ufile->uobj)) { 94 rw_exit(&(ufile->uobj.uo_lock)); 95 sol_ofs_uobj_deref(&ufile->uobj, uverbs_release_event_file); 96 } else { 97 rw_exit(&(ufile->uobj.uo_lock)); 98 } 99 } 100 101 /* 102 * Function: 103 * sol_uverbs_event_file_read 104 * Input: 105 * ufile - The user file pointer of the event channel. 106 * uiop - The user I/O pointer in which to place the event. 107 * cred - Pointer to the callers credentials. 108 * Output: 109 * uiop - Upon success the caller's buffer has been updated with 110 * the event details. 111 * Returns: 112 * Zero on success, else error code. 113 * EAGAIN - No event available and caller is non- 114 * blocking. 115 * ERESTART- A signal was received. 116 * EINVAL - Caller parameter/user buffer invalid. 117 * EFAULT - Failure copying data to user. 118 * Description: 119 * Perfrom a blocking read to retrieve an event (asynchronous or 120 * completion) for the user file specified. If an event is available it 121 * is immediately returned. If an event is not available, then the 122 * caller will block unless the open flags indicate it is a non- 123 * blocking open. If the caller does block it does so interruptable 124 * and therefore will return upon an event or receipt of a signal. 125 */ 126 /* ARGSUSED */ 127 int 128 sol_uverbs_event_file_read(uverbs_ufile_uobj_t *ufile, struct uio *uiop, 129 cred_t *cred) 130 { 131 int rc = 0; 132 uverbs_event_t *evt; 133 llist_head_t *entry; 134 int eventsz; 135 int ioflag; 136 137 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read(%p), " 138 "ufile = %p, is_async =%d, uio_resid=%d", 139 ufile, ufile->is_async, uiop->uio_resid); 140 141 ioflag = uiop->uio_fmode & (FNONBLOCK | FNDELAY); 142 143 mutex_enter(&ufile->lock); 144 145 /* 146 * If Event list not empty and CQ event notification is disabled 147 * by sol_ucma, do not return events. Either return EAGAIN (if 148 * flag is O_NONBLOCK, or wait using cv_wait_sig(). 149 */ 150 if (ufile->ufile_notify_enabled == SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE && 151 llist_empty(&ufile->event_list) != 0) { 152 if (ioflag) { 153 mutex_exit(&ufile->lock); 154 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 155 "event_file_read - notify disabled, no block"); 156 return (EAGAIN); 157 } 158 159 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) { 160 mutex_exit(&ufile->lock); 161 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 162 "event_file_read - sig_wakeup"); 163 return (ERESTART); 164 } 165 } 166 167 while (llist_empty(&ufile->event_list)) { 168 if (ioflag) { 169 mutex_exit(&ufile->lock); 170 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 171 "event_file_read - no events, no block"); 172 return (EAGAIN); 173 } 174 175 if (!cv_wait_sig(&ufile->poll_wait, &ufile->lock)) { 176 mutex_exit(&ufile->lock); 177 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 178 "event_file_read - sig_wakeup"); 179 return (ERESTART); 180 } 181 } 182 183 entry = ufile->event_list.nxt; 184 evt = entry->ptr; 185 186 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_read: " 187 "Event entry found: entry:%p, event:%p, evt_list %p", 188 entry, evt, &evt->ev_list); 189 190 if (ufile->is_async) { 191 eventsz = sizeof (struct ib_uverbs_async_event_desc); 192 } else { 193 eventsz = sizeof (struct ib_uverbs_comp_event_desc); 194 } 195 196 if (eventsz > uiop->uio_resid) { 197 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 198 "event_file_read - Event too big"); 199 rc = EINVAL; 200 evt = NULL; 201 } else { 202 llist_del(ufile->event_list.nxt); 203 if (evt->ev_counter) { 204 ++(*evt->ev_counter); 205 llist_del(&evt->ev_obj_list); 206 } 207 } 208 209 mutex_exit(&ufile->lock); 210 211 if (evt && (uiomove(evt, eventsz, UIO_READ, uiop) != 0)) { 212 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 213 "EVENT FILE READ: Error writing ev"); 214 rc = EFAULT; 215 } 216 217 if (evt) { 218 kmem_free(evt, sizeof (*evt)); 219 } 220 221 return (rc); 222 } 223 224 /* 225 * Function: 226 * sol_uverbs_event_file_poll 227 * Input: 228 * ufile - user file for desired completion channel event file 229 * events - The events that may occur. 230 * anyyet - A flag that is non-zero if any files in the set 231 * of descriptors has an event waiting. 232 * ct - Pointer to the callers context. 233 * Output: 234 * reventssp - A pointer updated to return a bitmask of events specified. 235 * phpp - A pointer to a pollhead pointer, updated to reflect the 236 * the event file's pollhead used for synchronization. 237 * Returns: 238 * Zero on success, else error code. 239 * EINVAL - Vnode does not point to valid event file. 240 * Description: 241 * Support for event channel polling interface, allows use of completion 242 * channel in asynchronous type environment. If events may be read 243 * without blocking indicate a POLLIN | POLLRDNORM event; otherwise if 244 * no other descriptors in the set have data waiting, set the pollhead 245 * pointer to our the associated completion event file's pollhead. 246 */ 247 int 248 sol_uverbs_event_file_poll(uverbs_ufile_uobj_t *ufile, short events, 249 int anyyet, short *reventsp, pollhead_t **phpp) 250 { 251 short revent = 0; 252 253 #ifdef DEBUG 254 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll(%p, %x)", 255 ufile, events); 256 #endif 257 258 if (!ufile) { 259 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "event_file_poll ", 260 "ufile %p", ufile); 261 return (EINVAL); 262 } 263 264 #ifdef DEBUG 265 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 266 "ufile = %p, is_async =%d", ufile, ufile->is_async); 267 #endif 268 269 mutex_enter(&ufile->lock); 270 271 /* 272 * If poll request and event is ready. 273 */ 274 if ((events & (POLLIN | POLLRDNORM)) && 275 !llist_empty(&ufile->event_list)) { 276 277 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 278 "Event entry available"); 279 280 revent |= POLLIN | POLLRDNORM; 281 } 282 283 /* 284 * If we didn't get an event or are edge-triggered 285 */ 286 if ((revent == 0 && !anyyet) || (events & POLLET)) { 287 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "event_file_poll " 288 "Event entry NOT available"); 289 290 *phpp = &ufile->poll_head; 291 } 292 293 mutex_exit(&ufile->lock); 294 *reventsp = revent; 295 return (0); 296 } 297 298 /* 299 * Function: 300 * uverbs_alloc_event_file 301 * Input: 302 * uctxt - The Solaris User Verbs user context associated with the 303 * event channel. 304 * is_async - Indicates the file is for asynchronous events if non-zero; 305 * other wise it is for completion events. 306 * Output: 307 * None. 308 * Returns: 309 * New user verb event file object or NULL on error. 310 * Description: 311 * Allocate an asynchronous or completion event file 312 */ 313 uverbs_ufile_uobj_t * 314 uverbs_alloc_event_file(uverbs_uctxt_uobj_t *uctxt, int is_async) 315 { 316 uverbs_ufile_uobj_t *ufile; 317 318 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_event_file(%p, %x)", 319 uctxt, is_async); 320 321 ufile = kmem_zalloc(sizeof (*ufile), KM_NOSLEEP); 322 if (!ufile) { 323 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 324 "alloc_event_file: mem alloc fail"); 325 return (NULL); 326 } 327 ufile->ufile_notify_enabled = SOL_UVERBS2UCMA_CQ_NOTIFY_ENABLE; 328 sol_ofs_uobj_init(&ufile->uobj, 0, SOL_UVERBS_UFILE_UOBJ_TYPE); 329 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 330 331 if (sol_ofs_uobj_add(&uverbs_ufile_uo_tbl, &ufile->uobj) != 0) { 332 /* 333 * The initialization routine set's the initial reference, 334 * we dereference the object here to clean it up. 335 */ 336 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 337 "ALLOC UFILE: Object add failed"); 338 rw_exit(&ufile->uobj.uo_lock); 339 ufile->uobj.uo_uobj_sz = sizeof (uverbs_ufile_uobj_t); 340 sol_ofs_uobj_deref(&ufile->uobj, sol_ofs_uobj_free); 341 return (NULL); 342 } 343 344 ufile->is_async = is_async ? 1 : 0; 345 llist_head_init(&ufile->event_list, NULL); 346 347 mutex_init(&ufile->lock, NULL, MUTEX_DRIVER, NULL); 348 cv_init(&ufile->poll_wait, NULL, CV_DRIVER, NULL); 349 350 ufile->uctxt = uctxt; 351 ufile->uobj.uo_live = 1; 352 rw_exit(&ufile->uobj.uo_lock); 353 return (ufile); 354 } 355 356 /* 357 * Function: 358 * uverbs_release_event_file 359 * Input: 360 * ufile - Pointer to the ufile user object that is being freed. 361 * Output: 362 * None. 363 * Returns: 364 * None. 365 * Description: 366 * Release/destroy event file resources before freeing memory. This 367 * routine should only be used if the event file is successfully 368 * created. 369 */ 370 void 371 uverbs_release_event_file(sol_ofs_uobj_t *uobj) 372 { 373 uverbs_ufile_uobj_t *ufile = (uverbs_ufile_uobj_t *)uobj; 374 uverbs_event_t *evt; 375 llist_head_t *entry; 376 llist_head_t *list_tmp; 377 378 if (!ufile) { 379 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 380 "UFILE RELEASE: Ufile NULL\n"); 381 return; 382 } 383 384 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 385 "UFILE RELEASE: Event file=%p, is async = %s", 386 ufile, ufile->is_async ? "yes" : "no"); 387 388 /* 389 * Release any events still queued to the event file. 390 */ 391 mutex_enter(&ufile->lock); 392 393 entry = ufile->event_list.nxt; 394 list_tmp = entry->nxt; 395 while (entry != &ufile->event_list) { 396 ASSERT(entry); 397 evt = (uverbs_event_t *)entry->ptr; 398 399 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 400 "UFILE RELEASE: Deleting event %p on event file %p", 401 evt, ufile); 402 403 llist_del(&evt->ev_list); 404 kmem_free(evt, sizeof (*evt)); 405 entry = list_tmp; 406 list_tmp = entry->nxt; 407 } 408 409 mutex_exit(&ufile->lock); 410 411 cv_destroy(&ufile->poll_wait); 412 mutex_destroy(&ufile->lock); 413 sol_ofs_uobj_free(uobj); 414 } 415 416 /* 417 * Function: 418 * uverbs_ibt_to_ofa_event_code 419 * Input: 420 * code - The OFA event code. 421 * Output: 422 * The OFED event code. 423 * Returns: 424 * Returns the OFA equivalent of an IBT Asynchronous Event code, -1 if 425 * a valid translation does not exist. 426 * Description: 427 * Map an IBT asynchronous event code to an OFED event code. 428 */ 429 enum ib_event_type 430 uverbs_ibt_to_ofa_event_code(ibt_async_code_t code) 431 { 432 enum ib_event_type ofa_code; 433 434 switch (code) { 435 case IBT_EVENT_PATH_MIGRATED: 436 ofa_code = IB_EVENT_PATH_MIG; 437 break; 438 439 case IBT_EVENT_SQD: 440 ofa_code = IB_EVENT_SQ_DRAINED; 441 break; 442 443 case IBT_EVENT_COM_EST: 444 ofa_code = IB_EVENT_COMM_EST; 445 break; 446 447 case IBT_ERROR_CATASTROPHIC_CHAN: 448 case IBT_ERROR_LOCAL_CATASTROPHIC: 449 ofa_code = IB_EVENT_QP_FATAL; 450 break; 451 452 case IBT_ERROR_INVALID_REQUEST_CHAN: 453 ofa_code = IB_EVENT_QP_REQ_ERR; 454 break; 455 456 case IBT_ERROR_ACCESS_VIOLATION_CHAN: 457 ofa_code = IB_EVENT_QP_ACCESS_ERR; 458 break; 459 460 case IBT_ERROR_PATH_MIGRATE_REQ: 461 ofa_code = IB_EVENT_PATH_MIG_ERR; 462 break; 463 464 case IBT_ERROR_CQ: 465 ofa_code = IB_EVENT_CQ_ERR; 466 break; 467 468 case IBT_EVENT_PORT_UP: 469 ofa_code = IB_EVENT_PORT_ACTIVE; 470 break; 471 472 case IBT_ERROR_PORT_DOWN: 473 ofa_code = IB_EVENT_PORT_ERR; 474 break; 475 476 case IBT_HCA_ATTACH_EVENT: 477 ofa_code = IB_EVENT_CLIENT_REREGISTER; 478 break; 479 480 case IBT_EVENT_LIMIT_REACHED_SRQ: 481 ofa_code = IB_EVENT_SRQ_LIMIT_REACHED; 482 break; 483 484 case IBT_ERROR_CATASTROPHIC_SRQ: 485 ofa_code = IB_EVENT_SRQ_ERR; 486 break; 487 488 case IBT_EVENT_EMPTY_CHAN: 489 ofa_code = IB_EVENT_QP_LAST_WQE_REACHED; 490 break; 491 492 /* 493 * No mapping exists. 494 */ 495 case IBT_ASYNC_OPAQUE1: 496 case IBT_ASYNC_OPAQUE2: 497 case IBT_ASYNC_OPAQUE3: 498 case IBT_ASYNC_OPAQUE4: 499 case IBT_HCA_DETACH_EVENT: 500 default: 501 ofa_code = -1; 502 } 503 504 return (ofa_code); 505 } 506 507 /* 508 * Function: 509 * uverbs_async_qp_event_handler 510 * Input: 511 * clnt_private - The IBT attach client handle. 512 * hca_hdl - The IBT hca handle associated with the notification. 513 * code - The OFED event identifier. 514 * event - The IBT event. 515 * Output: 516 * None 517 * Returns: 518 * None 519 * Description: 520 * Handle QP affiliated asynchronous event noficiation. 521 */ 522 /* ARGSUSED */ 523 void 524 uverbs_async_qp_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 525 enum ib_event_type code, ibt_async_event_t *event) 526 { 527 uverbs_uqp_uobj_t *uqp; 528 529 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 530 "async_qp_event_handler()"); 531 532 if (event->ev_chan_hdl == NULL) { 533 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 534 "async_qp_event_handler: event handle NULL"); 535 return; 536 } 537 uqp = ibt_get_qp_private(event->ev_chan_hdl); 538 ASSERT(uqp); 539 if (uqp->uqp_free_state == SOL_UVERBS2UCMA_FREE_PENDING) { 540 SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str, 541 "async_qp_event_handler: User QP context has been freed"); 542 return; 543 } 544 if (uqp->qp != event->ev_chan_hdl) { 545 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 546 "async_qp_event_handler: QP handle mismatch"); 547 return; 548 } 549 550 uverbs_async_event_common(uqp->uctxt, uqp->uobj.uo_user_handle, 551 code, &uqp->async_list, &uqp->async_events_reported); 552 553 } 554 555 /* 556 * Function: 557 * uverbs_async_cq_event_handler 558 * Input: 559 * clnt_private - The IBT attach client handle. 560 * hca_hdl - The IBT hca handle associated with the notification. 561 * code - The OFED event identifier. 562 * event - The IBT event. 563 * Output: 564 * None 565 * Returns: 566 * None 567 * Description: 568 * Handle a CQ affiliated asynchronous event notification. 569 */ 570 /* ARGSUSED */ 571 void 572 uverbs_async_cq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 573 enum ib_event_type code, ibt_async_event_t *event) 574 { 575 uverbs_ucq_uobj_t *ucq; 576 577 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 578 "ASYNC CQ EVENT HANDLER:"); 579 580 if (event->ev_cq_hdl == NULL) { 581 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 582 "ASYNC CQ EVENT HANDLER: event handle is NULL"); 583 return; 584 } 585 586 ucq = ibt_get_cq_private(event->ev_cq_hdl); 587 if (ucq->cq != event->ev_cq_hdl) { 588 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 589 "ASYNC CQ EVENT HANDLER: CQ handle mismatch"); 590 return; 591 } 592 593 uverbs_async_event_common(ucq->uctxt, ucq->uobj.uo_user_handle, 594 code, &ucq->async_list, &ucq->async_events_reported); 595 } 596 597 /* 598 * Function: 599 * uverbs_async_srq_event_handler 600 * Input: 601 * clnt_private - The IBT attach client handle. 602 * hca_hdl - The IBT hca handle associated with the notification. 603 * code - The OFED event identifier. 604 * event - The IBT event. 605 * Output: 606 * None 607 * Returns: 608 * None 609 * Description: 610 * Handle a shared receive queue asynchronous event notification. 611 */ 612 /* ARGSUSED */ 613 void 614 uverbs_async_srq_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 615 enum ib_event_type code, ibt_async_event_t *event) 616 { 617 uverbs_usrq_uobj_t *usrq; 618 619 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 620 "ASYNC SRQ EVENT HANDLER:"); 621 622 if (event->ev_srq_hdl == NULL) { 623 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 624 "ASYNC SRQ EVENT HANDLER: event handle is NULL"); 625 return; 626 } 627 628 usrq = ibt_get_srq_private(event->ev_srq_hdl); 629 if (usrq->srq != event->ev_srq_hdl) { 630 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 631 "ASYNC SRQ EVENT HANDLER: SRQ handle mismatch"); 632 return; 633 } 634 635 uverbs_async_event_common(usrq->uctxt, usrq->uobj.uo_user_handle, 636 code, &usrq->async_list, &usrq->async_events_reported); 637 } 638 639 /* 640 * Function: 641 * uverbs_async_unaff_event_handler 642 * Input: 643 * clnt_private - The IBT attach client handle. 644 * hca_hdl - The IBT hca handle associated with the notification. 645 * code - The OFED event identifier. 646 * event - The IBT event. 647 * Output: 648 * None 649 * Returns: 650 * None 651 * Description: 652 * Handle an unaffiliated asynchronous event notification. 653 */ 654 /* ARGSUSED */ 655 void 656 uverbs_async_unaff_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 657 enum ib_event_type code, ibt_async_event_t *event) 658 { 659 sol_ofs_uobj_table_t *uo_tbl = &uverbs_uctxt_uo_tbl; 660 sol_ofs_uobj_blk_t *blk; 661 uverbs_uctxt_uobj_t *uctxt; 662 int i, j; 663 664 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 665 "ASYNC UNAFF EVENT HANDLER:"); 666 667 /* 668 * Unaffiliated events are returned at the IBT client level. We must 669 * return the event to all user context allocated for the specific 670 * HCA device specified. 671 */ 672 rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER); 673 674 for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) { 675 blk = uo_tbl->uobj_tbl_uo_root[i]; 676 if (blk == NULL) { 677 continue; 678 } 679 for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) { 680 uctxt = (uverbs_uctxt_uobj_t *)blk->ofs_uoblk_blks[j]; 681 if (uctxt == NULL) { 682 continue; 683 } 684 /* 685 * OK, check to see if this user context belongs 686 * to the idicated hca. 687 */ 688 if (uctxt->hca->hdl == hca_hdl && uctxt->async_evfile) { 689 uverbs_async_event_common(uctxt, 690 event->ev_port, code, NULL, NULL); 691 } 692 } 693 } 694 rw_exit(&uo_tbl->uobj_tbl_lock); 695 } 696 697 /* 698 * Function: 699 * uverbs_async_event_handler 700 * Input: 701 * clnt_private - The IBT attach client handle. 702 * hca_hdl - The IBT hca handle associated with the notification. 703 * code - The OFED event identifier. 704 * event - The IBT event. 705 * Output: 706 * None 707 * Returns: 708 * None 709 * Description: 710 * Main IBT asynchronous event handler registered at ibt_attach. 711 * Convert to OFA event type and forward to the appropriate 712 * asynchronous handler. 713 */ 714 void 715 uverbs_async_event_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl, 716 ibt_async_code_t code, ibt_async_event_t *event) 717 { 718 enum ib_event_type ofa_type; 719 sol_uverbs_ib_event_handler_t *handler; 720 llist_head_t *entry; 721 sol_uverbs_hca_t *hca; 722 723 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 724 "ASYNNC EVENT HANDLER: entry, code=%d", code); 725 726 ofa_type = uverbs_ibt_to_ofa_event_code(code); 727 728 if ((int)ofa_type < 0) { 729 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, 730 "ASYNC EVENT HANDLER:Event %d did not map to OFA " 731 "Event", code); 732 return; 733 } 734 735 switch (ofa_type) { 736 case IB_EVENT_QP_FATAL: 737 case IB_EVENT_QP_REQ_ERR: 738 case IB_EVENT_QP_ACCESS_ERR: 739 case IB_EVENT_QP_LAST_WQE_REACHED: 740 case IB_EVENT_SQ_DRAINED: 741 case IB_EVENT_PATH_MIG: 742 /* 743 * These events are related with a QP 744 */ 745 uverbs_async_qp_event_handler(clnt_private, hca_hdl, 746 ofa_type, event); 747 break; 748 749 case IB_EVENT_CQ_ERR: 750 /* 751 * These events are related with a CQ 752 */ 753 uverbs_async_cq_event_handler(clnt_private, hca_hdl, 754 ofa_type, event); 755 break; 756 757 case IB_EVENT_SRQ_ERR: 758 case IB_EVENT_SRQ_LIMIT_REACHED: 759 /* 760 * These events are related with a SRQ 761 */ 762 uverbs_async_srq_event_handler(clnt_private, hca_hdl, 763 ofa_type, event); 764 break; 765 766 767 case IB_EVENT_PORT_ERR: 768 case IB_EVENT_PORT_ACTIVE: 769 case IB_EVENT_LID_CHANGE: 770 case IB_EVENT_PKEY_CHANGE: 771 case IB_EVENT_SM_CHANGE: 772 case IB_EVENT_CLIENT_REREGISTER: 773 case IB_EVENT_DEVICE_FATAL: 774 case IB_EVENT_PATH_MIG_ERR: 775 /* 776 * Unaffiliated asynchronous notifications. 777 */ 778 uverbs_async_unaff_event_handler(clnt_private, hca_hdl, 779 ofa_type, event); 780 break; 781 782 default: 783 break; 784 } 785 786 /* 787 * Give other kernel agents a notification. 788 */ 789 hca = sol_uverbs_ibt_hdl_to_hca(hca_hdl); 790 if (hca) { 791 mutex_enter(&hca->event_handler_lock); 792 list_for_each(entry, &hca->event_handler_list) { 793 handler = (sol_uverbs_ib_event_handler_t *)entry->ptr; 794 795 ASSERT(handler != NULL); 796 handler->handler(handler, hca_hdl, code, event); 797 } 798 mutex_exit(&hca->event_handler_lock); 799 } 800 } 801 802 /* 803 * Function: 804 * uverbs_async_event_common 805 * Input: 806 * uctxt - Pointer to the user context associated with the 807 * affiliated event. 808 * element - The users handle to the associated object. 809 * event - The event type. 810 * uobj_list - The list to enqueue the asynchronous event. 811 * counter - The counter to track the event delivery. 812 * Output: 813 * None 814 * Returns: 815 * None 816 * Description: 817 * Create an asyncronous event and enqueue it on the specified list. 818 * Then wake callers that may be blocked on the list. 819 */ 820 static void 821 uverbs_async_event_common(uverbs_uctxt_uobj_t *uctxt, uint64_t element, 822 uint32_t event, llist_head_t *obj_list, uint32_t *counter) 823 { 824 uverbs_ufile_uobj_t *ufile = uctxt->async_evfile; 825 uverbs_event_t *entry; 826 827 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common(%p, " 828 "%llx, %llx, %p, %p)", uctxt, element, event, obj_list, counter); 829 830 if (!ufile) { 831 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common " 832 "ufile %p", ufile); 833 return; 834 } 835 836 mutex_enter(&ufile->lock); 837 entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP); 838 if (!entry) { 839 mutex_exit(&ufile->lock); 840 SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "async_event_common " 841 "kmem_zalloc failed"); 842 return; 843 } 844 845 entry->ev_desc.async.element = element; 846 entry->ev_desc.async.event_type = event; 847 entry->ev_counter = counter; 848 849 llist_head_init(&entry->ev_list, entry); 850 llist_head_init(&entry->ev_obj_list, entry); 851 852 llist_add_tail(&entry->ev_list, &ufile->event_list); 853 854 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common " 855 "adding ASYNC entry-ev_list=%p, entry %p", 856 &entry->ev_list, entry); 857 858 if (obj_list) { 859 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "async_event_common " 860 "adding ASYNC entry-ev_obj_list=%p, entry=%p", 861 &entry->ev_obj_list, entry); 862 llist_add_tail(&entry->ev_obj_list, obj_list); 863 } 864 865 mutex_exit(&ufile->lock); 866 cv_signal(&ufile->poll_wait); 867 pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM); 868 } 869 870 /* 871 * Function: 872 * uverbs_release_ucq_channel 873 * Input: 874 * uctxt - A pointer to the callers user context. 875 * ufile - A pointer to the event file associated with a CQ. 876 * ucq - A pointer to the user CQ object. 877 * Output: 878 * None 879 * Returns: 880 * None 881 * Description: 882 * Release any completion and asynchronous events that may 883 * be queued to the specified completion channel/UCQ but not 884 * yet reaped. 885 */ 886 void 887 uverbs_release_ucq_channel(uverbs_uctxt_uobj_t *uctxt, 888 uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq) 889 { 890 uverbs_event_t *evt; 891 llist_head_t *entry; 892 llist_head_t *list_tmp; 893 894 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 895 "RELEASE UCQ CHANNEL: uctxt=%p, ufile=%p, ucq=%p", 896 uctxt, ufile, ucq); 897 898 /* 899 * Release completion events that have been queued on the CQ completion 900 * eventlist. 901 */ 902 if (ufile) { 903 rw_enter(&ufile->uobj.uo_lock, RW_WRITER); 904 ufile->ufile_cq_cnt--; 905 if (ufile->ufile_cq_cnt) { 906 rw_exit(&ufile->uobj.uo_lock); 907 uverbs_release_ucq_uevents(ufile, ucq); 908 return; 909 } 910 rw_exit(&ufile->uobj.uo_lock); 911 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 912 "release_ucq_chan : comp_list %p, prv %p, nxt %p", 913 &ucq->comp_list, ucq->comp_list.prv, 914 ucq->comp_list.nxt); 915 mutex_enter(&ufile->lock); 916 917 entry = ucq->comp_list.nxt; 918 list_tmp = entry->nxt; 919 while (entry != &ucq->comp_list) { 920 ASSERT(entry); 921 evt = (uverbs_event_t *)entry->ptr; 922 923 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 924 "RELEASE UCQ CHANNEL:Deleting event " 925 "on CQ comp list: %p", evt); 926 llist_del(&evt->ev_list); 927 llist_del(&evt->ev_obj_list); 928 kmem_free(evt, sizeof (*evt)); 929 entry = list_tmp; 930 list_tmp = entry->nxt; 931 } 932 mutex_exit(&ufile->lock); 933 934 uverbs_release_ucq_uevents(ufile, ucq); 935 } 936 937 } 938 939 /* 940 * Function: 941 * uverbs_release_ucq_uevents 942 * Input: 943 * ufile - A pointer to the asynchronous event file associated with a QP. 944 * ucq - A pointer to the user CQ object. 945 * Output: 946 * None 947 * Returns: 948 * None 949 * Description: 950 * Free any user asynchronous events that have been queued for the 951 * user CQ object specified. 952 */ 953 void 954 uverbs_release_ucq_uevents(uverbs_ufile_uobj_t *ufile, uverbs_ucq_uobj_t *ucq) 955 { 956 uverbs_event_t *evt; 957 llist_head_t *entry; 958 llist_head_t *list_tmp; 959 960 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 961 "RELEASE UCQ ASYNC EVENTS: ufile=%p, ucq=%p", ufile, ucq); 962 963 if (ufile) { 964 mutex_enter(&ufile->lock); 965 966 entry = ucq->async_list.nxt; 967 list_tmp = entry->nxt; 968 while (entry != &ucq->async_list) { 969 ASSERT(entry); 970 evt = (uverbs_event_t *)entry->ptr; 971 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 972 "RELEASE UCQ EVENTS: Deleting event " 973 "on CQ async list: %p", evt); 974 llist_del(&evt->ev_list); 975 llist_del(&evt->ev_obj_list); 976 kmem_free(evt, sizeof (*evt)); 977 entry = list_tmp; 978 list_tmp = entry->nxt; 979 } 980 mutex_exit(&ufile->lock); 981 } 982 } 983 984 /* 985 * Function: 986 * uverbs_release_uqp_uevents 987 * Input: 988 * ufile - A pointer to the asynchronous event file associated with a QP. 989 * uqp - A pointer to the user QP object. 990 * Output: 991 * None 992 * Returns: 993 * None 994 * Description: 995 * Free any user asynchronous events that have been queued for the 996 * user QP object specified. 997 */ 998 void 999 uverbs_release_uqp_uevents(uverbs_ufile_uobj_t *ufile, uverbs_uqp_uobj_t *uqp) 1000 { 1001 uverbs_event_t *evt; 1002 llist_head_t *entry; 1003 llist_head_t *list_tmp; 1004 1005 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1006 "RELEASE UQP EVENTS: ufile=%p, uqp=%p", ufile, uqp); 1007 1008 if (ufile) { 1009 mutex_enter(&ufile->lock); 1010 entry = uqp->async_list.nxt; 1011 list_tmp = entry->nxt; 1012 while (entry != &uqp->async_list) { 1013 ASSERT(entry); 1014 evt = (uverbs_event_t *)entry->ptr; 1015 ASSERT(evt); 1016 1017 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1018 "RELEASE UQP EVENTS: Deleting event " 1019 "ON qp async list: %p", evt); 1020 llist_del(&evt->ev_list); 1021 llist_del(&evt->ev_obj_list); 1022 kmem_free(evt, sizeof (*evt)); 1023 entry = list_tmp; 1024 list_tmp = entry->nxt; 1025 } 1026 mutex_exit(&ufile->lock); 1027 } 1028 } 1029 1030 /* 1031 * Function: 1032 * uverbs_release_usrq_uevents 1033 * Input: 1034 * ufile - A pointer to the asynchronous event file associated with a 1035 * SRQ. 1036 * uqp - A pointer to the user SRQ object. 1037 * Output: 1038 * None 1039 * Returns: 1040 * None 1041 * Description: 1042 * Free any user asynchronous events that have been queued for the 1043 * user SRQ object specified. 1044 */ 1045 void 1046 uverbs_release_usrq_uevents(uverbs_ufile_uobj_t *ufile, 1047 uverbs_usrq_uobj_t *usrq) 1048 { 1049 uverbs_event_t *evt; 1050 llist_head_t *entry; 1051 llist_head_t *list_tmp; 1052 1053 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1054 "RELEASE USRQ EVENTS: ufile=%p, usrq=%p", ufile, usrq); 1055 1056 if (ufile) { 1057 mutex_enter(&ufile->lock); 1058 1059 entry = usrq->async_list.nxt; 1060 list_tmp = entry->nxt; 1061 while (entry != &usrq->async_list) { 1062 ASSERT(entry); 1063 evt = (uverbs_event_t *)entry->ptr; 1064 SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, 1065 "RELEASE SRQ EVENTS: Deleting event " 1066 "on SRQ async list: %p", evt); 1067 llist_del(&evt->ev_list); 1068 llist_del(&evt->ev_obj_list); 1069 kmem_free(evt, sizeof (*evt)); 1070 entry = list_tmp; 1071 list_tmp = entry->nxt; 1072 } 1073 mutex_exit(&ufile->lock); 1074 } 1075 } 1076