1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Copyright (c) 2007 Oracle. All rights reserved. 27 * 28 * This software is available to you under a choice of one of two 29 * licenses. You may choose to be licensed under the terms of the GNU 30 * General Public License (GPL) Version 2, available from the file 31 * COPYING in the main directory of this source tree, or the 32 * OpenIB.org BSD license below: 33 * 34 * Redistribution and use in source and binary forms, with or 35 * without modification, are permitted provided that the following 36 * conditions are met: 37 * 38 * - Redistributions of source code must retain the above 39 * copyright notice, this list of conditions and the following 40 * disclaimer. 41 * 42 * - Redistributions in binary form must reproduce the above 43 * copyright notice, this list of conditions and the following 44 * disclaimer in the documentation and/or other materials 45 * provided with the distribution. 46 * 47 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 48 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 49 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 50 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 51 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 52 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 53 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 54 * SOFTWARE. 55 * 56 */ 57 #include <sys/ib/clients/of/rdma/ib_verbs.h> 58 #include <sys/ib/clients/of/rdma/ib_addr.h> 59 #include <sys/ib/clients/of/rdma/rdma_cm.h> 60 61 #include <sys/ib/clients/rdsv3/ib.h> 62 #include <sys/ib/clients/rdsv3/rdma.h> 63 #include <sys/ib/clients/rdsv3/rdsv3_debug.h> 64 65 #define DMA_TO_DEVICE 0 66 #define DMA_FROM_DEVICE 1 67 #define RB_CLEAR_NODE(nodep) AVL_SETPARENT(nodep, nodep); 68 69 /* 70 * XXX 71 * - build with sparse 72 * - should we limit the size of a mr region? let transport return failure? 73 * - should we detect duplicate keys on a socket? hmm. 74 * - an rdma is an mlock, apply rlimit? 75 */ 76 77 /* 78 * get the number of pages by looking at the page indices that the start and 79 * end addresses fall in. 80 * 81 * Returns 0 if the vec is invalid. It is invalid if the number of bytes 82 * causes the address to wrap or overflows an unsigned int. This comes 83 * from being stored in the 'length' member of 'struct rdsv3_scatterlist'. 84 */ 85 static unsigned int 86 rdsv3_pages_in_vec(struct rdsv3_iovec *vec) 87 { 88 if ((vec->addr + vec->bytes <= vec->addr) || 89 (vec->bytes > (uint64_t)UINT_MAX)) { 90 return (0); 91 } 92 93 return (((vec->addr + vec->bytes + PAGESIZE - 1) >> 94 PAGESHIFT) - (vec->addr >> PAGESHIFT)); 95 } 96 97 static struct rdsv3_mr * 98 rdsv3_mr_tree_walk(struct avl_tree *root, uint32_t key, 99 struct rdsv3_mr *insert) 100 { 101 struct rdsv3_mr *mr; 102 avl_index_t where; 103 104 mr = avl_find(root, &key, &where); 105 if ((mr == NULL) && (insert != NULL)) { 106 avl_insert(root, (void *)insert, where); 107 atomic_add_32(&insert->r_refcount, 1); 108 return (NULL); 109 } 110 111 return (mr); 112 } 113 114 /* 115 * Destroy the transport-specific part of a MR. 116 */ 117 static void 118 rdsv3_destroy_mr(struct rdsv3_mr *mr) 119 { 120 struct rdsv3_sock *rs = mr->r_sock; 121 void *trans_private = NULL; 122 avl_node_t *np; 123 124 RDSV3_DPRINTF5("rdsv3_destroy_mr", 125 "RDS: destroy mr key is %x refcnt %u", 126 mr->r_key, atomic_get(&mr->r_refcount)); 127 128 if (test_and_set_bit(RDSV3_MR_DEAD, &mr->r_state)) 129 return; 130 131 mutex_enter(&rs->rs_rdma_lock); 132 np = &mr->r_rb_node; 133 if (AVL_XPARENT(np) != np) 134 avl_remove(&rs->rs_rdma_keys, mr); 135 trans_private = mr->r_trans_private; 136 mr->r_trans_private = NULL; 137 mutex_exit(&rs->rs_rdma_lock); 138 139 if (trans_private) 140 mr->r_trans->free_mr(trans_private, mr->r_invalidate); 141 } 142 143 void 144 __rdsv3_put_mr_final(struct rdsv3_mr *mr) 145 { 146 rdsv3_destroy_mr(mr); 147 kmem_free(mr, sizeof (*mr)); 148 } 149 150 /* 151 * By the time this is called we can't have any more ioctls called on 152 * the socket so we don't need to worry about racing with others. 153 */ 154 void 155 rdsv3_rdma_drop_keys(struct rdsv3_sock *rs) 156 { 157 struct rdsv3_mr *mr; 158 struct avl_node *node; 159 160 /* Release any MRs associated with this socket */ 161 mutex_enter(&rs->rs_rdma_lock); 162 while ((node = avl_first(&rs->rs_rdma_keys))) { 163 mr = container_of(node, struct rdsv3_mr, r_rb_node); 164 if (mr->r_trans == rs->rs_transport) 165 mr->r_invalidate = 0; 166 avl_remove(&rs->rs_rdma_keys, &mr->r_rb_node); 167 RB_CLEAR_NODE(&mr->r_rb_node) 168 mutex_exit(&rs->rs_rdma_lock); 169 rdsv3_destroy_mr(mr); 170 rdsv3_mr_put(mr); 171 mutex_enter(&rs->rs_rdma_lock); 172 } 173 mutex_exit(&rs->rs_rdma_lock); 174 175 if (rs->rs_transport && rs->rs_transport->flush_mrs) 176 rs->rs_transport->flush_mrs(); 177 } 178 179 static int 180 __rdsv3_rdma_map(struct rdsv3_sock *rs, struct rdsv3_get_mr_args *args, 181 uint64_t *cookie_ret, struct rdsv3_mr **mr_ret) 182 { 183 struct rdsv3_mr *mr = NULL, *found; 184 void *trans_private; 185 rdsv3_rdma_cookie_t cookie; 186 unsigned int nents = 0; 187 int ret; 188 189 if (rs->rs_bound_addr == 0) { 190 ret = -ENOTCONN; /* XXX not a great errno */ 191 goto out; 192 } 193 194 if (!rs->rs_transport->get_mr) { 195 ret = -EOPNOTSUPP; 196 goto out; 197 } 198 199 mr = kmem_zalloc(sizeof (struct rdsv3_mr), KM_NOSLEEP); 200 if (!mr) { 201 ret = -ENOMEM; 202 goto out; 203 } 204 205 mr->r_refcount = 1; 206 RB_CLEAR_NODE(&mr->r_rb_node); 207 mr->r_trans = rs->rs_transport; 208 mr->r_sock = rs; 209 210 if (args->flags & RDSV3_RDMA_USE_ONCE) 211 mr->r_use_once = 1; 212 if (args->flags & RDSV3_RDMA_INVALIDATE) 213 mr->r_invalidate = 1; 214 if (args->flags & RDSV3_RDMA_READWRITE) 215 mr->r_write = 1; 216 217 /* 218 * Obtain a transport specific MR. If this succeeds, the 219 * s/g list is now owned by the MR. 220 * Note that dma_map() implies that pending writes are 221 * flushed to RAM, so no dma_sync is needed here. 222 */ 223 trans_private = rs->rs_transport->get_mr(&args->vec, nents, rs, 224 &mr->r_key); 225 226 if (IS_ERR(trans_private)) { 227 ret = PTR_ERR(trans_private); 228 goto out; 229 } 230 231 mr->r_trans_private = trans_private; 232 233 /* 234 * The user may pass us an unaligned address, but we can only 235 * map page aligned regions. So we keep the offset, and build 236 * a 64bit cookie containing <R_Key, offset> and pass that 237 * around. 238 */ 239 cookie = rdsv3_rdma_make_cookie(mr->r_key, args->vec.addr & ~PAGEMASK); 240 if (cookie_ret) 241 *cookie_ret = cookie; 242 243 /* 244 * copy value of cookie to user address at args->cookie_addr 245 */ 246 if (args->cookie_addr) { 247 ret = ddi_copyout((void *)&cookie, 248 (void *)((intptr_t)args->cookie_addr), 249 sizeof (rdsv3_rdma_cookie_t), 0); 250 if (ret != 0) { 251 ret = -EFAULT; 252 goto out; 253 } 254 } 255 256 RDSV3_DPRINTF5("__rdsv3_rdma_map", 257 "RDS: get_mr mr 0x%p addr 0x%llx key 0x%x", 258 mr, args->vec.addr, mr->r_key); 259 /* 260 * Inserting the new MR into the rbtree bumps its 261 * reference count. 262 */ 263 mutex_enter(&rs->rs_rdma_lock); 264 found = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr); 265 mutex_exit(&rs->rs_rdma_lock); 266 267 ASSERT(!(found && found != mr)); 268 269 if (mr_ret) { 270 atomic_add_32(&mr->r_refcount, 1); 271 *mr_ret = mr; 272 } 273 274 ret = 0; 275 out: 276 if (mr) 277 rdsv3_mr_put(mr); 278 return (ret); 279 } 280 281 int 282 rdsv3_get_mr(struct rdsv3_sock *rs, const void *optval, int optlen) 283 { 284 struct rdsv3_get_mr_args args; 285 286 if (optlen != sizeof (struct rdsv3_get_mr_args)) 287 return (-EINVAL); 288 289 #if 1 290 bcopy((struct rdsv3_get_mr_args *)optval, &args, 291 sizeof (struct rdsv3_get_mr_args)); 292 #else 293 if (ddi_copyin(optval, &args, optlen, 0)) 294 return (-EFAULT); 295 #endif 296 297 return (__rdsv3_rdma_map(rs, &args, NULL, NULL)); 298 } 299 300 int 301 rdsv3_get_mr_for_dest(struct rdsv3_sock *rs, const void *optval, 302 int optlen) 303 { 304 struct rdsv3_get_mr_for_dest_args args; 305 struct rdsv3_get_mr_args new_args; 306 307 if (optlen != sizeof (struct rdsv3_get_mr_for_dest_args)) 308 return (-EINVAL); 309 310 #if 1 311 bcopy((struct rdsv3_get_mr_for_dest_args *)optval, &args, 312 sizeof (struct rdsv3_get_mr_for_dest_args)); 313 #else 314 if (ddi_copyin(optval, &args, optlen, 0)) 315 return (-EFAULT); 316 #endif 317 318 /* 319 * Initially, just behave like get_mr(). 320 * TODO: Implement get_mr as wrapper around this 321 * and deprecate it. 322 */ 323 new_args.vec = args.vec; 324 new_args.cookie_addr = args.cookie_addr; 325 new_args.flags = args.flags; 326 327 return (__rdsv3_rdma_map(rs, &new_args, NULL, NULL)); 328 } 329 330 /* 331 * Free the MR indicated by the given R_Key 332 */ 333 int 334 rdsv3_free_mr(struct rdsv3_sock *rs, const void *optval, int optlen) 335 { 336 struct rdsv3_free_mr_args args; 337 struct rdsv3_mr *mr; 338 339 if (optlen != sizeof (struct rdsv3_free_mr_args)) 340 return (-EINVAL); 341 342 #if 1 343 bcopy((struct rdsv3_free_mr_args *)optval, &args, 344 sizeof (struct rdsv3_free_mr_args)); 345 #else 346 if (ddi_copyin((struct rdsv3_free_mr_args *)optval, &args, 347 sizeof (struct rdsv3_free_mr_args), 0)) 348 return (-EFAULT); 349 #endif 350 351 /* Special case - a null cookie means flush all unused MRs */ 352 if (args.cookie == 0) { 353 if (!rs->rs_transport || !rs->rs_transport->flush_mrs) 354 return (-EINVAL); 355 rs->rs_transport->flush_mrs(); 356 return (0); 357 } 358 359 /* 360 * Look up the MR given its R_key and remove it from the rbtree 361 * so nobody else finds it. 362 * This should also prevent races with rdsv3_rdma_unuse. 363 */ 364 mutex_enter(&rs->rs_rdma_lock); 365 mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, 366 rdsv3_rdma_cookie_key(args.cookie), NULL); 367 if (mr) { 368 avl_remove(&rs->rs_rdma_keys, &mr->r_rb_node); 369 RB_CLEAR_NODE(&mr->r_rb_node); 370 if (args.flags & RDSV3_RDMA_INVALIDATE) 371 mr->r_invalidate = 1; 372 } 373 mutex_exit(&rs->rs_rdma_lock); 374 375 if (!mr) 376 return (-EINVAL); 377 378 /* 379 * call rdsv3_destroy_mr() ourselves so that we're sure it's done 380 * by time we return. If we let rdsv3_mr_put() do it it might not 381 * happen until someone else drops their ref. 382 */ 383 rdsv3_destroy_mr(mr); 384 rdsv3_mr_put(mr); 385 return (0); 386 } 387 388 /* 389 * This is called when we receive an extension header that 390 * tells us this MR was used. It allows us to implement 391 * use_once semantics 392 */ 393 void 394 rdsv3_rdma_unuse(struct rdsv3_sock *rs, uint32_t r_key, int force) 395 { 396 struct rdsv3_mr *mr; 397 int zot_me = 0; 398 399 RDSV3_DPRINTF4("rdsv3_rdma_unuse", "Enter rkey: 0x%x", r_key); 400 401 mutex_enter(&rs->rs_rdma_lock); 402 mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL); 403 if (!mr) { 404 RDSV3_DPRINTF4("rdsv3_rdma_unuse", 405 "rdsv3: trying to unuse MR with unknown r_key %u!", r_key); 406 mutex_exit(&rs->rs_rdma_lock); 407 return; 408 } 409 410 if (mr->r_use_once || force) { 411 avl_remove(&rs->rs_rdma_keys, &mr->r_rb_node); 412 RB_CLEAR_NODE(&mr->r_rb_node); 413 zot_me = 1; 414 } 415 mutex_exit(&rs->rs_rdma_lock); 416 417 /* 418 * May have to issue a dma_sync on this memory region. 419 * Note we could avoid this if the operation was a RDMA READ, 420 * but at this point we can't tell. 421 */ 422 if (mr->r_trans->sync_mr) 423 mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE); 424 425 /* 426 * If the MR was marked as invalidate, this will 427 * trigger an async flush. 428 */ 429 if (zot_me) 430 rdsv3_destroy_mr(mr); 431 rdsv3_mr_put(mr); 432 RDSV3_DPRINTF4("rdsv3_rdma_unuse", "Return"); 433 } 434 435 void 436 rdsv3_rdma_free_op(struct rdsv3_rdma_op *ro) 437 { 438 unsigned int i; 439 440 /* deallocate RDMA resources on rdsv3_message */ 441 for (i = 0; i < ro->r_nents; i++) { 442 ddi_umem_unlock(ro->r_rdma_sg[i].umem_cookie); 443 } 444 445 if (ro->r_notifier) 446 kmem_free(ro->r_notifier, sizeof (*ro->r_notifier)); 447 kmem_free(ro, sizeof (*ro)); 448 } 449 450 /* 451 * args is a pointer to an in-kernel copy in the sendmsg cmsg. 452 */ 453 static struct rdsv3_rdma_op * 454 rdsv3_rdma_prepare(struct rdsv3_sock *rs, struct rdsv3_rdma_args *args) 455 { 456 struct rdsv3_iovec vec; 457 struct rdsv3_rdma_op *op = NULL; 458 unsigned int nr_bytes; 459 struct rdsv3_iovec *local_vec; 460 unsigned int nr; 461 unsigned int i; 462 ddi_umem_cookie_t umem_cookie; 463 size_t umem_len; 464 caddr_t umem_addr; 465 int ret; 466 467 if (rs->rs_bound_addr == 0) { 468 ret = -ENOTCONN; /* XXX not a great errno */ 469 goto out; 470 } 471 472 if (args->nr_local > (uint64_t)UINT_MAX) { 473 ret = -EMSGSIZE; 474 goto out; 475 } 476 477 op = kmem_zalloc(offsetof(struct rdsv3_rdma_op, 478 r_rdma_sg[args->nr_local]), KM_NOSLEEP); 479 if (op == NULL) { 480 ret = -ENOMEM; 481 goto out; 482 } 483 484 op->r_write = !!(args->flags & RDSV3_RDMA_READWRITE); 485 op->r_fence = !!(args->flags & RDSV3_RDMA_FENCE); 486 op->r_notify = !!(args->flags & RDSV3_RDMA_NOTIFY_ME); 487 op->r_recverr = rs->rs_recverr; 488 489 if (op->r_notify || op->r_recverr) { 490 /* 491 * We allocate an uninitialized notifier here, because 492 * we don't want to do that in the completion handler. We 493 * would have to use GFP_ATOMIC there, and don't want to deal 494 * with failed allocations. 495 */ 496 op->r_notifier = kmem_alloc(sizeof (struct rdsv3_notifier), 497 KM_NOSLEEP); 498 if (!op->r_notifier) { 499 ret = -ENOMEM; 500 goto out; 501 } 502 op->r_notifier->n_user_token = args->user_token; 503 op->r_notifier->n_status = RDSV3_RDMA_SUCCESS; 504 } 505 506 /* 507 * The cookie contains the R_Key of the remote memory region, and 508 * optionally an offset into it. This is how we implement RDMA into 509 * unaligned memory. 510 * When setting up the RDMA, we need to add that offset to the 511 * destination address (which is really an offset into the MR) 512 * FIXME: We may want to move this into ib_rdma.c 513 */ 514 op->r_key = rdsv3_rdma_cookie_key(args->cookie); 515 op->r_remote_addr = args->remote_vec.addr + 516 rdsv3_rdma_cookie_offset(args->cookie); 517 518 nr_bytes = 0; 519 520 RDSV3_DPRINTF5("rdsv3_rdma_prepare", 521 "RDS: rdma prepare nr_local %llu rva %llx rkey %x", 522 (unsigned long long)args->nr_local, 523 (unsigned long long)args->remote_vec.addr, 524 op->r_key); 525 526 local_vec = (struct rdsv3_iovec *)(unsigned long) args->local_vec_addr; 527 528 /* pin the scatter list of user buffers */ 529 for (i = 0; i < args->nr_local; i++) { 530 if (ddi_copyin(&local_vec[i], &vec, 531 sizeof (struct rdsv3_iovec), 0)) { 532 ret = -EFAULT; 533 goto out; 534 } 535 536 nr = rdsv3_pages_in_vec(&vec); 537 if (nr == 0) { 538 RDSV3_DPRINTF2("rdsv3_rdma_prepare", 539 "rdsv3_pages_in_vec returned 0"); 540 ret = -EINVAL; 541 goto out; 542 } 543 544 rs->rs_user_addr = vec.addr; 545 rs->rs_user_bytes = vec.bytes; 546 547 /* pin user memory pages */ 548 umem_len = ptob(btopr(vec.bytes + 549 ((uintptr_t)vec.addr & PAGEOFFSET))); 550 umem_addr = (caddr_t)((uintptr_t)vec.addr & ~PAGEOFFSET); 551 ret = umem_lockmemory(umem_addr, umem_len, 552 DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ, 553 &umem_cookie, NULL, NULL); 554 if (ret != 0) { 555 RDSV3_DPRINTF2("rdsv3_rdma_prepare", 556 "umem_lockmemory() returned %d", ret); 557 ret = -EFAULT; 558 goto out; 559 } 560 op->r_rdma_sg[i].umem_cookie = umem_cookie; 561 op->r_rdma_sg[i].iovec = vec; 562 nr_bytes += vec.bytes; 563 564 RDSV3_DPRINTF5("rdsv3_rdma_prepare", 565 "RDS: nr_bytes %u nr %u vec.bytes %llu vec.addr %llx", 566 nr_bytes, nr, vec.bytes, vec.addr); 567 } 568 op->r_nents = i; 569 570 if (nr_bytes > args->remote_vec.bytes) { 571 RDSV3_DPRINTF2("rdsv3_rdma_prepare", 572 "RDS nr_bytes %u remote_bytes %u do not match", 573 nr_bytes, (unsigned int) args->remote_vec.bytes); 574 ret = -EINVAL; 575 goto out; 576 } 577 op->r_bytes = nr_bytes; 578 579 ret = 0; 580 out: 581 if (ret) { 582 if (op) 583 rdsv3_rdma_free_op(op); 584 op = ERR_PTR(ret); 585 } 586 return (op); 587 } 588 589 /* 590 * The application asks for a RDMA transfer. 591 * Extract all arguments and set up the rdma_op 592 */ 593 int 594 rdsv3_cmsg_rdma_args(struct rdsv3_sock *rs, struct rdsv3_message *rm, 595 struct cmsghdr *cmsg) 596 { 597 struct rdsv3_rdma_op *op; 598 struct rdsv3_rdma_args *ap; 599 600 if (cmsg->cmsg_len < CMSG_LEN(sizeof (struct rdsv3_rdma_args)) || 601 rm->m_rdma_op != NULL) 602 return (-EINVAL); 603 604 /* uint64_t alignment on struct rdsv3_get_mr_args */ 605 ap = (struct rdsv3_rdma_args *)kmem_alloc(cmsg->cmsg_len, KM_SLEEP); 606 bcopy(CMSG_DATA(cmsg), ap, cmsg->cmsg_len); 607 op = rdsv3_rdma_prepare(rs, ap); 608 kmem_free(ap, cmsg->cmsg_len); 609 if (IS_ERR(op)) 610 return (PTR_ERR(op)); 611 rdsv3_stats_inc(s_send_rdma); 612 rm->m_rdma_op = op; 613 return (0); 614 } 615 616 /* 617 * The application wants us to pass an RDMA destination (aka MR) 618 * to the remote 619 */ 620 int 621 rdsv3_cmsg_rdma_dest(struct rdsv3_sock *rs, struct rdsv3_message *rm, 622 struct cmsghdr *cmsg) 623 { 624 struct rdsv3_mr *mr; 625 uint32_t r_key; 626 int err = 0; 627 628 if (cmsg->cmsg_len < CMSG_LEN(sizeof (rdsv3_rdma_cookie_t)) || 629 rm->m_rdma_cookie != 0) 630 return (-EINVAL); 631 632 (void) memcpy(&rm->m_rdma_cookie, CMSG_DATA(cmsg), 633 sizeof (rm->m_rdma_cookie)); 634 635 /* 636 * We are reusing a previously mapped MR here. Most likely, the 637 * application has written to the buffer, so we need to explicitly 638 * flush those writes to RAM. Otherwise the HCA may not see them 639 * when doing a DMA from that buffer. 640 */ 641 r_key = rdsv3_rdma_cookie_key(rm->m_rdma_cookie); 642 643 mutex_enter(&rs->rs_rdma_lock); 644 mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL); 645 if (!mr) 646 err = -EINVAL; /* invalid r_key */ 647 else 648 atomic_add_32(&mr->r_refcount, 1); 649 mutex_exit(&rs->rs_rdma_lock); 650 651 if (mr) { 652 mr->r_trans->sync_mr(mr->r_trans_private, DMA_TO_DEVICE); 653 rm->m_rdma_mr = mr; 654 } 655 return (err); 656 } 657 658 /* 659 * The application passes us an address range it wants to enable RDMA 660 * to/from. We map the area, and save the <R_Key,offset> pair 661 * in rm->m_rdma_cookie. This causes it to be sent along to the peer 662 * in an extension header. 663 */ 664 int 665 rdsv3_cmsg_rdma_map(struct rdsv3_sock *rs, struct rdsv3_message *rm, 666 struct cmsghdr *cmsg) 667 { 668 struct rdsv3_get_mr_args *mrp; 669 int status; 670 671 if (cmsg->cmsg_len < CMSG_LEN(sizeof (struct rdsv3_get_mr_args)) || 672 rm->m_rdma_cookie != 0) 673 return (-EINVAL); 674 675 /* uint64_t alignment on struct rdsv3_get_mr_args */ 676 mrp = (struct rdsv3_get_mr_args *)kmem_alloc(cmsg->cmsg_len, KM_SLEEP); 677 bcopy(CMSG_DATA(cmsg), mrp, cmsg->cmsg_len); 678 status = __rdsv3_rdma_map(rs, mrp, &rm->m_rdma_cookie, &rm->m_rdma_mr); 679 kmem_free(mrp, cmsg->cmsg_len); 680 return (status); 681 } 682