1 /* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36 #include <linux/file.h> 37 #include <linux/fs.h> 38 #include <linux/slab.h> 39 #include <linux/sched.h> 40 41 #include <linux/uaccess.h> 42 43 #include <rdma/uverbs_types.h> 44 #include <rdma/uverbs_std_types.h> 45 #include "rdma_core.h" 46 47 #include "uverbs.h" 48 #include "core_priv.h" 49 50 /* 51 * Copy a response to userspace. If the provided 'resp' is larger than the 52 * user buffer it is silently truncated. If the user provided a larger buffer 53 * then the trailing portion is zero filled. 54 * 55 * These semantics are intended to support future extension of the output 56 * structures. 57 */ 58 static int uverbs_response(struct uverbs_attr_bundle *attrs, const void *resp, 59 size_t resp_len) 60 { 61 int ret; 62 63 if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT)) 64 return uverbs_copy_to_struct_or_zero( 65 attrs, UVERBS_ATTR_CORE_OUT, resp, resp_len); 66 67 if (copy_to_user(attrs->ucore.outbuf, resp, 68 min(attrs->ucore.outlen, resp_len))) 69 return -EFAULT; 70 71 if (resp_len < attrs->ucore.outlen) { 72 /* 73 * Zero fill any extra memory that user 74 * space might have provided. 75 */ 76 ret = clear_user(attrs->ucore.outbuf + resp_len, 77 attrs->ucore.outlen - resp_len); 78 if (ret) 79 return -EFAULT; 80 } 81 82 return 0; 83 } 84 85 /* 86 * Copy a request from userspace. If the provided 'req' is larger than the 87 * user buffer then the user buffer is zero extended into the 'req'. If 'req' 88 * is smaller than the user buffer then the uncopied bytes in the user buffer 89 * must be zero. 90 */ 91 static int uverbs_request(struct uverbs_attr_bundle *attrs, void *req, 92 size_t req_len) 93 { 94 if (copy_from_user(req, attrs->ucore.inbuf, 95 min(attrs->ucore.inlen, req_len))) 96 return -EFAULT; 97 98 if (attrs->ucore.inlen < req_len) { 99 memset(req + attrs->ucore.inlen, 0, 100 req_len - attrs->ucore.inlen); 101 } else if (attrs->ucore.inlen > req_len) { 102 if (!ib_is_buffer_cleared(attrs->ucore.inbuf + req_len, 103 attrs->ucore.inlen - req_len)) 104 return -EOPNOTSUPP; 105 } 106 return 0; 107 } 108 109 /* 110 * Generate the value for the 'response_length' protocol used by write_ex. 111 * This is the number of bytes the kernel actually wrote. Userspace can use 112 * this to detect what structure members in the response the kernel 113 * understood. 114 */ 115 static u32 uverbs_response_length(struct uverbs_attr_bundle *attrs, 116 size_t resp_len) 117 { 118 return min_t(size_t, attrs->ucore.outlen, resp_len); 119 } 120 121 /* 122 * The iterator version of the request interface is for handlers that need to 123 * step over a flex array at the end of a command header. 124 */ 125 struct uverbs_req_iter { 126 const void __user *cur; 127 const void __user *end; 128 }; 129 130 static int uverbs_request_start(struct uverbs_attr_bundle *attrs, 131 struct uverbs_req_iter *iter, 132 void *req, 133 size_t req_len) 134 { 135 if (attrs->ucore.inlen < req_len) 136 return -ENOSPC; 137 138 if (copy_from_user(req, attrs->ucore.inbuf, req_len)) 139 return -EFAULT; 140 141 iter->cur = attrs->ucore.inbuf + req_len; 142 iter->end = attrs->ucore.inbuf + attrs->ucore.inlen; 143 return 0; 144 } 145 146 static int uverbs_request_next(struct uverbs_req_iter *iter, void *val, 147 size_t len) 148 { 149 if (iter->cur + len > iter->end) 150 return -ENOSPC; 151 152 if (copy_from_user(val, iter->cur, len)) 153 return -EFAULT; 154 155 iter->cur += len; 156 return 0; 157 } 158 159 static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter, 160 size_t len) 161 { 162 const void __user *res = iter->cur; 163 164 if (iter->cur + len > iter->end) 165 return (void __force __user *)ERR_PTR(-ENOSPC); 166 iter->cur += len; 167 return res; 168 } 169 170 static int uverbs_request_finish(struct uverbs_req_iter *iter) 171 { 172 if (!ib_is_buffer_cleared(iter->cur, iter->end - iter->cur)) 173 return -EOPNOTSUPP; 174 return 0; 175 } 176 177 /* 178 * When calling a destroy function during an error unwind we need to pass in 179 * the udata that is sanitized of all user arguments. Ie from the driver 180 * perspective it looks like no udata was passed. 181 */ 182 struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) 183 { 184 attrs->driver_udata = (struct ib_udata){}; 185 return &attrs->driver_udata; 186 } 187 188 static struct ib_uverbs_completion_event_file * 189 _ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs) 190 { 191 struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL, 192 fd, attrs); 193 194 if (IS_ERR(uobj)) 195 return (void *)uobj; 196 197 uverbs_uobject_get(uobj); 198 uobj_put_read(uobj); 199 200 return container_of(uobj, struct ib_uverbs_completion_event_file, 201 uobj); 202 } 203 #define ib_uverbs_lookup_comp_file(_fd, _ufile) \ 204 _ib_uverbs_lookup_comp_file((_fd)*typecheck(s32, _fd), _ufile) 205 206 static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs) 207 { 208 struct ib_uverbs_file *file = attrs->ufile; 209 struct ib_uverbs_get_context cmd; 210 struct ib_uverbs_get_context_resp resp; 211 struct ib_ucontext *ucontext; 212 struct file *filp; 213 struct ib_rdmacg_object cg_obj; 214 struct ib_device *ib_dev; 215 int ret; 216 217 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 218 if (ret) 219 return ret; 220 221 mutex_lock(&file->ucontext_lock); 222 ib_dev = srcu_dereference(file->device->ib_dev, 223 &file->device->disassociate_srcu); 224 if (!ib_dev) { 225 ret = -EIO; 226 goto err; 227 } 228 229 if (file->ucontext) { 230 ret = -EINVAL; 231 goto err; 232 } 233 234 ret = ib_rdmacg_try_charge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 235 if (ret) 236 goto err; 237 238 ucontext = rdma_zalloc_drv_obj(ib_dev, ib_ucontext); 239 if (!ucontext) { 240 ret = -ENOMEM; 241 goto err_alloc; 242 } 243 244 attrs->context = ucontext; 245 246 ucontext->res.type = RDMA_RESTRACK_CTX; 247 ucontext->device = ib_dev; 248 ucontext->cg_obj = cg_obj; 249 /* ufile is required when some objects are released */ 250 ucontext->ufile = file; 251 252 ucontext->closing = false; 253 ucontext->cleanup_retryable = false; 254 255 xa_init_flags(&ucontext->mmap_xa, XA_FLAGS_ALLOC); 256 257 ret = get_unused_fd_flags(O_CLOEXEC); 258 if (ret < 0) 259 goto err_free; 260 resp.async_fd = ret; 261 262 filp = ib_uverbs_alloc_async_event_file(file, ib_dev); 263 if (IS_ERR(filp)) { 264 ret = PTR_ERR(filp); 265 goto err_fd; 266 } 267 268 resp.num_comp_vectors = file->device->num_comp_vectors; 269 270 ret = uverbs_response(attrs, &resp, sizeof(resp)); 271 if (ret) 272 goto err_file; 273 274 ret = ib_dev->ops.alloc_ucontext(ucontext, &attrs->driver_udata); 275 if (ret) 276 goto err_file; 277 278 rdma_restrack_uadd(&ucontext->res); 279 280 fd_install(resp.async_fd, filp); 281 282 /* 283 * Make sure that ib_uverbs_get_ucontext() sees the pointer update 284 * only after all writes to setup the ucontext have completed 285 */ 286 smp_store_release(&file->ucontext, ucontext); 287 288 mutex_unlock(&file->ucontext_lock); 289 290 return 0; 291 292 err_file: 293 ib_uverbs_free_async_event_file(file); 294 fput(filp); 295 296 err_fd: 297 put_unused_fd(resp.async_fd); 298 299 err_free: 300 kfree(ucontext); 301 302 err_alloc: 303 ib_rdmacg_uncharge(&cg_obj, ib_dev, RDMACG_RESOURCE_HCA_HANDLE); 304 305 err: 306 mutex_unlock(&file->ucontext_lock); 307 return ret; 308 } 309 310 static void copy_query_dev_fields(struct ib_ucontext *ucontext, 311 struct ib_uverbs_query_device_resp *resp, 312 struct ib_device_attr *attr) 313 { 314 struct ib_device *ib_dev = ucontext->device; 315 316 resp->fw_ver = attr->fw_ver; 317 resp->node_guid = ib_dev->node_guid; 318 resp->sys_image_guid = attr->sys_image_guid; 319 resp->max_mr_size = attr->max_mr_size; 320 resp->page_size_cap = attr->page_size_cap; 321 resp->vendor_id = attr->vendor_id; 322 resp->vendor_part_id = attr->vendor_part_id; 323 resp->hw_ver = attr->hw_ver; 324 resp->max_qp = attr->max_qp; 325 resp->max_qp_wr = attr->max_qp_wr; 326 resp->device_cap_flags = lower_32_bits(attr->device_cap_flags); 327 resp->max_sge = min(attr->max_send_sge, attr->max_recv_sge); 328 resp->max_sge_rd = attr->max_sge_rd; 329 resp->max_cq = attr->max_cq; 330 resp->max_cqe = attr->max_cqe; 331 resp->max_mr = attr->max_mr; 332 resp->max_pd = attr->max_pd; 333 resp->max_qp_rd_atom = attr->max_qp_rd_atom; 334 resp->max_ee_rd_atom = attr->max_ee_rd_atom; 335 resp->max_res_rd_atom = attr->max_res_rd_atom; 336 resp->max_qp_init_rd_atom = attr->max_qp_init_rd_atom; 337 resp->max_ee_init_rd_atom = attr->max_ee_init_rd_atom; 338 resp->atomic_cap = attr->atomic_cap; 339 resp->max_ee = attr->max_ee; 340 resp->max_rdd = attr->max_rdd; 341 resp->max_mw = attr->max_mw; 342 resp->max_raw_ipv6_qp = attr->max_raw_ipv6_qp; 343 resp->max_raw_ethy_qp = attr->max_raw_ethy_qp; 344 resp->max_mcast_grp = attr->max_mcast_grp; 345 resp->max_mcast_qp_attach = attr->max_mcast_qp_attach; 346 resp->max_total_mcast_qp_attach = attr->max_total_mcast_qp_attach; 347 resp->max_ah = attr->max_ah; 348 resp->max_fmr = attr->max_fmr; 349 resp->max_map_per_fmr = attr->max_map_per_fmr; 350 resp->max_srq = attr->max_srq; 351 resp->max_srq_wr = attr->max_srq_wr; 352 resp->max_srq_sge = attr->max_srq_sge; 353 resp->max_pkeys = attr->max_pkeys; 354 resp->local_ca_ack_delay = attr->local_ca_ack_delay; 355 resp->phys_port_cnt = ib_dev->phys_port_cnt; 356 } 357 358 static int ib_uverbs_query_device(struct uverbs_attr_bundle *attrs) 359 { 360 struct ib_uverbs_query_device cmd; 361 struct ib_uverbs_query_device_resp resp; 362 struct ib_ucontext *ucontext; 363 int ret; 364 365 ucontext = ib_uverbs_get_ucontext(attrs); 366 if (IS_ERR(ucontext)) 367 return PTR_ERR(ucontext); 368 369 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 370 if (ret) 371 return ret; 372 373 memset(&resp, 0, sizeof resp); 374 copy_query_dev_fields(ucontext, &resp, &ucontext->device->attrs); 375 376 return uverbs_response(attrs, &resp, sizeof(resp)); 377 } 378 379 static int ib_uverbs_query_port(struct uverbs_attr_bundle *attrs) 380 { 381 struct ib_uverbs_query_port cmd; 382 struct ib_uverbs_query_port_resp resp; 383 struct ib_port_attr attr; 384 int ret; 385 struct ib_ucontext *ucontext; 386 struct ib_device *ib_dev; 387 388 ucontext = ib_uverbs_get_ucontext(attrs); 389 if (IS_ERR(ucontext)) 390 return PTR_ERR(ucontext); 391 ib_dev = ucontext->device; 392 393 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 394 if (ret) 395 return ret; 396 397 ret = ib_query_port(ib_dev, cmd.port_num, &attr); 398 if (ret) 399 return ret; 400 401 memset(&resp, 0, sizeof resp); 402 copy_port_attr_to_resp(&attr, &resp, ib_dev, cmd.port_num); 403 404 return uverbs_response(attrs, &resp, sizeof(resp)); 405 } 406 407 static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs) 408 { 409 struct ib_uverbs_alloc_pd cmd; 410 struct ib_uverbs_alloc_pd_resp resp; 411 struct ib_uobject *uobj; 412 struct ib_pd *pd; 413 int ret; 414 struct ib_device *ib_dev; 415 416 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 417 if (ret) 418 return ret; 419 420 uobj = uobj_alloc(UVERBS_OBJECT_PD, attrs, &ib_dev); 421 if (IS_ERR(uobj)) 422 return PTR_ERR(uobj); 423 424 pd = rdma_zalloc_drv_obj(ib_dev, ib_pd); 425 if (!pd) { 426 ret = -ENOMEM; 427 goto err; 428 } 429 430 pd->device = ib_dev; 431 pd->uobject = uobj; 432 pd->__internal_mr = NULL; 433 atomic_set(&pd->usecnt, 0); 434 pd->res.type = RDMA_RESTRACK_PD; 435 436 ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata); 437 if (ret) 438 goto err_alloc; 439 440 uobj->object = pd; 441 memset(&resp, 0, sizeof resp); 442 resp.pd_handle = uobj->id; 443 rdma_restrack_uadd(&pd->res); 444 445 ret = uverbs_response(attrs, &resp, sizeof(resp)); 446 if (ret) 447 goto err_copy; 448 449 return uobj_alloc_commit(uobj, attrs); 450 451 err_copy: 452 ib_dealloc_pd_user(pd, uverbs_get_cleared_udata(attrs)); 453 pd = NULL; 454 err_alloc: 455 kfree(pd); 456 err: 457 uobj_alloc_abort(uobj, attrs); 458 return ret; 459 } 460 461 static int ib_uverbs_dealloc_pd(struct uverbs_attr_bundle *attrs) 462 { 463 struct ib_uverbs_dealloc_pd cmd; 464 int ret; 465 466 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 467 if (ret) 468 return ret; 469 470 return uobj_perform_destroy(UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 471 } 472 473 struct xrcd_table_entry { 474 struct rb_node node; 475 struct ib_xrcd *xrcd; 476 struct inode *inode; 477 }; 478 479 static int xrcd_table_insert(struct ib_uverbs_device *dev, 480 struct inode *inode, 481 struct ib_xrcd *xrcd) 482 { 483 struct xrcd_table_entry *entry, *scan; 484 struct rb_node **p = &dev->xrcd_tree.rb_node; 485 struct rb_node *parent = NULL; 486 487 entry = kmalloc(sizeof *entry, GFP_KERNEL); 488 if (!entry) 489 return -ENOMEM; 490 491 entry->xrcd = xrcd; 492 entry->inode = inode; 493 494 while (*p) { 495 parent = *p; 496 scan = rb_entry(parent, struct xrcd_table_entry, node); 497 498 if (inode < scan->inode) { 499 p = &(*p)->rb_left; 500 } else if (inode > scan->inode) { 501 p = &(*p)->rb_right; 502 } else { 503 kfree(entry); 504 return -EEXIST; 505 } 506 } 507 508 rb_link_node(&entry->node, parent, p); 509 rb_insert_color(&entry->node, &dev->xrcd_tree); 510 igrab(inode); 511 return 0; 512 } 513 514 static struct xrcd_table_entry *xrcd_table_search(struct ib_uverbs_device *dev, 515 struct inode *inode) 516 { 517 struct xrcd_table_entry *entry; 518 struct rb_node *p = dev->xrcd_tree.rb_node; 519 520 while (p) { 521 entry = rb_entry(p, struct xrcd_table_entry, node); 522 523 if (inode < entry->inode) 524 p = p->rb_left; 525 else if (inode > entry->inode) 526 p = p->rb_right; 527 else 528 return entry; 529 } 530 531 return NULL; 532 } 533 534 static struct ib_xrcd *find_xrcd(struct ib_uverbs_device *dev, struct inode *inode) 535 { 536 struct xrcd_table_entry *entry; 537 538 entry = xrcd_table_search(dev, inode); 539 if (!entry) 540 return NULL; 541 542 return entry->xrcd; 543 } 544 545 static void xrcd_table_delete(struct ib_uverbs_device *dev, 546 struct inode *inode) 547 { 548 struct xrcd_table_entry *entry; 549 550 entry = xrcd_table_search(dev, inode); 551 if (entry) { 552 iput(inode); 553 rb_erase(&entry->node, &dev->xrcd_tree); 554 kfree(entry); 555 } 556 } 557 558 static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs) 559 { 560 struct ib_uverbs_device *ibudev = attrs->ufile->device; 561 struct ib_uverbs_open_xrcd cmd; 562 struct ib_uverbs_open_xrcd_resp resp; 563 struct ib_uxrcd_object *obj; 564 struct ib_xrcd *xrcd = NULL; 565 struct fd f = {NULL, 0}; 566 struct inode *inode = NULL; 567 int ret = 0; 568 int new_xrcd = 0; 569 struct ib_device *ib_dev; 570 571 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 572 if (ret) 573 return ret; 574 575 mutex_lock(&ibudev->xrcd_tree_mutex); 576 577 if (cmd.fd != -1) { 578 /* search for file descriptor */ 579 f = fdget(cmd.fd); 580 if (!f.file) { 581 ret = -EBADF; 582 goto err_tree_mutex_unlock; 583 } 584 585 inode = file_inode(f.file); 586 xrcd = find_xrcd(ibudev, inode); 587 if (!xrcd && !(cmd.oflags & O_CREAT)) { 588 /* no file descriptor. Need CREATE flag */ 589 ret = -EAGAIN; 590 goto err_tree_mutex_unlock; 591 } 592 593 if (xrcd && cmd.oflags & O_EXCL) { 594 ret = -EINVAL; 595 goto err_tree_mutex_unlock; 596 } 597 } 598 599 obj = (struct ib_uxrcd_object *)uobj_alloc(UVERBS_OBJECT_XRCD, attrs, 600 &ib_dev); 601 if (IS_ERR(obj)) { 602 ret = PTR_ERR(obj); 603 goto err_tree_mutex_unlock; 604 } 605 606 if (!xrcd) { 607 xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata); 608 if (IS_ERR(xrcd)) { 609 ret = PTR_ERR(xrcd); 610 goto err; 611 } 612 613 xrcd->inode = inode; 614 xrcd->device = ib_dev; 615 atomic_set(&xrcd->usecnt, 0); 616 mutex_init(&xrcd->tgt_qp_mutex); 617 INIT_LIST_HEAD(&xrcd->tgt_qp_list); 618 new_xrcd = 1; 619 } 620 621 atomic_set(&obj->refcnt, 0); 622 obj->uobject.object = xrcd; 623 memset(&resp, 0, sizeof resp); 624 resp.xrcd_handle = obj->uobject.id; 625 626 if (inode) { 627 if (new_xrcd) { 628 /* create new inode/xrcd table entry */ 629 ret = xrcd_table_insert(ibudev, inode, xrcd); 630 if (ret) 631 goto err_dealloc_xrcd; 632 } 633 atomic_inc(&xrcd->usecnt); 634 } 635 636 ret = uverbs_response(attrs, &resp, sizeof(resp)); 637 if (ret) 638 goto err_copy; 639 640 if (f.file) 641 fdput(f); 642 643 mutex_unlock(&ibudev->xrcd_tree_mutex); 644 645 return uobj_alloc_commit(&obj->uobject, attrs); 646 647 err_copy: 648 if (inode) { 649 if (new_xrcd) 650 xrcd_table_delete(ibudev, inode); 651 atomic_dec(&xrcd->usecnt); 652 } 653 654 err_dealloc_xrcd: 655 ib_dealloc_xrcd(xrcd, uverbs_get_cleared_udata(attrs)); 656 657 err: 658 uobj_alloc_abort(&obj->uobject, attrs); 659 660 err_tree_mutex_unlock: 661 if (f.file) 662 fdput(f); 663 664 mutex_unlock(&ibudev->xrcd_tree_mutex); 665 666 return ret; 667 } 668 669 static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs) 670 { 671 struct ib_uverbs_close_xrcd cmd; 672 int ret; 673 674 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 675 if (ret) 676 return ret; 677 678 return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs); 679 } 680 681 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd, 682 enum rdma_remove_reason why, 683 struct uverbs_attr_bundle *attrs) 684 { 685 struct inode *inode; 686 int ret; 687 struct ib_uverbs_device *dev = attrs->ufile->device; 688 689 inode = xrcd->inode; 690 if (inode && !atomic_dec_and_test(&xrcd->usecnt)) 691 return 0; 692 693 ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata); 694 695 if (ib_is_destroy_retryable(ret, why, uobject)) { 696 atomic_inc(&xrcd->usecnt); 697 return ret; 698 } 699 700 if (inode) 701 xrcd_table_delete(dev, inode); 702 703 return ret; 704 } 705 706 static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) 707 { 708 struct ib_uverbs_reg_mr cmd; 709 struct ib_uverbs_reg_mr_resp resp; 710 struct ib_uobject *uobj; 711 struct ib_pd *pd; 712 struct ib_mr *mr; 713 int ret; 714 struct ib_device *ib_dev; 715 716 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 717 if (ret) 718 return ret; 719 720 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 721 return -EINVAL; 722 723 ret = ib_check_mr_access(cmd.access_flags); 724 if (ret) 725 return ret; 726 727 uobj = uobj_alloc(UVERBS_OBJECT_MR, attrs, &ib_dev); 728 if (IS_ERR(uobj)) 729 return PTR_ERR(uobj); 730 731 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 732 if (!pd) { 733 ret = -EINVAL; 734 goto err_free; 735 } 736 737 if (cmd.access_flags & IB_ACCESS_ON_DEMAND) { 738 if (!(pd->device->attrs.device_cap_flags & 739 IB_DEVICE_ON_DEMAND_PAGING)) { 740 pr_debug("ODP support not available\n"); 741 ret = -EINVAL; 742 goto err_put; 743 } 744 } 745 746 mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 747 cmd.access_flags, 748 &attrs->driver_udata); 749 if (IS_ERR(mr)) { 750 ret = PTR_ERR(mr); 751 goto err_put; 752 } 753 754 mr->device = pd->device; 755 mr->pd = pd; 756 mr->type = IB_MR_TYPE_USER; 757 mr->dm = NULL; 758 mr->sig_attrs = NULL; 759 mr->uobject = uobj; 760 atomic_inc(&pd->usecnt); 761 mr->res.type = RDMA_RESTRACK_MR; 762 rdma_restrack_uadd(&mr->res); 763 764 uobj->object = mr; 765 766 memset(&resp, 0, sizeof resp); 767 resp.lkey = mr->lkey; 768 resp.rkey = mr->rkey; 769 resp.mr_handle = uobj->id; 770 771 ret = uverbs_response(attrs, &resp, sizeof(resp)); 772 if (ret) 773 goto err_copy; 774 775 uobj_put_obj_read(pd); 776 777 return uobj_alloc_commit(uobj, attrs); 778 779 err_copy: 780 ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs)); 781 782 err_put: 783 uobj_put_obj_read(pd); 784 785 err_free: 786 uobj_alloc_abort(uobj, attrs); 787 return ret; 788 } 789 790 static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) 791 { 792 struct ib_uverbs_rereg_mr cmd; 793 struct ib_uverbs_rereg_mr_resp resp; 794 struct ib_pd *pd = NULL; 795 struct ib_mr *mr; 796 struct ib_pd *old_pd; 797 int ret; 798 struct ib_uobject *uobj; 799 800 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 801 if (ret) 802 return ret; 803 804 if (cmd.flags & ~IB_MR_REREG_SUPPORTED || !cmd.flags) 805 return -EINVAL; 806 807 if ((cmd.flags & IB_MR_REREG_TRANS) && 808 (!cmd.start || !cmd.hca_va || 0 >= cmd.length || 809 (cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK))) 810 return -EINVAL; 811 812 uobj = uobj_get_write(UVERBS_OBJECT_MR, cmd.mr_handle, attrs); 813 if (IS_ERR(uobj)) 814 return PTR_ERR(uobj); 815 816 mr = uobj->object; 817 818 if (mr->dm) { 819 ret = -EINVAL; 820 goto put_uobjs; 821 } 822 823 if (cmd.flags & IB_MR_REREG_ACCESS) { 824 ret = ib_check_mr_access(cmd.access_flags); 825 if (ret) 826 goto put_uobjs; 827 } 828 829 if (cmd.flags & IB_MR_REREG_PD) { 830 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, 831 attrs); 832 if (!pd) { 833 ret = -EINVAL; 834 goto put_uobjs; 835 } 836 } 837 838 old_pd = mr->pd; 839 ret = mr->device->ops.rereg_user_mr(mr, cmd.flags, cmd.start, 840 cmd.length, cmd.hca_va, 841 cmd.access_flags, pd, 842 &attrs->driver_udata); 843 if (ret) 844 goto put_uobj_pd; 845 846 if (cmd.flags & IB_MR_REREG_PD) { 847 atomic_inc(&pd->usecnt); 848 mr->pd = pd; 849 atomic_dec(&old_pd->usecnt); 850 } 851 852 memset(&resp, 0, sizeof(resp)); 853 resp.lkey = mr->lkey; 854 resp.rkey = mr->rkey; 855 856 ret = uverbs_response(attrs, &resp, sizeof(resp)); 857 858 put_uobj_pd: 859 if (cmd.flags & IB_MR_REREG_PD) 860 uobj_put_obj_read(pd); 861 862 put_uobjs: 863 uobj_put_write(uobj); 864 865 return ret; 866 } 867 868 static int ib_uverbs_dereg_mr(struct uverbs_attr_bundle *attrs) 869 { 870 struct ib_uverbs_dereg_mr cmd; 871 int ret; 872 873 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 874 if (ret) 875 return ret; 876 877 return uobj_perform_destroy(UVERBS_OBJECT_MR, cmd.mr_handle, attrs); 878 } 879 880 static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs) 881 { 882 struct ib_uverbs_alloc_mw cmd; 883 struct ib_uverbs_alloc_mw_resp resp; 884 struct ib_uobject *uobj; 885 struct ib_pd *pd; 886 struct ib_mw *mw; 887 int ret; 888 struct ib_device *ib_dev; 889 890 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 891 if (ret) 892 return ret; 893 894 uobj = uobj_alloc(UVERBS_OBJECT_MW, attrs, &ib_dev); 895 if (IS_ERR(uobj)) 896 return PTR_ERR(uobj); 897 898 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 899 if (!pd) { 900 ret = -EINVAL; 901 goto err_free; 902 } 903 904 if (cmd.mw_type != IB_MW_TYPE_1 && cmd.mw_type != IB_MW_TYPE_2) { 905 ret = -EINVAL; 906 goto err_put; 907 } 908 909 mw = pd->device->ops.alloc_mw(pd, cmd.mw_type, &attrs->driver_udata); 910 if (IS_ERR(mw)) { 911 ret = PTR_ERR(mw); 912 goto err_put; 913 } 914 915 mw->device = pd->device; 916 mw->pd = pd; 917 mw->uobject = uobj; 918 atomic_inc(&pd->usecnt); 919 920 uobj->object = mw; 921 922 memset(&resp, 0, sizeof(resp)); 923 resp.rkey = mw->rkey; 924 resp.mw_handle = uobj->id; 925 926 ret = uverbs_response(attrs, &resp, sizeof(resp)); 927 if (ret) 928 goto err_copy; 929 930 uobj_put_obj_read(pd); 931 return uobj_alloc_commit(uobj, attrs); 932 933 err_copy: 934 uverbs_dealloc_mw(mw); 935 err_put: 936 uobj_put_obj_read(pd); 937 err_free: 938 uobj_alloc_abort(uobj, attrs); 939 return ret; 940 } 941 942 static int ib_uverbs_dealloc_mw(struct uverbs_attr_bundle *attrs) 943 { 944 struct ib_uverbs_dealloc_mw cmd; 945 int ret; 946 947 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 948 if (ret) 949 return ret; 950 951 return uobj_perform_destroy(UVERBS_OBJECT_MW, cmd.mw_handle, attrs); 952 } 953 954 static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs) 955 { 956 struct ib_uverbs_create_comp_channel cmd; 957 struct ib_uverbs_create_comp_channel_resp resp; 958 struct ib_uobject *uobj; 959 struct ib_uverbs_completion_event_file *ev_file; 960 struct ib_device *ib_dev; 961 int ret; 962 963 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 964 if (ret) 965 return ret; 966 967 uobj = uobj_alloc(UVERBS_OBJECT_COMP_CHANNEL, attrs, &ib_dev); 968 if (IS_ERR(uobj)) 969 return PTR_ERR(uobj); 970 971 resp.fd = uobj->id; 972 973 ev_file = container_of(uobj, struct ib_uverbs_completion_event_file, 974 uobj); 975 ib_uverbs_init_event_queue(&ev_file->ev_queue); 976 977 ret = uverbs_response(attrs, &resp, sizeof(resp)); 978 if (ret) { 979 uobj_alloc_abort(uobj, attrs); 980 return ret; 981 } 982 983 return uobj_alloc_commit(uobj, attrs); 984 } 985 986 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs, 987 struct ib_uverbs_ex_create_cq *cmd) 988 { 989 struct ib_ucq_object *obj; 990 struct ib_uverbs_completion_event_file *ev_file = NULL; 991 struct ib_cq *cq; 992 int ret; 993 struct ib_uverbs_ex_create_cq_resp resp; 994 struct ib_cq_init_attr attr = {}; 995 struct ib_device *ib_dev; 996 997 if (cmd->comp_vector >= attrs->ufile->device->num_comp_vectors) 998 return ERR_PTR(-EINVAL); 999 1000 obj = (struct ib_ucq_object *)uobj_alloc(UVERBS_OBJECT_CQ, attrs, 1001 &ib_dev); 1002 if (IS_ERR(obj)) 1003 return obj; 1004 1005 if (cmd->comp_channel >= 0) { 1006 ev_file = ib_uverbs_lookup_comp_file(cmd->comp_channel, attrs); 1007 if (IS_ERR(ev_file)) { 1008 ret = PTR_ERR(ev_file); 1009 goto err; 1010 } 1011 } 1012 1013 obj->uobject.user_handle = cmd->user_handle; 1014 obj->comp_events_reported = 0; 1015 obj->async_events_reported = 0; 1016 INIT_LIST_HEAD(&obj->comp_list); 1017 INIT_LIST_HEAD(&obj->async_list); 1018 1019 attr.cqe = cmd->cqe; 1020 attr.comp_vector = cmd->comp_vector; 1021 attr.flags = cmd->flags; 1022 1023 cq = rdma_zalloc_drv_obj(ib_dev, ib_cq); 1024 if (!cq) { 1025 ret = -ENOMEM; 1026 goto err_file; 1027 } 1028 cq->device = ib_dev; 1029 cq->uobject = &obj->uobject; 1030 cq->comp_handler = ib_uverbs_comp_handler; 1031 cq->event_handler = ib_uverbs_cq_event_handler; 1032 cq->cq_context = ev_file ? &ev_file->ev_queue : NULL; 1033 atomic_set(&cq->usecnt, 0); 1034 1035 ret = ib_dev->ops.create_cq(cq, &attr, &attrs->driver_udata); 1036 if (ret) 1037 goto err_free; 1038 1039 obj->uobject.object = cq; 1040 memset(&resp, 0, sizeof resp); 1041 resp.base.cq_handle = obj->uobject.id; 1042 resp.base.cqe = cq->cqe; 1043 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 1044 1045 cq->res.type = RDMA_RESTRACK_CQ; 1046 rdma_restrack_uadd(&cq->res); 1047 1048 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1049 if (ret) 1050 goto err_cb; 1051 1052 ret = uobj_alloc_commit(&obj->uobject, attrs); 1053 if (ret) 1054 return ERR_PTR(ret); 1055 return obj; 1056 1057 err_cb: 1058 ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs)); 1059 cq = NULL; 1060 err_free: 1061 kfree(cq); 1062 err_file: 1063 if (ev_file) 1064 ib_uverbs_release_ucq(attrs->ufile, ev_file, obj); 1065 1066 err: 1067 uobj_alloc_abort(&obj->uobject, attrs); 1068 1069 return ERR_PTR(ret); 1070 } 1071 1072 static int ib_uverbs_create_cq(struct uverbs_attr_bundle *attrs) 1073 { 1074 struct ib_uverbs_create_cq cmd; 1075 struct ib_uverbs_ex_create_cq cmd_ex; 1076 struct ib_ucq_object *obj; 1077 int ret; 1078 1079 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1080 if (ret) 1081 return ret; 1082 1083 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1084 cmd_ex.user_handle = cmd.user_handle; 1085 cmd_ex.cqe = cmd.cqe; 1086 cmd_ex.comp_vector = cmd.comp_vector; 1087 cmd_ex.comp_channel = cmd.comp_channel; 1088 1089 obj = create_cq(attrs, &cmd_ex); 1090 return PTR_ERR_OR_ZERO(obj); 1091 } 1092 1093 static int ib_uverbs_ex_create_cq(struct uverbs_attr_bundle *attrs) 1094 { 1095 struct ib_uverbs_ex_create_cq cmd; 1096 struct ib_ucq_object *obj; 1097 int ret; 1098 1099 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1100 if (ret) 1101 return ret; 1102 1103 if (cmd.comp_mask) 1104 return -EINVAL; 1105 1106 if (cmd.reserved) 1107 return -EINVAL; 1108 1109 obj = create_cq(attrs, &cmd); 1110 return PTR_ERR_OR_ZERO(obj); 1111 } 1112 1113 static int ib_uverbs_resize_cq(struct uverbs_attr_bundle *attrs) 1114 { 1115 struct ib_uverbs_resize_cq cmd; 1116 struct ib_uverbs_resize_cq_resp resp = {}; 1117 struct ib_cq *cq; 1118 int ret = -EINVAL; 1119 1120 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1121 if (ret) 1122 return ret; 1123 1124 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1125 if (!cq) 1126 return -EINVAL; 1127 1128 ret = cq->device->ops.resize_cq(cq, cmd.cqe, &attrs->driver_udata); 1129 if (ret) 1130 goto out; 1131 1132 resp.cqe = cq->cqe; 1133 1134 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1135 out: 1136 uobj_put_obj_read(cq); 1137 1138 return ret; 1139 } 1140 1141 static int copy_wc_to_user(struct ib_device *ib_dev, void __user *dest, 1142 struct ib_wc *wc) 1143 { 1144 struct ib_uverbs_wc tmp; 1145 1146 tmp.wr_id = wc->wr_id; 1147 tmp.status = wc->status; 1148 tmp.opcode = wc->opcode; 1149 tmp.vendor_err = wc->vendor_err; 1150 tmp.byte_len = wc->byte_len; 1151 tmp.ex.imm_data = wc->ex.imm_data; 1152 tmp.qp_num = wc->qp->qp_num; 1153 tmp.src_qp = wc->src_qp; 1154 tmp.wc_flags = wc->wc_flags; 1155 tmp.pkey_index = wc->pkey_index; 1156 if (rdma_cap_opa_ah(ib_dev, wc->port_num)) 1157 tmp.slid = OPA_TO_IB_UCAST_LID(wc->slid); 1158 else 1159 tmp.slid = ib_lid_cpu16(wc->slid); 1160 tmp.sl = wc->sl; 1161 tmp.dlid_path_bits = wc->dlid_path_bits; 1162 tmp.port_num = wc->port_num; 1163 tmp.reserved = 0; 1164 1165 if (copy_to_user(dest, &tmp, sizeof tmp)) 1166 return -EFAULT; 1167 1168 return 0; 1169 } 1170 1171 static int ib_uverbs_poll_cq(struct uverbs_attr_bundle *attrs) 1172 { 1173 struct ib_uverbs_poll_cq cmd; 1174 struct ib_uverbs_poll_cq_resp resp; 1175 u8 __user *header_ptr; 1176 u8 __user *data_ptr; 1177 struct ib_cq *cq; 1178 struct ib_wc wc; 1179 int ret; 1180 1181 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1182 if (ret) 1183 return ret; 1184 1185 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1186 if (!cq) 1187 return -EINVAL; 1188 1189 /* we copy a struct ib_uverbs_poll_cq_resp to user space */ 1190 header_ptr = attrs->ucore.outbuf; 1191 data_ptr = header_ptr + sizeof resp; 1192 1193 memset(&resp, 0, sizeof resp); 1194 while (resp.count < cmd.ne) { 1195 ret = ib_poll_cq(cq, 1, &wc); 1196 if (ret < 0) 1197 goto out_put; 1198 if (!ret) 1199 break; 1200 1201 ret = copy_wc_to_user(cq->device, data_ptr, &wc); 1202 if (ret) 1203 goto out_put; 1204 1205 data_ptr += sizeof(struct ib_uverbs_wc); 1206 ++resp.count; 1207 } 1208 1209 if (copy_to_user(header_ptr, &resp, sizeof resp)) { 1210 ret = -EFAULT; 1211 goto out_put; 1212 } 1213 ret = 0; 1214 1215 if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CORE_OUT)) 1216 ret = uverbs_output_written(attrs, UVERBS_ATTR_CORE_OUT); 1217 1218 out_put: 1219 uobj_put_obj_read(cq); 1220 return ret; 1221 } 1222 1223 static int ib_uverbs_req_notify_cq(struct uverbs_attr_bundle *attrs) 1224 { 1225 struct ib_uverbs_req_notify_cq cmd; 1226 struct ib_cq *cq; 1227 int ret; 1228 1229 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1230 if (ret) 1231 return ret; 1232 1233 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1234 if (!cq) 1235 return -EINVAL; 1236 1237 ib_req_notify_cq(cq, cmd.solicited_only ? 1238 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 1239 1240 uobj_put_obj_read(cq); 1241 1242 return 0; 1243 } 1244 1245 static int ib_uverbs_destroy_cq(struct uverbs_attr_bundle *attrs) 1246 { 1247 struct ib_uverbs_destroy_cq cmd; 1248 struct ib_uverbs_destroy_cq_resp resp; 1249 struct ib_uobject *uobj; 1250 struct ib_ucq_object *obj; 1251 int ret; 1252 1253 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1254 if (ret) 1255 return ret; 1256 1257 uobj = uobj_get_destroy(UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 1258 if (IS_ERR(uobj)) 1259 return PTR_ERR(uobj); 1260 1261 obj = container_of(uobj, struct ib_ucq_object, uobject); 1262 memset(&resp, 0, sizeof(resp)); 1263 resp.comp_events_reported = obj->comp_events_reported; 1264 resp.async_events_reported = obj->async_events_reported; 1265 1266 uobj_put_destroy(uobj); 1267 1268 return uverbs_response(attrs, &resp, sizeof(resp)); 1269 } 1270 1271 static int create_qp(struct uverbs_attr_bundle *attrs, 1272 struct ib_uverbs_ex_create_qp *cmd) 1273 { 1274 struct ib_uqp_object *obj; 1275 struct ib_device *device; 1276 struct ib_pd *pd = NULL; 1277 struct ib_xrcd *xrcd = NULL; 1278 struct ib_uobject *xrcd_uobj = ERR_PTR(-ENOENT); 1279 struct ib_cq *scq = NULL, *rcq = NULL; 1280 struct ib_srq *srq = NULL; 1281 struct ib_qp *qp; 1282 struct ib_qp_init_attr attr = {}; 1283 struct ib_uverbs_ex_create_qp_resp resp; 1284 int ret; 1285 struct ib_rwq_ind_table *ind_tbl = NULL; 1286 bool has_sq = true; 1287 struct ib_device *ib_dev; 1288 1289 if (cmd->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 1290 return -EPERM; 1291 1292 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs, 1293 &ib_dev); 1294 if (IS_ERR(obj)) 1295 return PTR_ERR(obj); 1296 obj->uxrcd = NULL; 1297 obj->uevent.uobject.user_handle = cmd->user_handle; 1298 mutex_init(&obj->mcast_lock); 1299 1300 if (cmd->comp_mask & IB_UVERBS_CREATE_QP_MASK_IND_TABLE) { 1301 ind_tbl = uobj_get_obj_read(rwq_ind_table, 1302 UVERBS_OBJECT_RWQ_IND_TBL, 1303 cmd->rwq_ind_tbl_handle, attrs); 1304 if (!ind_tbl) { 1305 ret = -EINVAL; 1306 goto err_put; 1307 } 1308 1309 attr.rwq_ind_tbl = ind_tbl; 1310 } 1311 1312 if (ind_tbl && (cmd->max_recv_wr || cmd->max_recv_sge || cmd->is_srq)) { 1313 ret = -EINVAL; 1314 goto err_put; 1315 } 1316 1317 if (ind_tbl && !cmd->max_send_wr) 1318 has_sq = false; 1319 1320 if (cmd->qp_type == IB_QPT_XRC_TGT) { 1321 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->pd_handle, 1322 attrs); 1323 1324 if (IS_ERR(xrcd_uobj)) { 1325 ret = -EINVAL; 1326 goto err_put; 1327 } 1328 1329 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1330 if (!xrcd) { 1331 ret = -EINVAL; 1332 goto err_put; 1333 } 1334 device = xrcd->device; 1335 } else { 1336 if (cmd->qp_type == IB_QPT_XRC_INI) { 1337 cmd->max_recv_wr = 0; 1338 cmd->max_recv_sge = 0; 1339 } else { 1340 if (cmd->is_srq) { 1341 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, 1342 cmd->srq_handle, attrs); 1343 if (!srq || srq->srq_type == IB_SRQT_XRC) { 1344 ret = -EINVAL; 1345 goto err_put; 1346 } 1347 } 1348 1349 if (!ind_tbl) { 1350 if (cmd->recv_cq_handle != cmd->send_cq_handle) { 1351 rcq = uobj_get_obj_read( 1352 cq, UVERBS_OBJECT_CQ, 1353 cmd->recv_cq_handle, attrs); 1354 if (!rcq) { 1355 ret = -EINVAL; 1356 goto err_put; 1357 } 1358 } 1359 } 1360 } 1361 1362 if (has_sq) 1363 scq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 1364 cmd->send_cq_handle, attrs); 1365 if (!ind_tbl) 1366 rcq = rcq ?: scq; 1367 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, 1368 attrs); 1369 if (!pd || (!scq && has_sq)) { 1370 ret = -EINVAL; 1371 goto err_put; 1372 } 1373 1374 device = pd->device; 1375 } 1376 1377 attr.event_handler = ib_uverbs_qp_event_handler; 1378 attr.qp_context = attrs->ufile; 1379 attr.send_cq = scq; 1380 attr.recv_cq = rcq; 1381 attr.srq = srq; 1382 attr.xrcd = xrcd; 1383 attr.sq_sig_type = cmd->sq_sig_all ? IB_SIGNAL_ALL_WR : 1384 IB_SIGNAL_REQ_WR; 1385 attr.qp_type = cmd->qp_type; 1386 attr.create_flags = 0; 1387 1388 attr.cap.max_send_wr = cmd->max_send_wr; 1389 attr.cap.max_recv_wr = cmd->max_recv_wr; 1390 attr.cap.max_send_sge = cmd->max_send_sge; 1391 attr.cap.max_recv_sge = cmd->max_recv_sge; 1392 attr.cap.max_inline_data = cmd->max_inline_data; 1393 1394 obj->uevent.events_reported = 0; 1395 INIT_LIST_HEAD(&obj->uevent.event_list); 1396 INIT_LIST_HEAD(&obj->mcast_list); 1397 1398 attr.create_flags = cmd->create_flags; 1399 if (attr.create_flags & ~(IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | 1400 IB_QP_CREATE_CROSS_CHANNEL | 1401 IB_QP_CREATE_MANAGED_SEND | 1402 IB_QP_CREATE_MANAGED_RECV | 1403 IB_QP_CREATE_SCATTER_FCS | 1404 IB_QP_CREATE_CVLAN_STRIPPING | 1405 IB_QP_CREATE_SOURCE_QPN | 1406 IB_QP_CREATE_PCI_WRITE_END_PADDING)) { 1407 ret = -EINVAL; 1408 goto err_put; 1409 } 1410 1411 if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) { 1412 if (!capable(CAP_NET_RAW)) { 1413 ret = -EPERM; 1414 goto err_put; 1415 } 1416 1417 attr.source_qpn = cmd->source_qpn; 1418 } 1419 1420 if (cmd->qp_type == IB_QPT_XRC_TGT) 1421 qp = ib_create_qp(pd, &attr); 1422 else 1423 qp = _ib_create_qp(device, pd, &attr, &attrs->driver_udata, 1424 &obj->uevent.uobject); 1425 1426 if (IS_ERR(qp)) { 1427 ret = PTR_ERR(qp); 1428 goto err_put; 1429 } 1430 1431 if (cmd->qp_type != IB_QPT_XRC_TGT) { 1432 ret = ib_create_qp_security(qp, device); 1433 if (ret) 1434 goto err_cb; 1435 1436 qp->pd = pd; 1437 qp->send_cq = attr.send_cq; 1438 qp->recv_cq = attr.recv_cq; 1439 qp->srq = attr.srq; 1440 qp->rwq_ind_tbl = ind_tbl; 1441 qp->event_handler = attr.event_handler; 1442 qp->qp_context = attr.qp_context; 1443 qp->qp_type = attr.qp_type; 1444 atomic_set(&qp->usecnt, 0); 1445 atomic_inc(&pd->usecnt); 1446 qp->port = 0; 1447 if (attr.send_cq) 1448 atomic_inc(&attr.send_cq->usecnt); 1449 if (attr.recv_cq) 1450 atomic_inc(&attr.recv_cq->usecnt); 1451 if (attr.srq) 1452 atomic_inc(&attr.srq->usecnt); 1453 if (ind_tbl) 1454 atomic_inc(&ind_tbl->usecnt); 1455 } else { 1456 /* It is done in _ib_create_qp for other QP types */ 1457 qp->uobject = &obj->uevent.uobject; 1458 } 1459 1460 obj->uevent.uobject.object = qp; 1461 1462 memset(&resp, 0, sizeof resp); 1463 resp.base.qpn = qp->qp_num; 1464 resp.base.qp_handle = obj->uevent.uobject.id; 1465 resp.base.max_recv_sge = attr.cap.max_recv_sge; 1466 resp.base.max_send_sge = attr.cap.max_send_sge; 1467 resp.base.max_recv_wr = attr.cap.max_recv_wr; 1468 resp.base.max_send_wr = attr.cap.max_send_wr; 1469 resp.base.max_inline_data = attr.cap.max_inline_data; 1470 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 1471 1472 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1473 if (ret) 1474 goto err_cb; 1475 1476 if (xrcd) { 1477 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, 1478 uobject); 1479 atomic_inc(&obj->uxrcd->refcnt); 1480 uobj_put_read(xrcd_uobj); 1481 } 1482 1483 if (pd) 1484 uobj_put_obj_read(pd); 1485 if (scq) 1486 uobj_put_obj_read(scq); 1487 if (rcq && rcq != scq) 1488 uobj_put_obj_read(rcq); 1489 if (srq) 1490 uobj_put_obj_read(srq); 1491 if (ind_tbl) 1492 uobj_put_obj_read(ind_tbl); 1493 1494 return uobj_alloc_commit(&obj->uevent.uobject, attrs); 1495 err_cb: 1496 ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); 1497 1498 err_put: 1499 if (!IS_ERR(xrcd_uobj)) 1500 uobj_put_read(xrcd_uobj); 1501 if (pd) 1502 uobj_put_obj_read(pd); 1503 if (scq) 1504 uobj_put_obj_read(scq); 1505 if (rcq && rcq != scq) 1506 uobj_put_obj_read(rcq); 1507 if (srq) 1508 uobj_put_obj_read(srq); 1509 if (ind_tbl) 1510 uobj_put_obj_read(ind_tbl); 1511 1512 uobj_alloc_abort(&obj->uevent.uobject, attrs); 1513 return ret; 1514 } 1515 1516 static int ib_uverbs_create_qp(struct uverbs_attr_bundle *attrs) 1517 { 1518 struct ib_uverbs_create_qp cmd; 1519 struct ib_uverbs_ex_create_qp cmd_ex; 1520 int ret; 1521 1522 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1523 if (ret) 1524 return ret; 1525 1526 memset(&cmd_ex, 0, sizeof(cmd_ex)); 1527 cmd_ex.user_handle = cmd.user_handle; 1528 cmd_ex.pd_handle = cmd.pd_handle; 1529 cmd_ex.send_cq_handle = cmd.send_cq_handle; 1530 cmd_ex.recv_cq_handle = cmd.recv_cq_handle; 1531 cmd_ex.srq_handle = cmd.srq_handle; 1532 cmd_ex.max_send_wr = cmd.max_send_wr; 1533 cmd_ex.max_recv_wr = cmd.max_recv_wr; 1534 cmd_ex.max_send_sge = cmd.max_send_sge; 1535 cmd_ex.max_recv_sge = cmd.max_recv_sge; 1536 cmd_ex.max_inline_data = cmd.max_inline_data; 1537 cmd_ex.sq_sig_all = cmd.sq_sig_all; 1538 cmd_ex.qp_type = cmd.qp_type; 1539 cmd_ex.is_srq = cmd.is_srq; 1540 1541 return create_qp(attrs, &cmd_ex); 1542 } 1543 1544 static int ib_uverbs_ex_create_qp(struct uverbs_attr_bundle *attrs) 1545 { 1546 struct ib_uverbs_ex_create_qp cmd; 1547 int ret; 1548 1549 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1550 if (ret) 1551 return ret; 1552 1553 if (cmd.comp_mask & ~IB_UVERBS_CREATE_QP_SUP_COMP_MASK) 1554 return -EINVAL; 1555 1556 if (cmd.reserved) 1557 return -EINVAL; 1558 1559 return create_qp(attrs, &cmd); 1560 } 1561 1562 static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs) 1563 { 1564 struct ib_uverbs_open_qp cmd; 1565 struct ib_uverbs_create_qp_resp resp; 1566 struct ib_uqp_object *obj; 1567 struct ib_xrcd *xrcd; 1568 struct ib_uobject *uninitialized_var(xrcd_uobj); 1569 struct ib_qp *qp; 1570 struct ib_qp_open_attr attr; 1571 int ret; 1572 struct ib_device *ib_dev; 1573 1574 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1575 if (ret) 1576 return ret; 1577 1578 obj = (struct ib_uqp_object *)uobj_alloc(UVERBS_OBJECT_QP, attrs, 1579 &ib_dev); 1580 if (IS_ERR(obj)) 1581 return PTR_ERR(obj); 1582 1583 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd.pd_handle, attrs); 1584 if (IS_ERR(xrcd_uobj)) { 1585 ret = -EINVAL; 1586 goto err_put; 1587 } 1588 1589 xrcd = (struct ib_xrcd *)xrcd_uobj->object; 1590 if (!xrcd) { 1591 ret = -EINVAL; 1592 goto err_xrcd; 1593 } 1594 1595 attr.event_handler = ib_uverbs_qp_event_handler; 1596 attr.qp_context = attrs->ufile; 1597 attr.qp_num = cmd.qpn; 1598 attr.qp_type = cmd.qp_type; 1599 1600 obj->uevent.events_reported = 0; 1601 INIT_LIST_HEAD(&obj->uevent.event_list); 1602 INIT_LIST_HEAD(&obj->mcast_list); 1603 1604 qp = ib_open_qp(xrcd, &attr); 1605 if (IS_ERR(qp)) { 1606 ret = PTR_ERR(qp); 1607 goto err_xrcd; 1608 } 1609 1610 obj->uevent.uobject.object = qp; 1611 obj->uevent.uobject.user_handle = cmd.user_handle; 1612 1613 memset(&resp, 0, sizeof resp); 1614 resp.qpn = qp->qp_num; 1615 resp.qp_handle = obj->uevent.uobject.id; 1616 1617 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1618 if (ret) 1619 goto err_destroy; 1620 1621 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 1622 atomic_inc(&obj->uxrcd->refcnt); 1623 qp->uobject = &obj->uevent.uobject; 1624 uobj_put_read(xrcd_uobj); 1625 1626 return uobj_alloc_commit(&obj->uevent.uobject, attrs); 1627 1628 err_destroy: 1629 ib_destroy_qp_user(qp, uverbs_get_cleared_udata(attrs)); 1630 err_xrcd: 1631 uobj_put_read(xrcd_uobj); 1632 err_put: 1633 uobj_alloc_abort(&obj->uevent.uobject, attrs); 1634 return ret; 1635 } 1636 1637 static void copy_ah_attr_to_uverbs(struct ib_uverbs_qp_dest *uverb_attr, 1638 struct rdma_ah_attr *rdma_attr) 1639 { 1640 const struct ib_global_route *grh; 1641 1642 uverb_attr->dlid = rdma_ah_get_dlid(rdma_attr); 1643 uverb_attr->sl = rdma_ah_get_sl(rdma_attr); 1644 uverb_attr->src_path_bits = rdma_ah_get_path_bits(rdma_attr); 1645 uverb_attr->static_rate = rdma_ah_get_static_rate(rdma_attr); 1646 uverb_attr->is_global = !!(rdma_ah_get_ah_flags(rdma_attr) & 1647 IB_AH_GRH); 1648 if (uverb_attr->is_global) { 1649 grh = rdma_ah_read_grh(rdma_attr); 1650 memcpy(uverb_attr->dgid, grh->dgid.raw, 16); 1651 uverb_attr->flow_label = grh->flow_label; 1652 uverb_attr->sgid_index = grh->sgid_index; 1653 uverb_attr->hop_limit = grh->hop_limit; 1654 uverb_attr->traffic_class = grh->traffic_class; 1655 } 1656 uverb_attr->port_num = rdma_ah_get_port_num(rdma_attr); 1657 } 1658 1659 static int ib_uverbs_query_qp(struct uverbs_attr_bundle *attrs) 1660 { 1661 struct ib_uverbs_query_qp cmd; 1662 struct ib_uverbs_query_qp_resp resp; 1663 struct ib_qp *qp; 1664 struct ib_qp_attr *attr; 1665 struct ib_qp_init_attr *init_attr; 1666 int ret; 1667 1668 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1669 if (ret) 1670 return ret; 1671 1672 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1673 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1674 if (!attr || !init_attr) { 1675 ret = -ENOMEM; 1676 goto out; 1677 } 1678 1679 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 1680 if (!qp) { 1681 ret = -EINVAL; 1682 goto out; 1683 } 1684 1685 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1686 1687 uobj_put_obj_read(qp); 1688 1689 if (ret) 1690 goto out; 1691 1692 memset(&resp, 0, sizeof resp); 1693 1694 resp.qp_state = attr->qp_state; 1695 resp.cur_qp_state = attr->cur_qp_state; 1696 resp.path_mtu = attr->path_mtu; 1697 resp.path_mig_state = attr->path_mig_state; 1698 resp.qkey = attr->qkey; 1699 resp.rq_psn = attr->rq_psn; 1700 resp.sq_psn = attr->sq_psn; 1701 resp.dest_qp_num = attr->dest_qp_num; 1702 resp.qp_access_flags = attr->qp_access_flags; 1703 resp.pkey_index = attr->pkey_index; 1704 resp.alt_pkey_index = attr->alt_pkey_index; 1705 resp.sq_draining = attr->sq_draining; 1706 resp.max_rd_atomic = attr->max_rd_atomic; 1707 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1708 resp.min_rnr_timer = attr->min_rnr_timer; 1709 resp.port_num = attr->port_num; 1710 resp.timeout = attr->timeout; 1711 resp.retry_cnt = attr->retry_cnt; 1712 resp.rnr_retry = attr->rnr_retry; 1713 resp.alt_port_num = attr->alt_port_num; 1714 resp.alt_timeout = attr->alt_timeout; 1715 1716 copy_ah_attr_to_uverbs(&resp.dest, &attr->ah_attr); 1717 copy_ah_attr_to_uverbs(&resp.alt_dest, &attr->alt_ah_attr); 1718 1719 resp.max_send_wr = init_attr->cap.max_send_wr; 1720 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1721 resp.max_send_sge = init_attr->cap.max_send_sge; 1722 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1723 resp.max_inline_data = init_attr->cap.max_inline_data; 1724 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1725 1726 ret = uverbs_response(attrs, &resp, sizeof(resp)); 1727 1728 out: 1729 kfree(attr); 1730 kfree(init_attr); 1731 1732 return ret; 1733 } 1734 1735 /* Remove ignored fields set in the attribute mask */ 1736 static int modify_qp_mask(enum ib_qp_type qp_type, int mask) 1737 { 1738 switch (qp_type) { 1739 case IB_QPT_XRC_INI: 1740 return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER); 1741 case IB_QPT_XRC_TGT: 1742 return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT | 1743 IB_QP_RNR_RETRY); 1744 default: 1745 return mask; 1746 } 1747 } 1748 1749 static void copy_ah_attr_from_uverbs(struct ib_device *dev, 1750 struct rdma_ah_attr *rdma_attr, 1751 struct ib_uverbs_qp_dest *uverb_attr) 1752 { 1753 rdma_attr->type = rdma_ah_find_type(dev, uverb_attr->port_num); 1754 if (uverb_attr->is_global) { 1755 rdma_ah_set_grh(rdma_attr, NULL, 1756 uverb_attr->flow_label, 1757 uverb_attr->sgid_index, 1758 uverb_attr->hop_limit, 1759 uverb_attr->traffic_class); 1760 rdma_ah_set_dgid_raw(rdma_attr, uverb_attr->dgid); 1761 } else { 1762 rdma_ah_set_ah_flags(rdma_attr, 0); 1763 } 1764 rdma_ah_set_dlid(rdma_attr, uverb_attr->dlid); 1765 rdma_ah_set_sl(rdma_attr, uverb_attr->sl); 1766 rdma_ah_set_path_bits(rdma_attr, uverb_attr->src_path_bits); 1767 rdma_ah_set_static_rate(rdma_attr, uverb_attr->static_rate); 1768 rdma_ah_set_port_num(rdma_attr, uverb_attr->port_num); 1769 rdma_ah_set_make_grd(rdma_attr, false); 1770 } 1771 1772 static int modify_qp(struct uverbs_attr_bundle *attrs, 1773 struct ib_uverbs_ex_modify_qp *cmd) 1774 { 1775 struct ib_qp_attr *attr; 1776 struct ib_qp *qp; 1777 int ret; 1778 1779 attr = kzalloc(sizeof(*attr), GFP_KERNEL); 1780 if (!attr) 1781 return -ENOMEM; 1782 1783 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd->base.qp_handle, 1784 attrs); 1785 if (!qp) { 1786 ret = -EINVAL; 1787 goto out; 1788 } 1789 1790 if ((cmd->base.attr_mask & IB_QP_PORT) && 1791 !rdma_is_port_valid(qp->device, cmd->base.port_num)) { 1792 ret = -EINVAL; 1793 goto release_qp; 1794 } 1795 1796 if ((cmd->base.attr_mask & IB_QP_AV)) { 1797 if (!rdma_is_port_valid(qp->device, cmd->base.dest.port_num)) { 1798 ret = -EINVAL; 1799 goto release_qp; 1800 } 1801 1802 if (cmd->base.attr_mask & IB_QP_STATE && 1803 cmd->base.qp_state == IB_QPS_RTR) { 1804 /* We are in INIT->RTR TRANSITION (if we are not, 1805 * this transition will be rejected in subsequent checks). 1806 * In the INIT->RTR transition, we cannot have IB_QP_PORT set, 1807 * but the IB_QP_STATE flag is required. 1808 * 1809 * Since kernel 3.14 (commit dbf727de7440), the uverbs driver, 1810 * when IB_QP_AV is set, has required inclusion of a valid 1811 * port number in the primary AV. (AVs are created and handled 1812 * differently for infiniband and ethernet (RoCE) ports). 1813 * 1814 * Check the port number included in the primary AV against 1815 * the port number in the qp struct, which was set (and saved) 1816 * in the RST->INIT transition. 1817 */ 1818 if (cmd->base.dest.port_num != qp->real_qp->port) { 1819 ret = -EINVAL; 1820 goto release_qp; 1821 } 1822 } else { 1823 /* We are in SQD->SQD. (If we are not, this transition will 1824 * be rejected later in the verbs layer checks). 1825 * Check for both IB_QP_PORT and IB_QP_AV, these can be set 1826 * together in the SQD->SQD transition. 1827 * 1828 * If only IP_QP_AV was set, add in IB_QP_PORT as well (the 1829 * verbs layer driver does not track primary port changes 1830 * resulting from path migration. Thus, in SQD, if the primary 1831 * AV is modified, the primary port should also be modified). 1832 * 1833 * Note that in this transition, the IB_QP_STATE flag 1834 * is not allowed. 1835 */ 1836 if (((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 1837 == (IB_QP_AV | IB_QP_PORT)) && 1838 cmd->base.port_num != cmd->base.dest.port_num) { 1839 ret = -EINVAL; 1840 goto release_qp; 1841 } 1842 if ((cmd->base.attr_mask & (IB_QP_AV | IB_QP_PORT)) 1843 == IB_QP_AV) { 1844 cmd->base.attr_mask |= IB_QP_PORT; 1845 cmd->base.port_num = cmd->base.dest.port_num; 1846 } 1847 } 1848 } 1849 1850 if ((cmd->base.attr_mask & IB_QP_ALT_PATH) && 1851 (!rdma_is_port_valid(qp->device, cmd->base.alt_port_num) || 1852 !rdma_is_port_valid(qp->device, cmd->base.alt_dest.port_num) || 1853 cmd->base.alt_port_num != cmd->base.alt_dest.port_num)) { 1854 ret = -EINVAL; 1855 goto release_qp; 1856 } 1857 1858 if ((cmd->base.attr_mask & IB_QP_CUR_STATE && 1859 cmd->base.cur_qp_state > IB_QPS_ERR) || 1860 (cmd->base.attr_mask & IB_QP_STATE && 1861 cmd->base.qp_state > IB_QPS_ERR)) { 1862 ret = -EINVAL; 1863 goto release_qp; 1864 } 1865 1866 if (cmd->base.attr_mask & IB_QP_STATE) 1867 attr->qp_state = cmd->base.qp_state; 1868 if (cmd->base.attr_mask & IB_QP_CUR_STATE) 1869 attr->cur_qp_state = cmd->base.cur_qp_state; 1870 if (cmd->base.attr_mask & IB_QP_PATH_MTU) 1871 attr->path_mtu = cmd->base.path_mtu; 1872 if (cmd->base.attr_mask & IB_QP_PATH_MIG_STATE) 1873 attr->path_mig_state = cmd->base.path_mig_state; 1874 if (cmd->base.attr_mask & IB_QP_QKEY) 1875 attr->qkey = cmd->base.qkey; 1876 if (cmd->base.attr_mask & IB_QP_RQ_PSN) 1877 attr->rq_psn = cmd->base.rq_psn; 1878 if (cmd->base.attr_mask & IB_QP_SQ_PSN) 1879 attr->sq_psn = cmd->base.sq_psn; 1880 if (cmd->base.attr_mask & IB_QP_DEST_QPN) 1881 attr->dest_qp_num = cmd->base.dest_qp_num; 1882 if (cmd->base.attr_mask & IB_QP_ACCESS_FLAGS) 1883 attr->qp_access_flags = cmd->base.qp_access_flags; 1884 if (cmd->base.attr_mask & IB_QP_PKEY_INDEX) 1885 attr->pkey_index = cmd->base.pkey_index; 1886 if (cmd->base.attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 1887 attr->en_sqd_async_notify = cmd->base.en_sqd_async_notify; 1888 if (cmd->base.attr_mask & IB_QP_MAX_QP_RD_ATOMIC) 1889 attr->max_rd_atomic = cmd->base.max_rd_atomic; 1890 if (cmd->base.attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) 1891 attr->max_dest_rd_atomic = cmd->base.max_dest_rd_atomic; 1892 if (cmd->base.attr_mask & IB_QP_MIN_RNR_TIMER) 1893 attr->min_rnr_timer = cmd->base.min_rnr_timer; 1894 if (cmd->base.attr_mask & IB_QP_PORT) 1895 attr->port_num = cmd->base.port_num; 1896 if (cmd->base.attr_mask & IB_QP_TIMEOUT) 1897 attr->timeout = cmd->base.timeout; 1898 if (cmd->base.attr_mask & IB_QP_RETRY_CNT) 1899 attr->retry_cnt = cmd->base.retry_cnt; 1900 if (cmd->base.attr_mask & IB_QP_RNR_RETRY) 1901 attr->rnr_retry = cmd->base.rnr_retry; 1902 if (cmd->base.attr_mask & IB_QP_ALT_PATH) { 1903 attr->alt_port_num = cmd->base.alt_port_num; 1904 attr->alt_timeout = cmd->base.alt_timeout; 1905 attr->alt_pkey_index = cmd->base.alt_pkey_index; 1906 } 1907 if (cmd->base.attr_mask & IB_QP_RATE_LIMIT) 1908 attr->rate_limit = cmd->rate_limit; 1909 1910 if (cmd->base.attr_mask & IB_QP_AV) 1911 copy_ah_attr_from_uverbs(qp->device, &attr->ah_attr, 1912 &cmd->base.dest); 1913 1914 if (cmd->base.attr_mask & IB_QP_ALT_PATH) 1915 copy_ah_attr_from_uverbs(qp->device, &attr->alt_ah_attr, 1916 &cmd->base.alt_dest); 1917 1918 ret = ib_modify_qp_with_udata(qp, attr, 1919 modify_qp_mask(qp->qp_type, 1920 cmd->base.attr_mask), 1921 &attrs->driver_udata); 1922 1923 release_qp: 1924 uobj_put_obj_read(qp); 1925 out: 1926 kfree(attr); 1927 1928 return ret; 1929 } 1930 1931 static int ib_uverbs_modify_qp(struct uverbs_attr_bundle *attrs) 1932 { 1933 struct ib_uverbs_ex_modify_qp cmd; 1934 int ret; 1935 1936 ret = uverbs_request(attrs, &cmd.base, sizeof(cmd.base)); 1937 if (ret) 1938 return ret; 1939 1940 if (cmd.base.attr_mask & 1941 ~((IB_USER_LEGACY_LAST_QP_ATTR_MASK << 1) - 1)) 1942 return -EOPNOTSUPP; 1943 1944 return modify_qp(attrs, &cmd); 1945 } 1946 1947 static int ib_uverbs_ex_modify_qp(struct uverbs_attr_bundle *attrs) 1948 { 1949 struct ib_uverbs_ex_modify_qp cmd; 1950 struct ib_uverbs_ex_modify_qp_resp resp = { 1951 .response_length = uverbs_response_length(attrs, sizeof(resp)) 1952 }; 1953 int ret; 1954 1955 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1956 if (ret) 1957 return ret; 1958 1959 /* 1960 * Last bit is reserved for extending the attr_mask by 1961 * using another field. 1962 */ 1963 BUILD_BUG_ON(IB_USER_LAST_QP_ATTR_MASK == (1 << 31)); 1964 1965 if (cmd.base.attr_mask & 1966 ~((IB_USER_LAST_QP_ATTR_MASK << 1) - 1)) 1967 return -EOPNOTSUPP; 1968 1969 ret = modify_qp(attrs, &cmd); 1970 if (ret) 1971 return ret; 1972 1973 return uverbs_response(attrs, &resp, sizeof(resp)); 1974 } 1975 1976 static int ib_uverbs_destroy_qp(struct uverbs_attr_bundle *attrs) 1977 { 1978 struct ib_uverbs_destroy_qp cmd; 1979 struct ib_uverbs_destroy_qp_resp resp; 1980 struct ib_uobject *uobj; 1981 struct ib_uqp_object *obj; 1982 int ret; 1983 1984 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 1985 if (ret) 1986 return ret; 1987 1988 uobj = uobj_get_destroy(UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 1989 if (IS_ERR(uobj)) 1990 return PTR_ERR(uobj); 1991 1992 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 1993 memset(&resp, 0, sizeof(resp)); 1994 resp.events_reported = obj->uevent.events_reported; 1995 1996 uobj_put_destroy(uobj); 1997 1998 return uverbs_response(attrs, &resp, sizeof(resp)); 1999 } 2000 2001 static void *alloc_wr(size_t wr_size, __u32 num_sge) 2002 { 2003 if (num_sge >= (U32_MAX - ALIGN(wr_size, sizeof (struct ib_sge))) / 2004 sizeof (struct ib_sge)) 2005 return NULL; 2006 2007 return kmalloc(ALIGN(wr_size, sizeof (struct ib_sge)) + 2008 num_sge * sizeof (struct ib_sge), GFP_KERNEL); 2009 } 2010 2011 static int ib_uverbs_post_send(struct uverbs_attr_bundle *attrs) 2012 { 2013 struct ib_uverbs_post_send cmd; 2014 struct ib_uverbs_post_send_resp resp; 2015 struct ib_uverbs_send_wr *user_wr; 2016 struct ib_send_wr *wr = NULL, *last, *next; 2017 const struct ib_send_wr *bad_wr; 2018 struct ib_qp *qp; 2019 int i, sg_ind; 2020 int is_ud; 2021 int ret, ret2; 2022 size_t next_size; 2023 const struct ib_sge __user *sgls; 2024 const void __user *wqes; 2025 struct uverbs_req_iter iter; 2026 2027 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 2028 if (ret) 2029 return ret; 2030 wqes = uverbs_request_next_ptr(&iter, cmd.wqe_size * cmd.wr_count); 2031 if (IS_ERR(wqes)) 2032 return PTR_ERR(wqes); 2033 sgls = uverbs_request_next_ptr( 2034 &iter, cmd.sge_count * sizeof(struct ib_uverbs_sge)); 2035 if (IS_ERR(sgls)) 2036 return PTR_ERR(sgls); 2037 ret = uverbs_request_finish(&iter); 2038 if (ret) 2039 return ret; 2040 2041 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 2042 if (!user_wr) 2043 return -ENOMEM; 2044 2045 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2046 if (!qp) { 2047 ret = -EINVAL; 2048 goto out; 2049 } 2050 2051 is_ud = qp->qp_type == IB_QPT_UD; 2052 sg_ind = 0; 2053 last = NULL; 2054 for (i = 0; i < cmd.wr_count; ++i) { 2055 if (copy_from_user(user_wr, wqes + i * cmd.wqe_size, 2056 cmd.wqe_size)) { 2057 ret = -EFAULT; 2058 goto out_put; 2059 } 2060 2061 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 2062 ret = -EINVAL; 2063 goto out_put; 2064 } 2065 2066 if (is_ud) { 2067 struct ib_ud_wr *ud; 2068 2069 if (user_wr->opcode != IB_WR_SEND && 2070 user_wr->opcode != IB_WR_SEND_WITH_IMM) { 2071 ret = -EINVAL; 2072 goto out_put; 2073 } 2074 2075 next_size = sizeof(*ud); 2076 ud = alloc_wr(next_size, user_wr->num_sge); 2077 if (!ud) { 2078 ret = -ENOMEM; 2079 goto out_put; 2080 } 2081 2082 ud->ah = uobj_get_obj_read(ah, UVERBS_OBJECT_AH, 2083 user_wr->wr.ud.ah, attrs); 2084 if (!ud->ah) { 2085 kfree(ud); 2086 ret = -EINVAL; 2087 goto out_put; 2088 } 2089 ud->remote_qpn = user_wr->wr.ud.remote_qpn; 2090 ud->remote_qkey = user_wr->wr.ud.remote_qkey; 2091 2092 next = &ud->wr; 2093 } else if (user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 2094 user_wr->opcode == IB_WR_RDMA_WRITE || 2095 user_wr->opcode == IB_WR_RDMA_READ) { 2096 struct ib_rdma_wr *rdma; 2097 2098 next_size = sizeof(*rdma); 2099 rdma = alloc_wr(next_size, user_wr->num_sge); 2100 if (!rdma) { 2101 ret = -ENOMEM; 2102 goto out_put; 2103 } 2104 2105 rdma->remote_addr = user_wr->wr.rdma.remote_addr; 2106 rdma->rkey = user_wr->wr.rdma.rkey; 2107 2108 next = &rdma->wr; 2109 } else if (user_wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2110 user_wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2111 struct ib_atomic_wr *atomic; 2112 2113 next_size = sizeof(*atomic); 2114 atomic = alloc_wr(next_size, user_wr->num_sge); 2115 if (!atomic) { 2116 ret = -ENOMEM; 2117 goto out_put; 2118 } 2119 2120 atomic->remote_addr = user_wr->wr.atomic.remote_addr; 2121 atomic->compare_add = user_wr->wr.atomic.compare_add; 2122 atomic->swap = user_wr->wr.atomic.swap; 2123 atomic->rkey = user_wr->wr.atomic.rkey; 2124 2125 next = &atomic->wr; 2126 } else if (user_wr->opcode == IB_WR_SEND || 2127 user_wr->opcode == IB_WR_SEND_WITH_IMM || 2128 user_wr->opcode == IB_WR_SEND_WITH_INV) { 2129 next_size = sizeof(*next); 2130 next = alloc_wr(next_size, user_wr->num_sge); 2131 if (!next) { 2132 ret = -ENOMEM; 2133 goto out_put; 2134 } 2135 } else { 2136 ret = -EINVAL; 2137 goto out_put; 2138 } 2139 2140 if (user_wr->opcode == IB_WR_SEND_WITH_IMM || 2141 user_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 2142 next->ex.imm_data = 2143 (__be32 __force) user_wr->ex.imm_data; 2144 } else if (user_wr->opcode == IB_WR_SEND_WITH_INV) { 2145 next->ex.invalidate_rkey = user_wr->ex.invalidate_rkey; 2146 } 2147 2148 if (!last) 2149 wr = next; 2150 else 2151 last->next = next; 2152 last = next; 2153 2154 next->next = NULL; 2155 next->wr_id = user_wr->wr_id; 2156 next->num_sge = user_wr->num_sge; 2157 next->opcode = user_wr->opcode; 2158 next->send_flags = user_wr->send_flags; 2159 2160 if (next->num_sge) { 2161 next->sg_list = (void *) next + 2162 ALIGN(next_size, sizeof(struct ib_sge)); 2163 if (copy_from_user(next->sg_list, sgls + sg_ind, 2164 next->num_sge * 2165 sizeof(struct ib_sge))) { 2166 ret = -EFAULT; 2167 goto out_put; 2168 } 2169 sg_ind += next->num_sge; 2170 } else 2171 next->sg_list = NULL; 2172 } 2173 2174 resp.bad_wr = 0; 2175 ret = qp->device->ops.post_send(qp->real_qp, wr, &bad_wr); 2176 if (ret) 2177 for (next = wr; next; next = next->next) { 2178 ++resp.bad_wr; 2179 if (next == bad_wr) 2180 break; 2181 } 2182 2183 ret2 = uverbs_response(attrs, &resp, sizeof(resp)); 2184 if (ret2) 2185 ret = ret2; 2186 2187 out_put: 2188 uobj_put_obj_read(qp); 2189 2190 while (wr) { 2191 if (is_ud && ud_wr(wr)->ah) 2192 uobj_put_obj_read(ud_wr(wr)->ah); 2193 next = wr->next; 2194 kfree(wr); 2195 wr = next; 2196 } 2197 2198 out: 2199 kfree(user_wr); 2200 2201 return ret; 2202 } 2203 2204 static struct ib_recv_wr * 2205 ib_uverbs_unmarshall_recv(struct uverbs_req_iter *iter, u32 wr_count, 2206 u32 wqe_size, u32 sge_count) 2207 { 2208 struct ib_uverbs_recv_wr *user_wr; 2209 struct ib_recv_wr *wr = NULL, *last, *next; 2210 int sg_ind; 2211 int i; 2212 int ret; 2213 const struct ib_sge __user *sgls; 2214 const void __user *wqes; 2215 2216 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 2217 return ERR_PTR(-EINVAL); 2218 2219 wqes = uverbs_request_next_ptr(iter, wqe_size * wr_count); 2220 if (IS_ERR(wqes)) 2221 return ERR_CAST(wqes); 2222 sgls = uverbs_request_next_ptr( 2223 iter, sge_count * sizeof(struct ib_uverbs_sge)); 2224 if (IS_ERR(sgls)) 2225 return ERR_CAST(sgls); 2226 ret = uverbs_request_finish(iter); 2227 if (ret) 2228 return ERR_PTR(ret); 2229 2230 user_wr = kmalloc(wqe_size, GFP_KERNEL); 2231 if (!user_wr) 2232 return ERR_PTR(-ENOMEM); 2233 2234 sg_ind = 0; 2235 last = NULL; 2236 for (i = 0; i < wr_count; ++i) { 2237 if (copy_from_user(user_wr, wqes + i * wqe_size, 2238 wqe_size)) { 2239 ret = -EFAULT; 2240 goto err; 2241 } 2242 2243 if (user_wr->num_sge + sg_ind > sge_count) { 2244 ret = -EINVAL; 2245 goto err; 2246 } 2247 2248 if (user_wr->num_sge >= 2249 (U32_MAX - ALIGN(sizeof *next, sizeof (struct ib_sge))) / 2250 sizeof (struct ib_sge)) { 2251 ret = -EINVAL; 2252 goto err; 2253 } 2254 2255 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 2256 user_wr->num_sge * sizeof (struct ib_sge), 2257 GFP_KERNEL); 2258 if (!next) { 2259 ret = -ENOMEM; 2260 goto err; 2261 } 2262 2263 if (!last) 2264 wr = next; 2265 else 2266 last->next = next; 2267 last = next; 2268 2269 next->next = NULL; 2270 next->wr_id = user_wr->wr_id; 2271 next->num_sge = user_wr->num_sge; 2272 2273 if (next->num_sge) { 2274 next->sg_list = (void *) next + 2275 ALIGN(sizeof *next, sizeof (struct ib_sge)); 2276 if (copy_from_user(next->sg_list, sgls + sg_ind, 2277 next->num_sge * 2278 sizeof(struct ib_sge))) { 2279 ret = -EFAULT; 2280 goto err; 2281 } 2282 sg_ind += next->num_sge; 2283 } else 2284 next->sg_list = NULL; 2285 } 2286 2287 kfree(user_wr); 2288 return wr; 2289 2290 err: 2291 kfree(user_wr); 2292 2293 while (wr) { 2294 next = wr->next; 2295 kfree(wr); 2296 wr = next; 2297 } 2298 2299 return ERR_PTR(ret); 2300 } 2301 2302 static int ib_uverbs_post_recv(struct uverbs_attr_bundle *attrs) 2303 { 2304 struct ib_uverbs_post_recv cmd; 2305 struct ib_uverbs_post_recv_resp resp; 2306 struct ib_recv_wr *wr, *next; 2307 const struct ib_recv_wr *bad_wr; 2308 struct ib_qp *qp; 2309 int ret, ret2; 2310 struct uverbs_req_iter iter; 2311 2312 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 2313 if (ret) 2314 return ret; 2315 2316 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size, 2317 cmd.sge_count); 2318 if (IS_ERR(wr)) 2319 return PTR_ERR(wr); 2320 2321 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2322 if (!qp) { 2323 ret = -EINVAL; 2324 goto out; 2325 } 2326 2327 resp.bad_wr = 0; 2328 ret = qp->device->ops.post_recv(qp->real_qp, wr, &bad_wr); 2329 2330 uobj_put_obj_read(qp); 2331 if (ret) { 2332 for (next = wr; next; next = next->next) { 2333 ++resp.bad_wr; 2334 if (next == bad_wr) 2335 break; 2336 } 2337 } 2338 2339 ret2 = uverbs_response(attrs, &resp, sizeof(resp)); 2340 if (ret2) 2341 ret = ret2; 2342 out: 2343 while (wr) { 2344 next = wr->next; 2345 kfree(wr); 2346 wr = next; 2347 } 2348 2349 return ret; 2350 } 2351 2352 static int ib_uverbs_post_srq_recv(struct uverbs_attr_bundle *attrs) 2353 { 2354 struct ib_uverbs_post_srq_recv cmd; 2355 struct ib_uverbs_post_srq_recv_resp resp; 2356 struct ib_recv_wr *wr, *next; 2357 const struct ib_recv_wr *bad_wr; 2358 struct ib_srq *srq; 2359 int ret, ret2; 2360 struct uverbs_req_iter iter; 2361 2362 ret = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 2363 if (ret) 2364 return ret; 2365 2366 wr = ib_uverbs_unmarshall_recv(&iter, cmd.wr_count, cmd.wqe_size, 2367 cmd.sge_count); 2368 if (IS_ERR(wr)) 2369 return PTR_ERR(wr); 2370 2371 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 2372 if (!srq) { 2373 ret = -EINVAL; 2374 goto out; 2375 } 2376 2377 resp.bad_wr = 0; 2378 ret = srq->device->ops.post_srq_recv(srq, wr, &bad_wr); 2379 2380 uobj_put_obj_read(srq); 2381 2382 if (ret) 2383 for (next = wr; next; next = next->next) { 2384 ++resp.bad_wr; 2385 if (next == bad_wr) 2386 break; 2387 } 2388 2389 ret2 = uverbs_response(attrs, &resp, sizeof(resp)); 2390 if (ret2) 2391 ret = ret2; 2392 2393 out: 2394 while (wr) { 2395 next = wr->next; 2396 kfree(wr); 2397 wr = next; 2398 } 2399 2400 return ret; 2401 } 2402 2403 static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs) 2404 { 2405 struct ib_uverbs_create_ah cmd; 2406 struct ib_uverbs_create_ah_resp resp; 2407 struct ib_uobject *uobj; 2408 struct ib_pd *pd; 2409 struct ib_ah *ah; 2410 struct rdma_ah_attr attr = {}; 2411 int ret; 2412 struct ib_device *ib_dev; 2413 2414 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2415 if (ret) 2416 return ret; 2417 2418 uobj = uobj_alloc(UVERBS_OBJECT_AH, attrs, &ib_dev); 2419 if (IS_ERR(uobj)) 2420 return PTR_ERR(uobj); 2421 2422 if (!rdma_is_port_valid(ib_dev, cmd.attr.port_num)) { 2423 ret = -EINVAL; 2424 goto err; 2425 } 2426 2427 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 2428 if (!pd) { 2429 ret = -EINVAL; 2430 goto err; 2431 } 2432 2433 attr.type = rdma_ah_find_type(ib_dev, cmd.attr.port_num); 2434 rdma_ah_set_make_grd(&attr, false); 2435 rdma_ah_set_dlid(&attr, cmd.attr.dlid); 2436 rdma_ah_set_sl(&attr, cmd.attr.sl); 2437 rdma_ah_set_path_bits(&attr, cmd.attr.src_path_bits); 2438 rdma_ah_set_static_rate(&attr, cmd.attr.static_rate); 2439 rdma_ah_set_port_num(&attr, cmd.attr.port_num); 2440 2441 if (cmd.attr.is_global) { 2442 rdma_ah_set_grh(&attr, NULL, cmd.attr.grh.flow_label, 2443 cmd.attr.grh.sgid_index, 2444 cmd.attr.grh.hop_limit, 2445 cmd.attr.grh.traffic_class); 2446 rdma_ah_set_dgid_raw(&attr, cmd.attr.grh.dgid); 2447 } else { 2448 rdma_ah_set_ah_flags(&attr, 0); 2449 } 2450 2451 ah = rdma_create_user_ah(pd, &attr, &attrs->driver_udata); 2452 if (IS_ERR(ah)) { 2453 ret = PTR_ERR(ah); 2454 goto err_put; 2455 } 2456 2457 ah->uobject = uobj; 2458 uobj->user_handle = cmd.user_handle; 2459 uobj->object = ah; 2460 2461 resp.ah_handle = uobj->id; 2462 2463 ret = uverbs_response(attrs, &resp, sizeof(resp)); 2464 if (ret) 2465 goto err_copy; 2466 2467 uobj_put_obj_read(pd); 2468 return uobj_alloc_commit(uobj, attrs); 2469 2470 err_copy: 2471 rdma_destroy_ah_user(ah, RDMA_DESTROY_AH_SLEEPABLE, 2472 uverbs_get_cleared_udata(attrs)); 2473 2474 err_put: 2475 uobj_put_obj_read(pd); 2476 2477 err: 2478 uobj_alloc_abort(uobj, attrs); 2479 return ret; 2480 } 2481 2482 static int ib_uverbs_destroy_ah(struct uverbs_attr_bundle *attrs) 2483 { 2484 struct ib_uverbs_destroy_ah cmd; 2485 int ret; 2486 2487 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2488 if (ret) 2489 return ret; 2490 2491 return uobj_perform_destroy(UVERBS_OBJECT_AH, cmd.ah_handle, attrs); 2492 } 2493 2494 static int ib_uverbs_attach_mcast(struct uverbs_attr_bundle *attrs) 2495 { 2496 struct ib_uverbs_attach_mcast cmd; 2497 struct ib_qp *qp; 2498 struct ib_uqp_object *obj; 2499 struct ib_uverbs_mcast_entry *mcast; 2500 int ret; 2501 2502 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2503 if (ret) 2504 return ret; 2505 2506 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2507 if (!qp) 2508 return -EINVAL; 2509 2510 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2511 2512 mutex_lock(&obj->mcast_lock); 2513 list_for_each_entry(mcast, &obj->mcast_list, list) 2514 if (cmd.mlid == mcast->lid && 2515 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2516 ret = 0; 2517 goto out_put; 2518 } 2519 2520 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 2521 if (!mcast) { 2522 ret = -ENOMEM; 2523 goto out_put; 2524 } 2525 2526 mcast->lid = cmd.mlid; 2527 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 2528 2529 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 2530 if (!ret) 2531 list_add_tail(&mcast->list, &obj->mcast_list); 2532 else 2533 kfree(mcast); 2534 2535 out_put: 2536 mutex_unlock(&obj->mcast_lock); 2537 uobj_put_obj_read(qp); 2538 2539 return ret; 2540 } 2541 2542 static int ib_uverbs_detach_mcast(struct uverbs_attr_bundle *attrs) 2543 { 2544 struct ib_uverbs_detach_mcast cmd; 2545 struct ib_uqp_object *obj; 2546 struct ib_qp *qp; 2547 struct ib_uverbs_mcast_entry *mcast; 2548 int ret; 2549 bool found = false; 2550 2551 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 2552 if (ret) 2553 return ret; 2554 2555 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 2556 if (!qp) 2557 return -EINVAL; 2558 2559 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 2560 mutex_lock(&obj->mcast_lock); 2561 2562 list_for_each_entry(mcast, &obj->mcast_list, list) 2563 if (cmd.mlid == mcast->lid && 2564 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 2565 list_del(&mcast->list); 2566 kfree(mcast); 2567 found = true; 2568 break; 2569 } 2570 2571 if (!found) { 2572 ret = -EINVAL; 2573 goto out_put; 2574 } 2575 2576 ret = ib_detach_mcast(qp, (union ib_gid *)cmd.gid, cmd.mlid); 2577 2578 out_put: 2579 mutex_unlock(&obj->mcast_lock); 2580 uobj_put_obj_read(qp); 2581 return ret; 2582 } 2583 2584 struct ib_uflow_resources *flow_resources_alloc(size_t num_specs) 2585 { 2586 struct ib_uflow_resources *resources; 2587 2588 resources = kzalloc(sizeof(*resources), GFP_KERNEL); 2589 2590 if (!resources) 2591 return NULL; 2592 2593 if (!num_specs) 2594 goto out; 2595 2596 resources->counters = 2597 kcalloc(num_specs, sizeof(*resources->counters), GFP_KERNEL); 2598 resources->collection = 2599 kcalloc(num_specs, sizeof(*resources->collection), GFP_KERNEL); 2600 2601 if (!resources->counters || !resources->collection) 2602 goto err; 2603 2604 out: 2605 resources->max = num_specs; 2606 return resources; 2607 2608 err: 2609 kfree(resources->counters); 2610 kfree(resources); 2611 2612 return NULL; 2613 } 2614 EXPORT_SYMBOL(flow_resources_alloc); 2615 2616 void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res) 2617 { 2618 unsigned int i; 2619 2620 if (!uflow_res) 2621 return; 2622 2623 for (i = 0; i < uflow_res->collection_num; i++) 2624 atomic_dec(&uflow_res->collection[i]->usecnt); 2625 2626 for (i = 0; i < uflow_res->counters_num; i++) 2627 atomic_dec(&uflow_res->counters[i]->usecnt); 2628 2629 kfree(uflow_res->collection); 2630 kfree(uflow_res->counters); 2631 kfree(uflow_res); 2632 } 2633 EXPORT_SYMBOL(ib_uverbs_flow_resources_free); 2634 2635 void flow_resources_add(struct ib_uflow_resources *uflow_res, 2636 enum ib_flow_spec_type type, 2637 void *ibobj) 2638 { 2639 WARN_ON(uflow_res->num >= uflow_res->max); 2640 2641 switch (type) { 2642 case IB_FLOW_SPEC_ACTION_HANDLE: 2643 atomic_inc(&((struct ib_flow_action *)ibobj)->usecnt); 2644 uflow_res->collection[uflow_res->collection_num++] = 2645 (struct ib_flow_action *)ibobj; 2646 break; 2647 case IB_FLOW_SPEC_ACTION_COUNT: 2648 atomic_inc(&((struct ib_counters *)ibobj)->usecnt); 2649 uflow_res->counters[uflow_res->counters_num++] = 2650 (struct ib_counters *)ibobj; 2651 break; 2652 default: 2653 WARN_ON(1); 2654 } 2655 2656 uflow_res->num++; 2657 } 2658 EXPORT_SYMBOL(flow_resources_add); 2659 2660 static int kern_spec_to_ib_spec_action(struct uverbs_attr_bundle *attrs, 2661 struct ib_uverbs_flow_spec *kern_spec, 2662 union ib_flow_spec *ib_spec, 2663 struct ib_uflow_resources *uflow_res) 2664 { 2665 ib_spec->type = kern_spec->type; 2666 switch (ib_spec->type) { 2667 case IB_FLOW_SPEC_ACTION_TAG: 2668 if (kern_spec->flow_tag.size != 2669 sizeof(struct ib_uverbs_flow_spec_action_tag)) 2670 return -EINVAL; 2671 2672 ib_spec->flow_tag.size = sizeof(struct ib_flow_spec_action_tag); 2673 ib_spec->flow_tag.tag_id = kern_spec->flow_tag.tag_id; 2674 break; 2675 case IB_FLOW_SPEC_ACTION_DROP: 2676 if (kern_spec->drop.size != 2677 sizeof(struct ib_uverbs_flow_spec_action_drop)) 2678 return -EINVAL; 2679 2680 ib_spec->drop.size = sizeof(struct ib_flow_spec_action_drop); 2681 break; 2682 case IB_FLOW_SPEC_ACTION_HANDLE: 2683 if (kern_spec->action.size != 2684 sizeof(struct ib_uverbs_flow_spec_action_handle)) 2685 return -EOPNOTSUPP; 2686 ib_spec->action.act = uobj_get_obj_read(flow_action, 2687 UVERBS_OBJECT_FLOW_ACTION, 2688 kern_spec->action.handle, 2689 attrs); 2690 if (!ib_spec->action.act) 2691 return -EINVAL; 2692 ib_spec->action.size = 2693 sizeof(struct ib_flow_spec_action_handle); 2694 flow_resources_add(uflow_res, 2695 IB_FLOW_SPEC_ACTION_HANDLE, 2696 ib_spec->action.act); 2697 uobj_put_obj_read(ib_spec->action.act); 2698 break; 2699 case IB_FLOW_SPEC_ACTION_COUNT: 2700 if (kern_spec->flow_count.size != 2701 sizeof(struct ib_uverbs_flow_spec_action_count)) 2702 return -EINVAL; 2703 ib_spec->flow_count.counters = 2704 uobj_get_obj_read(counters, 2705 UVERBS_OBJECT_COUNTERS, 2706 kern_spec->flow_count.handle, 2707 attrs); 2708 if (!ib_spec->flow_count.counters) 2709 return -EINVAL; 2710 ib_spec->flow_count.size = 2711 sizeof(struct ib_flow_spec_action_count); 2712 flow_resources_add(uflow_res, 2713 IB_FLOW_SPEC_ACTION_COUNT, 2714 ib_spec->flow_count.counters); 2715 uobj_put_obj_read(ib_spec->flow_count.counters); 2716 break; 2717 default: 2718 return -EINVAL; 2719 } 2720 return 0; 2721 } 2722 2723 static size_t kern_spec_filter_sz(const struct ib_uverbs_flow_spec_hdr *spec) 2724 { 2725 /* Returns user space filter size, includes padding */ 2726 return (spec->size - sizeof(struct ib_uverbs_flow_spec_hdr)) / 2; 2727 } 2728 2729 static ssize_t spec_filter_size(const void *kern_spec_filter, u16 kern_filter_size, 2730 u16 ib_real_filter_sz) 2731 { 2732 /* 2733 * User space filter structures must be 64 bit aligned, otherwise this 2734 * may pass, but we won't handle additional new attributes. 2735 */ 2736 2737 if (kern_filter_size > ib_real_filter_sz) { 2738 if (memchr_inv(kern_spec_filter + 2739 ib_real_filter_sz, 0, 2740 kern_filter_size - ib_real_filter_sz)) 2741 return -EINVAL; 2742 return ib_real_filter_sz; 2743 } 2744 return kern_filter_size; 2745 } 2746 2747 int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type, 2748 const void *kern_spec_mask, 2749 const void *kern_spec_val, 2750 size_t kern_filter_sz, 2751 union ib_flow_spec *ib_spec) 2752 { 2753 ssize_t actual_filter_sz; 2754 ssize_t ib_filter_sz; 2755 2756 /* User flow spec size must be aligned to 4 bytes */ 2757 if (kern_filter_sz != ALIGN(kern_filter_sz, 4)) 2758 return -EINVAL; 2759 2760 ib_spec->type = type; 2761 2762 if (ib_spec->type == (IB_FLOW_SPEC_INNER | IB_FLOW_SPEC_VXLAN_TUNNEL)) 2763 return -EINVAL; 2764 2765 switch (ib_spec->type & ~IB_FLOW_SPEC_INNER) { 2766 case IB_FLOW_SPEC_ETH: 2767 ib_filter_sz = offsetof(struct ib_flow_eth_filter, real_sz); 2768 actual_filter_sz = spec_filter_size(kern_spec_mask, 2769 kern_filter_sz, 2770 ib_filter_sz); 2771 if (actual_filter_sz <= 0) 2772 return -EINVAL; 2773 ib_spec->size = sizeof(struct ib_flow_spec_eth); 2774 memcpy(&ib_spec->eth.val, kern_spec_val, actual_filter_sz); 2775 memcpy(&ib_spec->eth.mask, kern_spec_mask, actual_filter_sz); 2776 break; 2777 case IB_FLOW_SPEC_IPV4: 2778 ib_filter_sz = offsetof(struct ib_flow_ipv4_filter, real_sz); 2779 actual_filter_sz = spec_filter_size(kern_spec_mask, 2780 kern_filter_sz, 2781 ib_filter_sz); 2782 if (actual_filter_sz <= 0) 2783 return -EINVAL; 2784 ib_spec->size = sizeof(struct ib_flow_spec_ipv4); 2785 memcpy(&ib_spec->ipv4.val, kern_spec_val, actual_filter_sz); 2786 memcpy(&ib_spec->ipv4.mask, kern_spec_mask, actual_filter_sz); 2787 break; 2788 case IB_FLOW_SPEC_IPV6: 2789 ib_filter_sz = offsetof(struct ib_flow_ipv6_filter, real_sz); 2790 actual_filter_sz = spec_filter_size(kern_spec_mask, 2791 kern_filter_sz, 2792 ib_filter_sz); 2793 if (actual_filter_sz <= 0) 2794 return -EINVAL; 2795 ib_spec->size = sizeof(struct ib_flow_spec_ipv6); 2796 memcpy(&ib_spec->ipv6.val, kern_spec_val, actual_filter_sz); 2797 memcpy(&ib_spec->ipv6.mask, kern_spec_mask, actual_filter_sz); 2798 2799 if ((ntohl(ib_spec->ipv6.mask.flow_label)) >= BIT(20) || 2800 (ntohl(ib_spec->ipv6.val.flow_label)) >= BIT(20)) 2801 return -EINVAL; 2802 break; 2803 case IB_FLOW_SPEC_TCP: 2804 case IB_FLOW_SPEC_UDP: 2805 ib_filter_sz = offsetof(struct ib_flow_tcp_udp_filter, real_sz); 2806 actual_filter_sz = spec_filter_size(kern_spec_mask, 2807 kern_filter_sz, 2808 ib_filter_sz); 2809 if (actual_filter_sz <= 0) 2810 return -EINVAL; 2811 ib_spec->size = sizeof(struct ib_flow_spec_tcp_udp); 2812 memcpy(&ib_spec->tcp_udp.val, kern_spec_val, actual_filter_sz); 2813 memcpy(&ib_spec->tcp_udp.mask, kern_spec_mask, actual_filter_sz); 2814 break; 2815 case IB_FLOW_SPEC_VXLAN_TUNNEL: 2816 ib_filter_sz = offsetof(struct ib_flow_tunnel_filter, real_sz); 2817 actual_filter_sz = spec_filter_size(kern_spec_mask, 2818 kern_filter_sz, 2819 ib_filter_sz); 2820 if (actual_filter_sz <= 0) 2821 return -EINVAL; 2822 ib_spec->tunnel.size = sizeof(struct ib_flow_spec_tunnel); 2823 memcpy(&ib_spec->tunnel.val, kern_spec_val, actual_filter_sz); 2824 memcpy(&ib_spec->tunnel.mask, kern_spec_mask, actual_filter_sz); 2825 2826 if ((ntohl(ib_spec->tunnel.mask.tunnel_id)) >= BIT(24) || 2827 (ntohl(ib_spec->tunnel.val.tunnel_id)) >= BIT(24)) 2828 return -EINVAL; 2829 break; 2830 case IB_FLOW_SPEC_ESP: 2831 ib_filter_sz = offsetof(struct ib_flow_esp_filter, real_sz); 2832 actual_filter_sz = spec_filter_size(kern_spec_mask, 2833 kern_filter_sz, 2834 ib_filter_sz); 2835 if (actual_filter_sz <= 0) 2836 return -EINVAL; 2837 ib_spec->esp.size = sizeof(struct ib_flow_spec_esp); 2838 memcpy(&ib_spec->esp.val, kern_spec_val, actual_filter_sz); 2839 memcpy(&ib_spec->esp.mask, kern_spec_mask, actual_filter_sz); 2840 break; 2841 case IB_FLOW_SPEC_GRE: 2842 ib_filter_sz = offsetof(struct ib_flow_gre_filter, real_sz); 2843 actual_filter_sz = spec_filter_size(kern_spec_mask, 2844 kern_filter_sz, 2845 ib_filter_sz); 2846 if (actual_filter_sz <= 0) 2847 return -EINVAL; 2848 ib_spec->gre.size = sizeof(struct ib_flow_spec_gre); 2849 memcpy(&ib_spec->gre.val, kern_spec_val, actual_filter_sz); 2850 memcpy(&ib_spec->gre.mask, kern_spec_mask, actual_filter_sz); 2851 break; 2852 case IB_FLOW_SPEC_MPLS: 2853 ib_filter_sz = offsetof(struct ib_flow_mpls_filter, real_sz); 2854 actual_filter_sz = spec_filter_size(kern_spec_mask, 2855 kern_filter_sz, 2856 ib_filter_sz); 2857 if (actual_filter_sz <= 0) 2858 return -EINVAL; 2859 ib_spec->mpls.size = sizeof(struct ib_flow_spec_mpls); 2860 memcpy(&ib_spec->mpls.val, kern_spec_val, actual_filter_sz); 2861 memcpy(&ib_spec->mpls.mask, kern_spec_mask, actual_filter_sz); 2862 break; 2863 default: 2864 return -EINVAL; 2865 } 2866 return 0; 2867 } 2868 2869 static int kern_spec_to_ib_spec_filter(struct ib_uverbs_flow_spec *kern_spec, 2870 union ib_flow_spec *ib_spec) 2871 { 2872 ssize_t kern_filter_sz; 2873 void *kern_spec_mask; 2874 void *kern_spec_val; 2875 2876 kern_filter_sz = kern_spec_filter_sz(&kern_spec->hdr); 2877 2878 kern_spec_val = (void *)kern_spec + 2879 sizeof(struct ib_uverbs_flow_spec_hdr); 2880 kern_spec_mask = kern_spec_val + kern_filter_sz; 2881 2882 return ib_uverbs_kern_spec_to_ib_spec_filter(kern_spec->type, 2883 kern_spec_mask, 2884 kern_spec_val, 2885 kern_filter_sz, ib_spec); 2886 } 2887 2888 static int kern_spec_to_ib_spec(struct uverbs_attr_bundle *attrs, 2889 struct ib_uverbs_flow_spec *kern_spec, 2890 union ib_flow_spec *ib_spec, 2891 struct ib_uflow_resources *uflow_res) 2892 { 2893 if (kern_spec->reserved) 2894 return -EINVAL; 2895 2896 if (kern_spec->type >= IB_FLOW_SPEC_ACTION_TAG) 2897 return kern_spec_to_ib_spec_action(attrs, kern_spec, ib_spec, 2898 uflow_res); 2899 else 2900 return kern_spec_to_ib_spec_filter(kern_spec, ib_spec); 2901 } 2902 2903 static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs) 2904 { 2905 struct ib_uverbs_ex_create_wq cmd; 2906 struct ib_uverbs_ex_create_wq_resp resp = {}; 2907 struct ib_uwq_object *obj; 2908 int err = 0; 2909 struct ib_cq *cq; 2910 struct ib_pd *pd; 2911 struct ib_wq *wq; 2912 struct ib_wq_init_attr wq_init_attr = {}; 2913 struct ib_device *ib_dev; 2914 2915 err = uverbs_request(attrs, &cmd, sizeof(cmd)); 2916 if (err) 2917 return err; 2918 2919 if (cmd.comp_mask) 2920 return -EOPNOTSUPP; 2921 2922 obj = (struct ib_uwq_object *)uobj_alloc(UVERBS_OBJECT_WQ, attrs, 2923 &ib_dev); 2924 if (IS_ERR(obj)) 2925 return PTR_ERR(obj); 2926 2927 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd.pd_handle, attrs); 2928 if (!pd) { 2929 err = -EINVAL; 2930 goto err_uobj; 2931 } 2932 2933 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 2934 if (!cq) { 2935 err = -EINVAL; 2936 goto err_put_pd; 2937 } 2938 2939 wq_init_attr.cq = cq; 2940 wq_init_attr.max_sge = cmd.max_sge; 2941 wq_init_attr.max_wr = cmd.max_wr; 2942 wq_init_attr.wq_context = attrs->ufile; 2943 wq_init_attr.wq_type = cmd.wq_type; 2944 wq_init_attr.event_handler = ib_uverbs_wq_event_handler; 2945 wq_init_attr.create_flags = cmd.create_flags; 2946 obj->uevent.events_reported = 0; 2947 INIT_LIST_HEAD(&obj->uevent.event_list); 2948 2949 wq = pd->device->ops.create_wq(pd, &wq_init_attr, &attrs->driver_udata); 2950 if (IS_ERR(wq)) { 2951 err = PTR_ERR(wq); 2952 goto err_put_cq; 2953 } 2954 2955 wq->uobject = &obj->uevent.uobject; 2956 obj->uevent.uobject.object = wq; 2957 wq->wq_type = wq_init_attr.wq_type; 2958 wq->cq = cq; 2959 wq->pd = pd; 2960 wq->device = pd->device; 2961 wq->wq_context = wq_init_attr.wq_context; 2962 atomic_set(&wq->usecnt, 0); 2963 atomic_inc(&pd->usecnt); 2964 atomic_inc(&cq->usecnt); 2965 wq->uobject = &obj->uevent.uobject; 2966 obj->uevent.uobject.object = wq; 2967 2968 memset(&resp, 0, sizeof(resp)); 2969 resp.wq_handle = obj->uevent.uobject.id; 2970 resp.max_sge = wq_init_attr.max_sge; 2971 resp.max_wr = wq_init_attr.max_wr; 2972 resp.wqn = wq->wq_num; 2973 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 2974 err = uverbs_response(attrs, &resp, sizeof(resp)); 2975 if (err) 2976 goto err_copy; 2977 2978 uobj_put_obj_read(pd); 2979 uobj_put_obj_read(cq); 2980 return uobj_alloc_commit(&obj->uevent.uobject, attrs); 2981 2982 err_copy: 2983 ib_destroy_wq(wq, uverbs_get_cleared_udata(attrs)); 2984 err_put_cq: 2985 uobj_put_obj_read(cq); 2986 err_put_pd: 2987 uobj_put_obj_read(pd); 2988 err_uobj: 2989 uobj_alloc_abort(&obj->uevent.uobject, attrs); 2990 2991 return err; 2992 } 2993 2994 static int ib_uverbs_ex_destroy_wq(struct uverbs_attr_bundle *attrs) 2995 { 2996 struct ib_uverbs_ex_destroy_wq cmd; 2997 struct ib_uverbs_ex_destroy_wq_resp resp = {}; 2998 struct ib_uobject *uobj; 2999 struct ib_uwq_object *obj; 3000 int ret; 3001 3002 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3003 if (ret) 3004 return ret; 3005 3006 if (cmd.comp_mask) 3007 return -EOPNOTSUPP; 3008 3009 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 3010 uobj = uobj_get_destroy(UVERBS_OBJECT_WQ, cmd.wq_handle, attrs); 3011 if (IS_ERR(uobj)) 3012 return PTR_ERR(uobj); 3013 3014 obj = container_of(uobj, struct ib_uwq_object, uevent.uobject); 3015 resp.events_reported = obj->uevent.events_reported; 3016 3017 uobj_put_destroy(uobj); 3018 3019 return uverbs_response(attrs, &resp, sizeof(resp)); 3020 } 3021 3022 static int ib_uverbs_ex_modify_wq(struct uverbs_attr_bundle *attrs) 3023 { 3024 struct ib_uverbs_ex_modify_wq cmd; 3025 struct ib_wq *wq; 3026 struct ib_wq_attr wq_attr = {}; 3027 int ret; 3028 3029 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3030 if (ret) 3031 return ret; 3032 3033 if (!cmd.attr_mask) 3034 return -EINVAL; 3035 3036 if (cmd.attr_mask > (IB_WQ_STATE | IB_WQ_CUR_STATE | IB_WQ_FLAGS)) 3037 return -EINVAL; 3038 3039 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, cmd.wq_handle, attrs); 3040 if (!wq) 3041 return -EINVAL; 3042 3043 wq_attr.curr_wq_state = cmd.curr_wq_state; 3044 wq_attr.wq_state = cmd.wq_state; 3045 if (cmd.attr_mask & IB_WQ_FLAGS) { 3046 wq_attr.flags = cmd.flags; 3047 wq_attr.flags_mask = cmd.flags_mask; 3048 } 3049 ret = wq->device->ops.modify_wq(wq, &wq_attr, cmd.attr_mask, 3050 &attrs->driver_udata); 3051 uobj_put_obj_read(wq); 3052 return ret; 3053 } 3054 3055 static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs) 3056 { 3057 struct ib_uverbs_ex_create_rwq_ind_table cmd; 3058 struct ib_uverbs_ex_create_rwq_ind_table_resp resp = {}; 3059 struct ib_uobject *uobj; 3060 int err; 3061 struct ib_rwq_ind_table_init_attr init_attr = {}; 3062 struct ib_rwq_ind_table *rwq_ind_tbl; 3063 struct ib_wq **wqs = NULL; 3064 u32 *wqs_handles = NULL; 3065 struct ib_wq *wq = NULL; 3066 int i, j, num_read_wqs; 3067 u32 num_wq_handles; 3068 struct uverbs_req_iter iter; 3069 struct ib_device *ib_dev; 3070 3071 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 3072 if (err) 3073 return err; 3074 3075 if (cmd.comp_mask) 3076 return -EOPNOTSUPP; 3077 3078 if (cmd.log_ind_tbl_size > IB_USER_VERBS_MAX_LOG_IND_TBL_SIZE) 3079 return -EINVAL; 3080 3081 num_wq_handles = 1 << cmd.log_ind_tbl_size; 3082 wqs_handles = kcalloc(num_wq_handles, sizeof(*wqs_handles), 3083 GFP_KERNEL); 3084 if (!wqs_handles) 3085 return -ENOMEM; 3086 3087 err = uverbs_request_next(&iter, wqs_handles, 3088 num_wq_handles * sizeof(__u32)); 3089 if (err) 3090 goto err_free; 3091 3092 err = uverbs_request_finish(&iter); 3093 if (err) 3094 goto err_free; 3095 3096 wqs = kcalloc(num_wq_handles, sizeof(*wqs), GFP_KERNEL); 3097 if (!wqs) { 3098 err = -ENOMEM; 3099 goto err_free; 3100 } 3101 3102 for (num_read_wqs = 0; num_read_wqs < num_wq_handles; 3103 num_read_wqs++) { 3104 wq = uobj_get_obj_read(wq, UVERBS_OBJECT_WQ, 3105 wqs_handles[num_read_wqs], attrs); 3106 if (!wq) { 3107 err = -EINVAL; 3108 goto put_wqs; 3109 } 3110 3111 wqs[num_read_wqs] = wq; 3112 } 3113 3114 uobj = uobj_alloc(UVERBS_OBJECT_RWQ_IND_TBL, attrs, &ib_dev); 3115 if (IS_ERR(uobj)) { 3116 err = PTR_ERR(uobj); 3117 goto put_wqs; 3118 } 3119 3120 init_attr.log_ind_tbl_size = cmd.log_ind_tbl_size; 3121 init_attr.ind_tbl = wqs; 3122 3123 rwq_ind_tbl = ib_dev->ops.create_rwq_ind_table(ib_dev, &init_attr, 3124 &attrs->driver_udata); 3125 3126 if (IS_ERR(rwq_ind_tbl)) { 3127 err = PTR_ERR(rwq_ind_tbl); 3128 goto err_uobj; 3129 } 3130 3131 rwq_ind_tbl->ind_tbl = wqs; 3132 rwq_ind_tbl->log_ind_tbl_size = init_attr.log_ind_tbl_size; 3133 rwq_ind_tbl->uobject = uobj; 3134 uobj->object = rwq_ind_tbl; 3135 rwq_ind_tbl->device = ib_dev; 3136 atomic_set(&rwq_ind_tbl->usecnt, 0); 3137 3138 for (i = 0; i < num_wq_handles; i++) 3139 atomic_inc(&wqs[i]->usecnt); 3140 3141 resp.ind_tbl_handle = uobj->id; 3142 resp.ind_tbl_num = rwq_ind_tbl->ind_tbl_num; 3143 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 3144 3145 err = uverbs_response(attrs, &resp, sizeof(resp)); 3146 if (err) 3147 goto err_copy; 3148 3149 kfree(wqs_handles); 3150 3151 for (j = 0; j < num_read_wqs; j++) 3152 uobj_put_obj_read(wqs[j]); 3153 3154 return uobj_alloc_commit(uobj, attrs); 3155 3156 err_copy: 3157 ib_destroy_rwq_ind_table(rwq_ind_tbl); 3158 err_uobj: 3159 uobj_alloc_abort(uobj, attrs); 3160 put_wqs: 3161 for (j = 0; j < num_read_wqs; j++) 3162 uobj_put_obj_read(wqs[j]); 3163 err_free: 3164 kfree(wqs_handles); 3165 kfree(wqs); 3166 return err; 3167 } 3168 3169 static int ib_uverbs_ex_destroy_rwq_ind_table(struct uverbs_attr_bundle *attrs) 3170 { 3171 struct ib_uverbs_ex_destroy_rwq_ind_table cmd; 3172 int ret; 3173 3174 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3175 if (ret) 3176 return ret; 3177 3178 if (cmd.comp_mask) 3179 return -EOPNOTSUPP; 3180 3181 return uobj_perform_destroy(UVERBS_OBJECT_RWQ_IND_TBL, 3182 cmd.ind_tbl_handle, attrs); 3183 } 3184 3185 static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs) 3186 { 3187 struct ib_uverbs_create_flow cmd; 3188 struct ib_uverbs_create_flow_resp resp; 3189 struct ib_uobject *uobj; 3190 struct ib_flow *flow_id; 3191 struct ib_uverbs_flow_attr *kern_flow_attr; 3192 struct ib_flow_attr *flow_attr; 3193 struct ib_qp *qp; 3194 struct ib_uflow_resources *uflow_res; 3195 struct ib_uverbs_flow_spec_hdr *kern_spec; 3196 struct uverbs_req_iter iter; 3197 int err; 3198 void *ib_spec; 3199 int i; 3200 struct ib_device *ib_dev; 3201 3202 err = uverbs_request_start(attrs, &iter, &cmd, sizeof(cmd)); 3203 if (err) 3204 return err; 3205 3206 if (cmd.comp_mask) 3207 return -EINVAL; 3208 3209 if (!capable(CAP_NET_RAW)) 3210 return -EPERM; 3211 3212 if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED) 3213 return -EINVAL; 3214 3215 if ((cmd.flow_attr.flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) && 3216 ((cmd.flow_attr.type == IB_FLOW_ATTR_ALL_DEFAULT) || 3217 (cmd.flow_attr.type == IB_FLOW_ATTR_MC_DEFAULT))) 3218 return -EINVAL; 3219 3220 if (cmd.flow_attr.num_of_specs > IB_FLOW_SPEC_SUPPORT_LAYERS) 3221 return -EINVAL; 3222 3223 if (cmd.flow_attr.size > 3224 (cmd.flow_attr.num_of_specs * sizeof(struct ib_uverbs_flow_spec))) 3225 return -EINVAL; 3226 3227 if (cmd.flow_attr.reserved[0] || 3228 cmd.flow_attr.reserved[1]) 3229 return -EINVAL; 3230 3231 if (cmd.flow_attr.num_of_specs) { 3232 kern_flow_attr = kmalloc(sizeof(*kern_flow_attr) + cmd.flow_attr.size, 3233 GFP_KERNEL); 3234 if (!kern_flow_attr) 3235 return -ENOMEM; 3236 3237 *kern_flow_attr = cmd.flow_attr; 3238 err = uverbs_request_next(&iter, &kern_flow_attr->flow_specs, 3239 cmd.flow_attr.size); 3240 if (err) 3241 goto err_free_attr; 3242 } else { 3243 kern_flow_attr = &cmd.flow_attr; 3244 } 3245 3246 err = uverbs_request_finish(&iter); 3247 if (err) 3248 goto err_free_attr; 3249 3250 uobj = uobj_alloc(UVERBS_OBJECT_FLOW, attrs, &ib_dev); 3251 if (IS_ERR(uobj)) { 3252 err = PTR_ERR(uobj); 3253 goto err_free_attr; 3254 } 3255 3256 qp = uobj_get_obj_read(qp, UVERBS_OBJECT_QP, cmd.qp_handle, attrs); 3257 if (!qp) { 3258 err = -EINVAL; 3259 goto err_uobj; 3260 } 3261 3262 if (qp->qp_type != IB_QPT_UD && qp->qp_type != IB_QPT_RAW_PACKET) { 3263 err = -EINVAL; 3264 goto err_put; 3265 } 3266 3267 flow_attr = kzalloc(struct_size(flow_attr, flows, 3268 cmd.flow_attr.num_of_specs), GFP_KERNEL); 3269 if (!flow_attr) { 3270 err = -ENOMEM; 3271 goto err_put; 3272 } 3273 uflow_res = flow_resources_alloc(cmd.flow_attr.num_of_specs); 3274 if (!uflow_res) { 3275 err = -ENOMEM; 3276 goto err_free_flow_attr; 3277 } 3278 3279 flow_attr->type = kern_flow_attr->type; 3280 flow_attr->priority = kern_flow_attr->priority; 3281 flow_attr->num_of_specs = kern_flow_attr->num_of_specs; 3282 flow_attr->port = kern_flow_attr->port; 3283 flow_attr->flags = kern_flow_attr->flags; 3284 flow_attr->size = sizeof(*flow_attr); 3285 3286 kern_spec = kern_flow_attr->flow_specs; 3287 ib_spec = flow_attr + 1; 3288 for (i = 0; i < flow_attr->num_of_specs && 3289 cmd.flow_attr.size >= sizeof(*kern_spec) && 3290 cmd.flow_attr.size >= kern_spec->size; 3291 i++) { 3292 err = kern_spec_to_ib_spec( 3293 attrs, (struct ib_uverbs_flow_spec *)kern_spec, 3294 ib_spec, uflow_res); 3295 if (err) 3296 goto err_free; 3297 3298 flow_attr->size += 3299 ((union ib_flow_spec *) ib_spec)->size; 3300 cmd.flow_attr.size -= kern_spec->size; 3301 kern_spec = ((void *)kern_spec) + kern_spec->size; 3302 ib_spec += ((union ib_flow_spec *) ib_spec)->size; 3303 } 3304 if (cmd.flow_attr.size || (i != flow_attr->num_of_specs)) { 3305 pr_warn("create flow failed, flow %d: %d bytes left from uverb cmd\n", 3306 i, cmd.flow_attr.size); 3307 err = -EINVAL; 3308 goto err_free; 3309 } 3310 3311 flow_id = qp->device->ops.create_flow( 3312 qp, flow_attr, IB_FLOW_DOMAIN_USER, &attrs->driver_udata); 3313 3314 if (IS_ERR(flow_id)) { 3315 err = PTR_ERR(flow_id); 3316 goto err_free; 3317 } 3318 3319 ib_set_flow(uobj, flow_id, qp, qp->device, uflow_res); 3320 3321 memset(&resp, 0, sizeof(resp)); 3322 resp.flow_handle = uobj->id; 3323 3324 err = uverbs_response(attrs, &resp, sizeof(resp)); 3325 if (err) 3326 goto err_copy; 3327 3328 uobj_put_obj_read(qp); 3329 kfree(flow_attr); 3330 if (cmd.flow_attr.num_of_specs) 3331 kfree(kern_flow_attr); 3332 return uobj_alloc_commit(uobj, attrs); 3333 err_copy: 3334 if (!qp->device->ops.destroy_flow(flow_id)) 3335 atomic_dec(&qp->usecnt); 3336 err_free: 3337 ib_uverbs_flow_resources_free(uflow_res); 3338 err_free_flow_attr: 3339 kfree(flow_attr); 3340 err_put: 3341 uobj_put_obj_read(qp); 3342 err_uobj: 3343 uobj_alloc_abort(uobj, attrs); 3344 err_free_attr: 3345 if (cmd.flow_attr.num_of_specs) 3346 kfree(kern_flow_attr); 3347 return err; 3348 } 3349 3350 static int ib_uverbs_ex_destroy_flow(struct uverbs_attr_bundle *attrs) 3351 { 3352 struct ib_uverbs_destroy_flow cmd; 3353 int ret; 3354 3355 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3356 if (ret) 3357 return ret; 3358 3359 if (cmd.comp_mask) 3360 return -EINVAL; 3361 3362 return uobj_perform_destroy(UVERBS_OBJECT_FLOW, cmd.flow_handle, attrs); 3363 } 3364 3365 static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs, 3366 struct ib_uverbs_create_xsrq *cmd, 3367 struct ib_udata *udata) 3368 { 3369 struct ib_uverbs_create_srq_resp resp; 3370 struct ib_usrq_object *obj; 3371 struct ib_pd *pd; 3372 struct ib_srq *srq; 3373 struct ib_uobject *uninitialized_var(xrcd_uobj); 3374 struct ib_srq_init_attr attr; 3375 int ret; 3376 struct ib_device *ib_dev; 3377 3378 obj = (struct ib_usrq_object *)uobj_alloc(UVERBS_OBJECT_SRQ, attrs, 3379 &ib_dev); 3380 if (IS_ERR(obj)) 3381 return PTR_ERR(obj); 3382 3383 if (cmd->srq_type == IB_SRQT_TM) 3384 attr.ext.tag_matching.max_num_tags = cmd->max_num_tags; 3385 3386 if (cmd->srq_type == IB_SRQT_XRC) { 3387 xrcd_uobj = uobj_get_read(UVERBS_OBJECT_XRCD, cmd->xrcd_handle, 3388 attrs); 3389 if (IS_ERR(xrcd_uobj)) { 3390 ret = -EINVAL; 3391 goto err; 3392 } 3393 3394 attr.ext.xrc.xrcd = (struct ib_xrcd *)xrcd_uobj->object; 3395 if (!attr.ext.xrc.xrcd) { 3396 ret = -EINVAL; 3397 goto err_put_xrcd; 3398 } 3399 3400 obj->uxrcd = container_of(xrcd_uobj, struct ib_uxrcd_object, uobject); 3401 atomic_inc(&obj->uxrcd->refcnt); 3402 } 3403 3404 if (ib_srq_has_cq(cmd->srq_type)) { 3405 attr.ext.cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, 3406 cmd->cq_handle, attrs); 3407 if (!attr.ext.cq) { 3408 ret = -EINVAL; 3409 goto err_put_xrcd; 3410 } 3411 } 3412 3413 pd = uobj_get_obj_read(pd, UVERBS_OBJECT_PD, cmd->pd_handle, attrs); 3414 if (!pd) { 3415 ret = -EINVAL; 3416 goto err_put_cq; 3417 } 3418 3419 attr.event_handler = ib_uverbs_srq_event_handler; 3420 attr.srq_context = attrs->ufile; 3421 attr.srq_type = cmd->srq_type; 3422 attr.attr.max_wr = cmd->max_wr; 3423 attr.attr.max_sge = cmd->max_sge; 3424 attr.attr.srq_limit = cmd->srq_limit; 3425 3426 obj->uevent.events_reported = 0; 3427 INIT_LIST_HEAD(&obj->uevent.event_list); 3428 3429 srq = rdma_zalloc_drv_obj(ib_dev, ib_srq); 3430 if (!srq) { 3431 ret = -ENOMEM; 3432 goto err_put; 3433 } 3434 3435 srq->device = pd->device; 3436 srq->pd = pd; 3437 srq->srq_type = cmd->srq_type; 3438 srq->uobject = &obj->uevent.uobject; 3439 srq->event_handler = attr.event_handler; 3440 srq->srq_context = attr.srq_context; 3441 3442 ret = pd->device->ops.create_srq(srq, &attr, udata); 3443 if (ret) 3444 goto err_free; 3445 3446 if (ib_srq_has_cq(cmd->srq_type)) { 3447 srq->ext.cq = attr.ext.cq; 3448 atomic_inc(&attr.ext.cq->usecnt); 3449 } 3450 3451 if (cmd->srq_type == IB_SRQT_XRC) { 3452 srq->ext.xrc.xrcd = attr.ext.xrc.xrcd; 3453 atomic_inc(&attr.ext.xrc.xrcd->usecnt); 3454 } 3455 3456 atomic_inc(&pd->usecnt); 3457 atomic_set(&srq->usecnt, 0); 3458 3459 obj->uevent.uobject.object = srq; 3460 obj->uevent.uobject.user_handle = cmd->user_handle; 3461 3462 memset(&resp, 0, sizeof resp); 3463 resp.srq_handle = obj->uevent.uobject.id; 3464 resp.max_wr = attr.attr.max_wr; 3465 resp.max_sge = attr.attr.max_sge; 3466 if (cmd->srq_type == IB_SRQT_XRC) 3467 resp.srqn = srq->ext.xrc.srq_num; 3468 3469 ret = uverbs_response(attrs, &resp, sizeof(resp)); 3470 if (ret) 3471 goto err_copy; 3472 3473 if (cmd->srq_type == IB_SRQT_XRC) 3474 uobj_put_read(xrcd_uobj); 3475 3476 if (ib_srq_has_cq(cmd->srq_type)) 3477 uobj_put_obj_read(attr.ext.cq); 3478 3479 uobj_put_obj_read(pd); 3480 return uobj_alloc_commit(&obj->uevent.uobject, attrs); 3481 3482 err_copy: 3483 ib_destroy_srq_user(srq, uverbs_get_cleared_udata(attrs)); 3484 /* It was released in ib_destroy_srq_user */ 3485 srq = NULL; 3486 err_free: 3487 kfree(srq); 3488 err_put: 3489 uobj_put_obj_read(pd); 3490 3491 err_put_cq: 3492 if (ib_srq_has_cq(cmd->srq_type)) 3493 uobj_put_obj_read(attr.ext.cq); 3494 3495 err_put_xrcd: 3496 if (cmd->srq_type == IB_SRQT_XRC) { 3497 atomic_dec(&obj->uxrcd->refcnt); 3498 uobj_put_read(xrcd_uobj); 3499 } 3500 3501 err: 3502 uobj_alloc_abort(&obj->uevent.uobject, attrs); 3503 return ret; 3504 } 3505 3506 static int ib_uverbs_create_srq(struct uverbs_attr_bundle *attrs) 3507 { 3508 struct ib_uverbs_create_srq cmd; 3509 struct ib_uverbs_create_xsrq xcmd; 3510 int ret; 3511 3512 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3513 if (ret) 3514 return ret; 3515 3516 memset(&xcmd, 0, sizeof(xcmd)); 3517 xcmd.response = cmd.response; 3518 xcmd.user_handle = cmd.user_handle; 3519 xcmd.srq_type = IB_SRQT_BASIC; 3520 xcmd.pd_handle = cmd.pd_handle; 3521 xcmd.max_wr = cmd.max_wr; 3522 xcmd.max_sge = cmd.max_sge; 3523 xcmd.srq_limit = cmd.srq_limit; 3524 3525 return __uverbs_create_xsrq(attrs, &xcmd, &attrs->driver_udata); 3526 } 3527 3528 static int ib_uverbs_create_xsrq(struct uverbs_attr_bundle *attrs) 3529 { 3530 struct ib_uverbs_create_xsrq cmd; 3531 int ret; 3532 3533 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3534 if (ret) 3535 return ret; 3536 3537 return __uverbs_create_xsrq(attrs, &cmd, &attrs->driver_udata); 3538 } 3539 3540 static int ib_uverbs_modify_srq(struct uverbs_attr_bundle *attrs) 3541 { 3542 struct ib_uverbs_modify_srq cmd; 3543 struct ib_srq *srq; 3544 struct ib_srq_attr attr; 3545 int ret; 3546 3547 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3548 if (ret) 3549 return ret; 3550 3551 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 3552 if (!srq) 3553 return -EINVAL; 3554 3555 attr.max_wr = cmd.max_wr; 3556 attr.srq_limit = cmd.srq_limit; 3557 3558 ret = srq->device->ops.modify_srq(srq, &attr, cmd.attr_mask, 3559 &attrs->driver_udata); 3560 3561 uobj_put_obj_read(srq); 3562 3563 return ret; 3564 } 3565 3566 static int ib_uverbs_query_srq(struct uverbs_attr_bundle *attrs) 3567 { 3568 struct ib_uverbs_query_srq cmd; 3569 struct ib_uverbs_query_srq_resp resp; 3570 struct ib_srq_attr attr; 3571 struct ib_srq *srq; 3572 int ret; 3573 3574 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3575 if (ret) 3576 return ret; 3577 3578 srq = uobj_get_obj_read(srq, UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 3579 if (!srq) 3580 return -EINVAL; 3581 3582 ret = ib_query_srq(srq, &attr); 3583 3584 uobj_put_obj_read(srq); 3585 3586 if (ret) 3587 return ret; 3588 3589 memset(&resp, 0, sizeof resp); 3590 3591 resp.max_wr = attr.max_wr; 3592 resp.max_sge = attr.max_sge; 3593 resp.srq_limit = attr.srq_limit; 3594 3595 return uverbs_response(attrs, &resp, sizeof(resp)); 3596 } 3597 3598 static int ib_uverbs_destroy_srq(struct uverbs_attr_bundle *attrs) 3599 { 3600 struct ib_uverbs_destroy_srq cmd; 3601 struct ib_uverbs_destroy_srq_resp resp; 3602 struct ib_uobject *uobj; 3603 struct ib_uevent_object *obj; 3604 int ret; 3605 3606 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3607 if (ret) 3608 return ret; 3609 3610 uobj = uobj_get_destroy(UVERBS_OBJECT_SRQ, cmd.srq_handle, attrs); 3611 if (IS_ERR(uobj)) 3612 return PTR_ERR(uobj); 3613 3614 obj = container_of(uobj, struct ib_uevent_object, uobject); 3615 memset(&resp, 0, sizeof(resp)); 3616 resp.events_reported = obj->events_reported; 3617 3618 uobj_put_destroy(uobj); 3619 3620 return uverbs_response(attrs, &resp, sizeof(resp)); 3621 } 3622 3623 static int ib_uverbs_ex_query_device(struct uverbs_attr_bundle *attrs) 3624 { 3625 struct ib_uverbs_ex_query_device_resp resp = {}; 3626 struct ib_uverbs_ex_query_device cmd; 3627 struct ib_device_attr attr = {0}; 3628 struct ib_ucontext *ucontext; 3629 struct ib_device *ib_dev; 3630 int err; 3631 3632 ucontext = ib_uverbs_get_ucontext(attrs); 3633 if (IS_ERR(ucontext)) 3634 return PTR_ERR(ucontext); 3635 ib_dev = ucontext->device; 3636 3637 err = uverbs_request(attrs, &cmd, sizeof(cmd)); 3638 if (err) 3639 return err; 3640 3641 if (cmd.comp_mask) 3642 return -EINVAL; 3643 3644 if (cmd.reserved) 3645 return -EINVAL; 3646 3647 err = ib_dev->ops.query_device(ib_dev, &attr, &attrs->driver_udata); 3648 if (err) 3649 return err; 3650 3651 copy_query_dev_fields(ucontext, &resp.base, &attr); 3652 3653 resp.odp_caps.general_caps = attr.odp_caps.general_caps; 3654 resp.odp_caps.per_transport_caps.rc_odp_caps = 3655 attr.odp_caps.per_transport_caps.rc_odp_caps; 3656 resp.odp_caps.per_transport_caps.uc_odp_caps = 3657 attr.odp_caps.per_transport_caps.uc_odp_caps; 3658 resp.odp_caps.per_transport_caps.ud_odp_caps = 3659 attr.odp_caps.per_transport_caps.ud_odp_caps; 3660 resp.xrc_odp_caps = attr.odp_caps.per_transport_caps.xrc_odp_caps; 3661 3662 resp.timestamp_mask = attr.timestamp_mask; 3663 resp.hca_core_clock = attr.hca_core_clock; 3664 resp.device_cap_flags_ex = attr.device_cap_flags; 3665 resp.rss_caps.supported_qpts = attr.rss_caps.supported_qpts; 3666 resp.rss_caps.max_rwq_indirection_tables = 3667 attr.rss_caps.max_rwq_indirection_tables; 3668 resp.rss_caps.max_rwq_indirection_table_size = 3669 attr.rss_caps.max_rwq_indirection_table_size; 3670 resp.max_wq_type_rq = attr.max_wq_type_rq; 3671 resp.raw_packet_caps = attr.raw_packet_caps; 3672 resp.tm_caps.max_rndv_hdr_size = attr.tm_caps.max_rndv_hdr_size; 3673 resp.tm_caps.max_num_tags = attr.tm_caps.max_num_tags; 3674 resp.tm_caps.max_ops = attr.tm_caps.max_ops; 3675 resp.tm_caps.max_sge = attr.tm_caps.max_sge; 3676 resp.tm_caps.flags = attr.tm_caps.flags; 3677 resp.cq_moderation_caps.max_cq_moderation_count = 3678 attr.cq_caps.max_cq_moderation_count; 3679 resp.cq_moderation_caps.max_cq_moderation_period = 3680 attr.cq_caps.max_cq_moderation_period; 3681 resp.max_dm_size = attr.max_dm_size; 3682 resp.response_length = uverbs_response_length(attrs, sizeof(resp)); 3683 3684 return uverbs_response(attrs, &resp, sizeof(resp)); 3685 } 3686 3687 static int ib_uverbs_ex_modify_cq(struct uverbs_attr_bundle *attrs) 3688 { 3689 struct ib_uverbs_ex_modify_cq cmd; 3690 struct ib_cq *cq; 3691 int ret; 3692 3693 ret = uverbs_request(attrs, &cmd, sizeof(cmd)); 3694 if (ret) 3695 return ret; 3696 3697 if (!cmd.attr_mask || cmd.reserved) 3698 return -EINVAL; 3699 3700 if (cmd.attr_mask > IB_CQ_MODERATE) 3701 return -EOPNOTSUPP; 3702 3703 cq = uobj_get_obj_read(cq, UVERBS_OBJECT_CQ, cmd.cq_handle, attrs); 3704 if (!cq) 3705 return -EINVAL; 3706 3707 ret = rdma_set_cq_moderation(cq, cmd.attr.cq_count, cmd.attr.cq_period); 3708 3709 uobj_put_obj_read(cq); 3710 3711 return ret; 3712 } 3713 3714 /* 3715 * Describe the input structs for write(). Some write methods have an input 3716 * only struct, most have an input and output. If the struct has an output then 3717 * the 'response' u64 must be the first field in the request structure. 3718 * 3719 * If udata is present then both the request and response structs have a 3720 * trailing driver_data flex array. In this case the size of the base struct 3721 * cannot be changed. 3722 */ 3723 #define UAPI_DEF_WRITE_IO(req, resp) \ 3724 .write.has_resp = 1 + \ 3725 BUILD_BUG_ON_ZERO(offsetof(req, response) != 0) + \ 3726 BUILD_BUG_ON_ZERO(sizeof(((req *)0)->response) != \ 3727 sizeof(u64)), \ 3728 .write.req_size = sizeof(req), .write.resp_size = sizeof(resp) 3729 3730 #define UAPI_DEF_WRITE_I(req) .write.req_size = sizeof(req) 3731 3732 #define UAPI_DEF_WRITE_UDATA_IO(req, resp) \ 3733 UAPI_DEF_WRITE_IO(req, resp), \ 3734 .write.has_udata = \ 3735 1 + \ 3736 BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \ 3737 sizeof(req)) + \ 3738 BUILD_BUG_ON_ZERO(offsetof(resp, driver_data) != \ 3739 sizeof(resp)) 3740 3741 #define UAPI_DEF_WRITE_UDATA_I(req) \ 3742 UAPI_DEF_WRITE_I(req), \ 3743 .write.has_udata = \ 3744 1 + BUILD_BUG_ON_ZERO(offsetof(req, driver_data) != \ 3745 sizeof(req)) 3746 3747 /* 3748 * The _EX versions are for use with WRITE_EX and allow the last struct member 3749 * to be specified. Buffers that do not include that member will be rejected. 3750 */ 3751 #define UAPI_DEF_WRITE_IO_EX(req, req_last_member, resp, resp_last_member) \ 3752 .write.has_resp = 1, \ 3753 .write.req_size = offsetofend(req, req_last_member), \ 3754 .write.resp_size = offsetofend(resp, resp_last_member) 3755 3756 #define UAPI_DEF_WRITE_I_EX(req, req_last_member) \ 3757 .write.req_size = offsetofend(req, req_last_member) 3758 3759 const struct uapi_definition uverbs_def_write_intf[] = { 3760 DECLARE_UVERBS_OBJECT( 3761 UVERBS_OBJECT_AH, 3762 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_AH, 3763 ib_uverbs_create_ah, 3764 UAPI_DEF_WRITE_UDATA_IO( 3765 struct ib_uverbs_create_ah, 3766 struct ib_uverbs_create_ah_resp), 3767 UAPI_DEF_METHOD_NEEDS_FN(create_ah)), 3768 DECLARE_UVERBS_WRITE( 3769 IB_USER_VERBS_CMD_DESTROY_AH, 3770 ib_uverbs_destroy_ah, 3771 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_ah), 3772 UAPI_DEF_METHOD_NEEDS_FN(destroy_ah))), 3773 3774 DECLARE_UVERBS_OBJECT( 3775 UVERBS_OBJECT_COMP_CHANNEL, 3776 DECLARE_UVERBS_WRITE( 3777 IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL, 3778 ib_uverbs_create_comp_channel, 3779 UAPI_DEF_WRITE_IO( 3780 struct ib_uverbs_create_comp_channel, 3781 struct ib_uverbs_create_comp_channel_resp))), 3782 3783 DECLARE_UVERBS_OBJECT( 3784 UVERBS_OBJECT_CQ, 3785 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_CQ, 3786 ib_uverbs_create_cq, 3787 UAPI_DEF_WRITE_UDATA_IO( 3788 struct ib_uverbs_create_cq, 3789 struct ib_uverbs_create_cq_resp), 3790 UAPI_DEF_METHOD_NEEDS_FN(create_cq)), 3791 DECLARE_UVERBS_WRITE( 3792 IB_USER_VERBS_CMD_DESTROY_CQ, 3793 ib_uverbs_destroy_cq, 3794 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_cq, 3795 struct ib_uverbs_destroy_cq_resp), 3796 UAPI_DEF_METHOD_NEEDS_FN(destroy_cq)), 3797 DECLARE_UVERBS_WRITE( 3798 IB_USER_VERBS_CMD_POLL_CQ, 3799 ib_uverbs_poll_cq, 3800 UAPI_DEF_WRITE_IO(struct ib_uverbs_poll_cq, 3801 struct ib_uverbs_poll_cq_resp), 3802 UAPI_DEF_METHOD_NEEDS_FN(poll_cq)), 3803 DECLARE_UVERBS_WRITE( 3804 IB_USER_VERBS_CMD_REQ_NOTIFY_CQ, 3805 ib_uverbs_req_notify_cq, 3806 UAPI_DEF_WRITE_I(struct ib_uverbs_req_notify_cq), 3807 UAPI_DEF_METHOD_NEEDS_FN(req_notify_cq)), 3808 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_RESIZE_CQ, 3809 ib_uverbs_resize_cq, 3810 UAPI_DEF_WRITE_UDATA_IO( 3811 struct ib_uverbs_resize_cq, 3812 struct ib_uverbs_resize_cq_resp), 3813 UAPI_DEF_METHOD_NEEDS_FN(resize_cq)), 3814 DECLARE_UVERBS_WRITE_EX( 3815 IB_USER_VERBS_EX_CMD_CREATE_CQ, 3816 ib_uverbs_ex_create_cq, 3817 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_cq, 3818 reserved, 3819 struct ib_uverbs_ex_create_cq_resp, 3820 response_length), 3821 UAPI_DEF_METHOD_NEEDS_FN(create_cq)), 3822 DECLARE_UVERBS_WRITE_EX( 3823 IB_USER_VERBS_EX_CMD_MODIFY_CQ, 3824 ib_uverbs_ex_modify_cq, 3825 UAPI_DEF_WRITE_I(struct ib_uverbs_ex_modify_cq), 3826 UAPI_DEF_METHOD_NEEDS_FN(create_cq))), 3827 3828 DECLARE_UVERBS_OBJECT( 3829 UVERBS_OBJECT_DEVICE, 3830 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_GET_CONTEXT, 3831 ib_uverbs_get_context, 3832 UAPI_DEF_WRITE_UDATA_IO( 3833 struct ib_uverbs_get_context, 3834 struct ib_uverbs_get_context_resp)), 3835 DECLARE_UVERBS_WRITE( 3836 IB_USER_VERBS_CMD_QUERY_DEVICE, 3837 ib_uverbs_query_device, 3838 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_device, 3839 struct ib_uverbs_query_device_resp)), 3840 DECLARE_UVERBS_WRITE( 3841 IB_USER_VERBS_CMD_QUERY_PORT, 3842 ib_uverbs_query_port, 3843 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_port, 3844 struct ib_uverbs_query_port_resp), 3845 UAPI_DEF_METHOD_NEEDS_FN(query_port)), 3846 DECLARE_UVERBS_WRITE_EX( 3847 IB_USER_VERBS_EX_CMD_QUERY_DEVICE, 3848 ib_uverbs_ex_query_device, 3849 UAPI_DEF_WRITE_IO_EX( 3850 struct ib_uverbs_ex_query_device, 3851 reserved, 3852 struct ib_uverbs_ex_query_device_resp, 3853 response_length), 3854 UAPI_DEF_METHOD_NEEDS_FN(query_device)), 3855 UAPI_DEF_OBJ_NEEDS_FN(alloc_ucontext), 3856 UAPI_DEF_OBJ_NEEDS_FN(dealloc_ucontext)), 3857 3858 DECLARE_UVERBS_OBJECT( 3859 UVERBS_OBJECT_FLOW, 3860 DECLARE_UVERBS_WRITE_EX( 3861 IB_USER_VERBS_EX_CMD_CREATE_FLOW, 3862 ib_uverbs_ex_create_flow, 3863 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_create_flow, 3864 flow_attr, 3865 struct ib_uverbs_create_flow_resp, 3866 flow_handle), 3867 UAPI_DEF_METHOD_NEEDS_FN(create_flow)), 3868 DECLARE_UVERBS_WRITE_EX( 3869 IB_USER_VERBS_EX_CMD_DESTROY_FLOW, 3870 ib_uverbs_ex_destroy_flow, 3871 UAPI_DEF_WRITE_I(struct ib_uverbs_destroy_flow), 3872 UAPI_DEF_METHOD_NEEDS_FN(destroy_flow))), 3873 3874 DECLARE_UVERBS_OBJECT( 3875 UVERBS_OBJECT_MR, 3876 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_DEREG_MR, 3877 ib_uverbs_dereg_mr, 3878 UAPI_DEF_WRITE_I(struct ib_uverbs_dereg_mr), 3879 UAPI_DEF_METHOD_NEEDS_FN(dereg_mr)), 3880 DECLARE_UVERBS_WRITE( 3881 IB_USER_VERBS_CMD_REG_MR, 3882 ib_uverbs_reg_mr, 3883 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_reg_mr, 3884 struct ib_uverbs_reg_mr_resp), 3885 UAPI_DEF_METHOD_NEEDS_FN(reg_user_mr)), 3886 DECLARE_UVERBS_WRITE( 3887 IB_USER_VERBS_CMD_REREG_MR, 3888 ib_uverbs_rereg_mr, 3889 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_rereg_mr, 3890 struct ib_uverbs_rereg_mr_resp), 3891 UAPI_DEF_METHOD_NEEDS_FN(rereg_user_mr))), 3892 3893 DECLARE_UVERBS_OBJECT( 3894 UVERBS_OBJECT_MW, 3895 DECLARE_UVERBS_WRITE( 3896 IB_USER_VERBS_CMD_ALLOC_MW, 3897 ib_uverbs_alloc_mw, 3898 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_mw, 3899 struct ib_uverbs_alloc_mw_resp), 3900 UAPI_DEF_METHOD_NEEDS_FN(alloc_mw)), 3901 DECLARE_UVERBS_WRITE( 3902 IB_USER_VERBS_CMD_DEALLOC_MW, 3903 ib_uverbs_dealloc_mw, 3904 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_mw), 3905 UAPI_DEF_METHOD_NEEDS_FN(dealloc_mw))), 3906 3907 DECLARE_UVERBS_OBJECT( 3908 UVERBS_OBJECT_PD, 3909 DECLARE_UVERBS_WRITE( 3910 IB_USER_VERBS_CMD_ALLOC_PD, 3911 ib_uverbs_alloc_pd, 3912 UAPI_DEF_WRITE_UDATA_IO(struct ib_uverbs_alloc_pd, 3913 struct ib_uverbs_alloc_pd_resp), 3914 UAPI_DEF_METHOD_NEEDS_FN(alloc_pd)), 3915 DECLARE_UVERBS_WRITE( 3916 IB_USER_VERBS_CMD_DEALLOC_PD, 3917 ib_uverbs_dealloc_pd, 3918 UAPI_DEF_WRITE_I(struct ib_uverbs_dealloc_pd), 3919 UAPI_DEF_METHOD_NEEDS_FN(dealloc_pd))), 3920 3921 DECLARE_UVERBS_OBJECT( 3922 UVERBS_OBJECT_QP, 3923 DECLARE_UVERBS_WRITE( 3924 IB_USER_VERBS_CMD_ATTACH_MCAST, 3925 ib_uverbs_attach_mcast, 3926 UAPI_DEF_WRITE_I(struct ib_uverbs_attach_mcast), 3927 UAPI_DEF_METHOD_NEEDS_FN(attach_mcast), 3928 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)), 3929 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_QP, 3930 ib_uverbs_create_qp, 3931 UAPI_DEF_WRITE_UDATA_IO( 3932 struct ib_uverbs_create_qp, 3933 struct ib_uverbs_create_qp_resp), 3934 UAPI_DEF_METHOD_NEEDS_FN(create_qp)), 3935 DECLARE_UVERBS_WRITE( 3936 IB_USER_VERBS_CMD_DESTROY_QP, 3937 ib_uverbs_destroy_qp, 3938 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_qp, 3939 struct ib_uverbs_destroy_qp_resp), 3940 UAPI_DEF_METHOD_NEEDS_FN(destroy_qp)), 3941 DECLARE_UVERBS_WRITE( 3942 IB_USER_VERBS_CMD_DETACH_MCAST, 3943 ib_uverbs_detach_mcast, 3944 UAPI_DEF_WRITE_I(struct ib_uverbs_detach_mcast), 3945 UAPI_DEF_METHOD_NEEDS_FN(detach_mcast)), 3946 DECLARE_UVERBS_WRITE( 3947 IB_USER_VERBS_CMD_MODIFY_QP, 3948 ib_uverbs_modify_qp, 3949 UAPI_DEF_WRITE_I(struct ib_uverbs_modify_qp), 3950 UAPI_DEF_METHOD_NEEDS_FN(modify_qp)), 3951 DECLARE_UVERBS_WRITE( 3952 IB_USER_VERBS_CMD_POST_RECV, 3953 ib_uverbs_post_recv, 3954 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_recv, 3955 struct ib_uverbs_post_recv_resp), 3956 UAPI_DEF_METHOD_NEEDS_FN(post_recv)), 3957 DECLARE_UVERBS_WRITE( 3958 IB_USER_VERBS_CMD_POST_SEND, 3959 ib_uverbs_post_send, 3960 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_send, 3961 struct ib_uverbs_post_send_resp), 3962 UAPI_DEF_METHOD_NEEDS_FN(post_send)), 3963 DECLARE_UVERBS_WRITE( 3964 IB_USER_VERBS_CMD_QUERY_QP, 3965 ib_uverbs_query_qp, 3966 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_qp, 3967 struct ib_uverbs_query_qp_resp), 3968 UAPI_DEF_METHOD_NEEDS_FN(query_qp)), 3969 DECLARE_UVERBS_WRITE_EX( 3970 IB_USER_VERBS_EX_CMD_CREATE_QP, 3971 ib_uverbs_ex_create_qp, 3972 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_qp, 3973 comp_mask, 3974 struct ib_uverbs_ex_create_qp_resp, 3975 response_length), 3976 UAPI_DEF_METHOD_NEEDS_FN(create_qp)), 3977 DECLARE_UVERBS_WRITE_EX( 3978 IB_USER_VERBS_EX_CMD_MODIFY_QP, 3979 ib_uverbs_ex_modify_qp, 3980 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_modify_qp, 3981 base, 3982 struct ib_uverbs_ex_modify_qp_resp, 3983 response_length), 3984 UAPI_DEF_METHOD_NEEDS_FN(modify_qp))), 3985 3986 DECLARE_UVERBS_OBJECT( 3987 UVERBS_OBJECT_RWQ_IND_TBL, 3988 DECLARE_UVERBS_WRITE_EX( 3989 IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL, 3990 ib_uverbs_ex_create_rwq_ind_table, 3991 UAPI_DEF_WRITE_IO_EX( 3992 struct ib_uverbs_ex_create_rwq_ind_table, 3993 log_ind_tbl_size, 3994 struct ib_uverbs_ex_create_rwq_ind_table_resp, 3995 ind_tbl_num), 3996 UAPI_DEF_METHOD_NEEDS_FN(create_rwq_ind_table)), 3997 DECLARE_UVERBS_WRITE_EX( 3998 IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL, 3999 ib_uverbs_ex_destroy_rwq_ind_table, 4000 UAPI_DEF_WRITE_I( 4001 struct ib_uverbs_ex_destroy_rwq_ind_table), 4002 UAPI_DEF_METHOD_NEEDS_FN(destroy_rwq_ind_table))), 4003 4004 DECLARE_UVERBS_OBJECT( 4005 UVERBS_OBJECT_WQ, 4006 DECLARE_UVERBS_WRITE_EX( 4007 IB_USER_VERBS_EX_CMD_CREATE_WQ, 4008 ib_uverbs_ex_create_wq, 4009 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_create_wq, 4010 max_sge, 4011 struct ib_uverbs_ex_create_wq_resp, 4012 wqn), 4013 UAPI_DEF_METHOD_NEEDS_FN(create_wq)), 4014 DECLARE_UVERBS_WRITE_EX( 4015 IB_USER_VERBS_EX_CMD_DESTROY_WQ, 4016 ib_uverbs_ex_destroy_wq, 4017 UAPI_DEF_WRITE_IO_EX(struct ib_uverbs_ex_destroy_wq, 4018 wq_handle, 4019 struct ib_uverbs_ex_destroy_wq_resp, 4020 reserved), 4021 UAPI_DEF_METHOD_NEEDS_FN(destroy_wq)), 4022 DECLARE_UVERBS_WRITE_EX( 4023 IB_USER_VERBS_EX_CMD_MODIFY_WQ, 4024 ib_uverbs_ex_modify_wq, 4025 UAPI_DEF_WRITE_I_EX(struct ib_uverbs_ex_modify_wq, 4026 curr_wq_state), 4027 UAPI_DEF_METHOD_NEEDS_FN(modify_wq))), 4028 4029 DECLARE_UVERBS_OBJECT( 4030 UVERBS_OBJECT_SRQ, 4031 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_SRQ, 4032 ib_uverbs_create_srq, 4033 UAPI_DEF_WRITE_UDATA_IO( 4034 struct ib_uverbs_create_srq, 4035 struct ib_uverbs_create_srq_resp), 4036 UAPI_DEF_METHOD_NEEDS_FN(create_srq)), 4037 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_CREATE_XSRQ, 4038 ib_uverbs_create_xsrq, 4039 UAPI_DEF_WRITE_UDATA_IO( 4040 struct ib_uverbs_create_xsrq, 4041 struct ib_uverbs_create_srq_resp), 4042 UAPI_DEF_METHOD_NEEDS_FN(create_srq)), 4043 DECLARE_UVERBS_WRITE( 4044 IB_USER_VERBS_CMD_DESTROY_SRQ, 4045 ib_uverbs_destroy_srq, 4046 UAPI_DEF_WRITE_IO(struct ib_uverbs_destroy_srq, 4047 struct ib_uverbs_destroy_srq_resp), 4048 UAPI_DEF_METHOD_NEEDS_FN(destroy_srq)), 4049 DECLARE_UVERBS_WRITE( 4050 IB_USER_VERBS_CMD_MODIFY_SRQ, 4051 ib_uverbs_modify_srq, 4052 UAPI_DEF_WRITE_UDATA_I(struct ib_uverbs_modify_srq), 4053 UAPI_DEF_METHOD_NEEDS_FN(modify_srq)), 4054 DECLARE_UVERBS_WRITE( 4055 IB_USER_VERBS_CMD_POST_SRQ_RECV, 4056 ib_uverbs_post_srq_recv, 4057 UAPI_DEF_WRITE_IO(struct ib_uverbs_post_srq_recv, 4058 struct ib_uverbs_post_srq_recv_resp), 4059 UAPI_DEF_METHOD_NEEDS_FN(post_srq_recv)), 4060 DECLARE_UVERBS_WRITE( 4061 IB_USER_VERBS_CMD_QUERY_SRQ, 4062 ib_uverbs_query_srq, 4063 UAPI_DEF_WRITE_IO(struct ib_uverbs_query_srq, 4064 struct ib_uverbs_query_srq_resp), 4065 UAPI_DEF_METHOD_NEEDS_FN(query_srq))), 4066 4067 DECLARE_UVERBS_OBJECT( 4068 UVERBS_OBJECT_XRCD, 4069 DECLARE_UVERBS_WRITE( 4070 IB_USER_VERBS_CMD_CLOSE_XRCD, 4071 ib_uverbs_close_xrcd, 4072 UAPI_DEF_WRITE_I(struct ib_uverbs_close_xrcd), 4073 UAPI_DEF_METHOD_NEEDS_FN(dealloc_xrcd)), 4074 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_QP, 4075 ib_uverbs_open_qp, 4076 UAPI_DEF_WRITE_UDATA_IO( 4077 struct ib_uverbs_open_qp, 4078 struct ib_uverbs_create_qp_resp)), 4079 DECLARE_UVERBS_WRITE(IB_USER_VERBS_CMD_OPEN_XRCD, 4080 ib_uverbs_open_xrcd, 4081 UAPI_DEF_WRITE_UDATA_IO( 4082 struct ib_uverbs_open_xrcd, 4083 struct ib_uverbs_open_xrcd_resp), 4084 UAPI_DEF_METHOD_NEEDS_FN(alloc_xrcd))), 4085 4086 {}, 4087 }; 4088