1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 20 if (udata->inlen || udata->outlen) 21 return -EINVAL; 22 23 memset(props, 0, sizeof(*props)); 24 addrconf_addr_eui48((u8 *)&props->sys_image_guid, 25 iwdev->netdev->dev_addr); 26 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 27 irdma_fw_minor_ver(&rf->sc_dev); 28 props->device_cap_flags = IB_DEVICE_MEM_WINDOW | 29 IB_DEVICE_MEM_MGT_EXTENSIONS; 30 props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 31 props->vendor_id = pcidev->vendor; 32 props->vendor_part_id = pcidev->device; 33 34 props->hw_ver = rf->pcidev->revision; 35 props->page_size_cap = hw_attrs->page_size_cap; 36 props->max_mr_size = hw_attrs->max_mr_size; 37 props->max_qp = rf->max_qp - rf->used_qps; 38 props->max_qp_wr = hw_attrs->max_qp_wr; 39 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 40 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 41 props->max_cq = rf->max_cq - rf->used_cqs; 42 props->max_cqe = rf->max_cqe - 1; 43 props->max_mr = rf->max_mr - rf->used_mrs; 44 props->max_mw = props->max_mr; 45 props->max_pd = rf->max_pd - rf->used_pds; 46 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 47 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 48 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 49 if (rdma_protocol_roce(ibdev, 1)) { 50 props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; 51 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 52 } 53 54 props->max_ah = rf->max_ah; 55 props->max_mcast_grp = rf->max_mcg; 56 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 57 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 58 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 59 #define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff 60 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2) 61 props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK; 62 63 return 0; 64 } 65 66 /** 67 * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed 68 * @link_speed: netdev phy link speed 69 * @active_speed: IB port speed 70 * @active_width: IB port width 71 */ 72 static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed, 73 u8 *active_width) 74 { 75 if (link_speed <= SPEED_1000) { 76 *active_width = IB_WIDTH_1X; 77 *active_speed = IB_SPEED_SDR; 78 } else if (link_speed <= SPEED_10000) { 79 *active_width = IB_WIDTH_1X; 80 *active_speed = IB_SPEED_FDR10; 81 } else if (link_speed <= SPEED_20000) { 82 *active_width = IB_WIDTH_4X; 83 *active_speed = IB_SPEED_DDR; 84 } else if (link_speed <= SPEED_25000) { 85 *active_width = IB_WIDTH_1X; 86 *active_speed = IB_SPEED_EDR; 87 } else if (link_speed <= SPEED_40000) { 88 *active_width = IB_WIDTH_4X; 89 *active_speed = IB_SPEED_FDR10; 90 } else { 91 *active_width = IB_WIDTH_4X; 92 *active_speed = IB_SPEED_EDR; 93 } 94 } 95 96 /** 97 * irdma_query_port - get port attributes 98 * @ibdev: device pointer from stack 99 * @port: port number for query 100 * @props: returning device attributes 101 */ 102 static int irdma_query_port(struct ib_device *ibdev, u32 port, 103 struct ib_port_attr *props) 104 { 105 struct irdma_device *iwdev = to_iwdev(ibdev); 106 struct net_device *netdev = iwdev->netdev; 107 108 /* no need to zero out pros here. done by caller */ 109 110 props->max_mtu = IB_MTU_4096; 111 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 112 props->lid = 1; 113 props->lmc = 0; 114 props->sm_lid = 0; 115 props->sm_sl = 0; 116 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 117 props->state = IB_PORT_ACTIVE; 118 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 119 } else { 120 props->state = IB_PORT_DOWN; 121 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 122 } 123 irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, 124 &props->active_width); 125 126 if (rdma_protocol_roce(ibdev, 1)) { 127 props->gid_tbl_len = 32; 128 props->ip_gids = true; 129 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 130 } else { 131 props->gid_tbl_len = 1; 132 } 133 props->qkey_viol_cntr = 0; 134 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 135 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 136 137 return 0; 138 } 139 140 /** 141 * irdma_disassociate_ucontext - Disassociate user context 142 * @context: ib user context 143 */ 144 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 145 { 146 } 147 148 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 149 struct vm_area_struct *vma) 150 { 151 u64 pfn; 152 153 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 154 return -EINVAL; 155 156 vma->vm_private_data = ucontext; 157 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 158 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 159 160 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 161 pgprot_noncached(vma->vm_page_prot), NULL); 162 } 163 164 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 165 { 166 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 167 168 kfree(entry); 169 } 170 171 static struct rdma_user_mmap_entry* 172 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 173 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 174 { 175 struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 176 int ret; 177 178 if (!entry) 179 return NULL; 180 181 entry->bar_offset = bar_offset; 182 entry->mmap_flag = mmap_flag; 183 184 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 185 &entry->rdma_entry, PAGE_SIZE); 186 if (ret) { 187 kfree(entry); 188 return NULL; 189 } 190 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 191 192 return &entry->rdma_entry; 193 } 194 195 /** 196 * irdma_mmap - user memory map 197 * @context: context created during alloc 198 * @vma: kernel info for user memory map 199 */ 200 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 201 { 202 struct rdma_user_mmap_entry *rdma_entry; 203 struct irdma_user_mmap_entry *entry; 204 struct irdma_ucontext *ucontext; 205 u64 pfn; 206 int ret; 207 208 ucontext = to_ucontext(context); 209 210 /* Legacy support for libi40iw with hard-coded mmap key */ 211 if (ucontext->legacy_mode) 212 return irdma_mmap_legacy(ucontext, vma); 213 214 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 215 if (!rdma_entry) { 216 ibdev_dbg(&ucontext->iwdev->ibdev, 217 "VERBS: pgoff[0x%lx] does not have valid entry\n", 218 vma->vm_pgoff); 219 return -EINVAL; 220 } 221 222 entry = to_irdma_mmap_entry(rdma_entry); 223 ibdev_dbg(&ucontext->iwdev->ibdev, 224 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 225 entry->bar_offset, entry->mmap_flag); 226 227 pfn = (entry->bar_offset + 228 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 229 230 switch (entry->mmap_flag) { 231 case IRDMA_MMAP_IO_NC: 232 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 233 pgprot_noncached(vma->vm_page_prot), 234 rdma_entry); 235 break; 236 case IRDMA_MMAP_IO_WC: 237 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 238 pgprot_writecombine(vma->vm_page_prot), 239 rdma_entry); 240 break; 241 default: 242 ret = -EINVAL; 243 } 244 245 if (ret) 246 ibdev_dbg(&ucontext->iwdev->ibdev, 247 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 248 entry->bar_offset, entry->mmap_flag, ret); 249 rdma_user_mmap_entry_put(rdma_entry); 250 251 return ret; 252 } 253 254 /** 255 * irdma_alloc_push_page - allocate a push page for qp 256 * @iwqp: qp pointer 257 */ 258 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 259 { 260 struct irdma_cqp_request *cqp_request; 261 struct cqp_cmds_info *cqp_info; 262 struct irdma_device *iwdev = iwqp->iwdev; 263 struct irdma_sc_qp *qp = &iwqp->sc_qp; 264 int status; 265 266 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 267 if (!cqp_request) 268 return; 269 270 cqp_info = &cqp_request->info; 271 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 272 cqp_info->post_sq = 1; 273 cqp_info->in.u.manage_push_page.info.push_idx = 0; 274 cqp_info->in.u.manage_push_page.info.qs_handle = 275 qp->vsi->qos[qp->user_pri].qs_handle; 276 cqp_info->in.u.manage_push_page.info.free_page = 0; 277 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 278 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 279 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 280 281 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 282 if (!status && cqp_request->compl_info.op_ret_val < 283 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 284 qp->push_idx = cqp_request->compl_info.op_ret_val; 285 qp->push_offset = 0; 286 } 287 288 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 289 } 290 291 /** 292 * irdma_alloc_ucontext - Allocate the user context data structure 293 * @uctx: uverbs context pointer 294 * @udata: user data 295 * 296 * This keeps track of all objects associated with a particular 297 * user-mode client. 298 */ 299 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 300 struct ib_udata *udata) 301 { 302 #define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) 303 #define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) 304 struct ib_device *ibdev = uctx->device; 305 struct irdma_device *iwdev = to_iwdev(ibdev); 306 struct irdma_alloc_ucontext_req req = {}; 307 struct irdma_alloc_ucontext_resp uresp = {}; 308 struct irdma_ucontext *ucontext = to_ucontext(uctx); 309 struct irdma_uk_attrs *uk_attrs; 310 311 if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || 312 udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) 313 return -EINVAL; 314 315 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 316 return -EINVAL; 317 318 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 319 goto ver_error; 320 321 ucontext->iwdev = iwdev; 322 ucontext->abi_ver = req.userspace_ver; 323 324 uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 325 /* GEN_1 legacy support with libi40iw */ 326 if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { 327 if (uk_attrs->hw_rev != IRDMA_GEN_1) 328 return -EOPNOTSUPP; 329 330 ucontext->legacy_mode = true; 331 uresp.max_qps = iwdev->rf->max_qp; 332 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 333 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 334 uresp.kernel_ver = req.userspace_ver; 335 if (ib_copy_to_udata(udata, &uresp, 336 min(sizeof(uresp), udata->outlen))) 337 return -EFAULT; 338 } else { 339 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 340 341 ucontext->db_mmap_entry = 342 irdma_user_mmap_entry_insert(ucontext, bar_off, 343 IRDMA_MMAP_IO_NC, 344 &uresp.db_mmap_key); 345 if (!ucontext->db_mmap_entry) 346 return -ENOMEM; 347 348 uresp.kernel_ver = IRDMA_ABI_VER; 349 uresp.feature_flags = uk_attrs->feature_flags; 350 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 351 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 352 uresp.max_hw_inline = uk_attrs->max_hw_inline; 353 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 354 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 355 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 356 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 357 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 358 uresp.hw_rev = uk_attrs->hw_rev; 359 if (ib_copy_to_udata(udata, &uresp, 360 min(sizeof(uresp), udata->outlen))) { 361 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 362 return -EFAULT; 363 } 364 } 365 366 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 367 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 368 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 369 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 370 371 return 0; 372 373 ver_error: 374 ibdev_err(&iwdev->ibdev, 375 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 376 req.userspace_ver, IRDMA_ABI_VER); 377 return -EINVAL; 378 } 379 380 /** 381 * irdma_dealloc_ucontext - deallocate the user context data structure 382 * @context: user context created during alloc 383 */ 384 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 385 { 386 struct irdma_ucontext *ucontext = to_ucontext(context); 387 388 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 389 } 390 391 /** 392 * irdma_alloc_pd - allocate protection domain 393 * @pd: PD pointer 394 * @udata: user data 395 */ 396 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 397 { 398 #define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) 399 struct irdma_pd *iwpd = to_iwpd(pd); 400 struct irdma_device *iwdev = to_iwdev(pd->device); 401 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 402 struct irdma_pci_f *rf = iwdev->rf; 403 struct irdma_alloc_pd_resp uresp = {}; 404 struct irdma_sc_pd *sc_pd; 405 u32 pd_id = 0; 406 int err; 407 408 if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) 409 return -EINVAL; 410 411 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 412 &rf->next_pd); 413 if (err) 414 return err; 415 416 sc_pd = &iwpd->sc_pd; 417 if (udata) { 418 struct irdma_ucontext *ucontext = 419 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 420 ibucontext); 421 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 422 uresp.pd_id = pd_id; 423 if (ib_copy_to_udata(udata, &uresp, 424 min(sizeof(uresp), udata->outlen))) { 425 err = -EFAULT; 426 goto error; 427 } 428 } else { 429 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 430 } 431 432 return 0; 433 error: 434 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 435 436 return err; 437 } 438 439 /** 440 * irdma_dealloc_pd - deallocate pd 441 * @ibpd: ptr of pd to be deallocated 442 * @udata: user data 443 */ 444 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 445 { 446 struct irdma_pd *iwpd = to_iwpd(ibpd); 447 struct irdma_device *iwdev = to_iwdev(ibpd->device); 448 449 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 450 451 return 0; 452 } 453 454 /** 455 * irdma_get_pbl - Retrieve pbl from a list given a virtual 456 * address 457 * @va: user virtual address 458 * @pbl_list: pbl list to search in (QP's or CQ's) 459 */ 460 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 461 struct list_head *pbl_list) 462 { 463 struct irdma_pbl *iwpbl; 464 465 list_for_each_entry (iwpbl, pbl_list, list) { 466 if (iwpbl->user_base == va) { 467 list_del(&iwpbl->list); 468 iwpbl->on_list = false; 469 return iwpbl; 470 } 471 } 472 473 return NULL; 474 } 475 476 /** 477 * irdma_clean_cqes - clean cq entries for qp 478 * @iwqp: qp ptr (user or kernel) 479 * @iwcq: cq ptr 480 */ 481 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 482 { 483 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 484 unsigned long flags; 485 486 spin_lock_irqsave(&iwcq->lock, flags); 487 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 488 spin_unlock_irqrestore(&iwcq->lock, flags); 489 } 490 491 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 492 { 493 if (iwqp->push_db_mmap_entry) { 494 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 495 iwqp->push_db_mmap_entry = NULL; 496 } 497 if (iwqp->push_wqe_mmap_entry) { 498 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 499 iwqp->push_wqe_mmap_entry = NULL; 500 } 501 } 502 503 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 504 struct irdma_qp *iwqp, 505 u64 *push_wqe_mmap_key, 506 u64 *push_db_mmap_key) 507 { 508 struct irdma_device *iwdev = ucontext->iwdev; 509 u64 rsvd, bar_off; 510 511 rsvd = IRDMA_PF_BAR_RSVD; 512 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 513 /* skip over db page */ 514 bar_off += IRDMA_HW_PAGE_SIZE; 515 /* push wqe page */ 516 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 517 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 518 bar_off, IRDMA_MMAP_IO_WC, 519 push_wqe_mmap_key); 520 if (!iwqp->push_wqe_mmap_entry) 521 return -ENOMEM; 522 523 /* push doorbell page */ 524 bar_off += IRDMA_HW_PAGE_SIZE; 525 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 526 bar_off, IRDMA_MMAP_IO_NC, 527 push_db_mmap_key); 528 if (!iwqp->push_db_mmap_entry) { 529 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 530 return -ENOMEM; 531 } 532 533 return 0; 534 } 535 536 /** 537 * irdma_destroy_qp - destroy qp 538 * @ibqp: qp's ib pointer also to get to device's qp address 539 * @udata: user data 540 */ 541 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 542 { 543 struct irdma_qp *iwqp = to_iwqp(ibqp); 544 struct irdma_device *iwdev = iwqp->iwdev; 545 546 iwqp->sc_qp.qp_uk.destroy_pending = true; 547 548 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) 549 irdma_modify_qp_to_err(&iwqp->sc_qp); 550 551 if (!iwqp->user_mode) 552 cancel_delayed_work_sync(&iwqp->dwork_flush); 553 554 irdma_qp_rem_ref(&iwqp->ibqp); 555 wait_for_completion(&iwqp->free_qp); 556 irdma_free_lsmm_rsrc(iwqp); 557 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 558 559 if (!iwqp->user_mode) { 560 if (iwqp->iwscq) { 561 irdma_clean_cqes(iwqp, iwqp->iwscq); 562 if (iwqp->iwrcq != iwqp->iwscq) 563 irdma_clean_cqes(iwqp, iwqp->iwrcq); 564 } 565 } 566 irdma_remove_push_mmap_entries(iwqp); 567 irdma_free_qp_rsrc(iwqp); 568 569 return 0; 570 } 571 572 /** 573 * irdma_setup_virt_qp - setup for allocation of virtual qp 574 * @iwdev: irdma device 575 * @iwqp: qp ptr 576 * @init_info: initialize info to return 577 */ 578 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 579 struct irdma_qp *iwqp, 580 struct irdma_qp_init_info *init_info) 581 { 582 struct irdma_pbl *iwpbl = iwqp->iwpbl; 583 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 584 585 iwqp->page = qpmr->sq_page; 586 init_info->shadow_area_pa = qpmr->shadow; 587 if (iwpbl->pbl_allocated) { 588 init_info->virtual_map = true; 589 init_info->sq_pa = qpmr->sq_pbl.idx; 590 init_info->rq_pa = qpmr->rq_pbl.idx; 591 } else { 592 init_info->sq_pa = qpmr->sq_pbl.addr; 593 init_info->rq_pa = qpmr->rq_pbl.addr; 594 } 595 } 596 597 /** 598 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 599 * @iwdev: iwarp device 600 * @iwqp: qp ptr (user or kernel) 601 * @info: initialize info to return 602 * @init_attr: Initial QP create attributes 603 */ 604 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 605 struct irdma_qp *iwqp, 606 struct irdma_qp_init_info *info, 607 struct ib_qp_init_attr *init_attr) 608 { 609 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 610 u32 sqdepth, rqdepth; 611 u8 sqshift, rqshift; 612 u32 size; 613 int status; 614 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 615 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 616 617 irdma_get_wqe_shift(uk_attrs, 618 uk_attrs->hw_rev >= IRDMA_GEN_2 ? ukinfo->max_sq_frag_cnt + 1 : 619 ukinfo->max_sq_frag_cnt, 620 ukinfo->max_inline_data, &sqshift); 621 status = irdma_get_sqdepth(uk_attrs, ukinfo->sq_size, sqshift, 622 &sqdepth); 623 if (status) 624 return status; 625 626 if (uk_attrs->hw_rev == IRDMA_GEN_1) 627 rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; 628 else 629 irdma_get_wqe_shift(uk_attrs, ukinfo->max_rq_frag_cnt, 0, 630 &rqshift); 631 632 status = irdma_get_rqdepth(uk_attrs, ukinfo->rq_size, rqshift, 633 &rqdepth); 634 if (status) 635 return status; 636 637 iwqp->kqp.sq_wrid_mem = 638 kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); 639 if (!iwqp->kqp.sq_wrid_mem) 640 return -ENOMEM; 641 642 iwqp->kqp.rq_wrid_mem = 643 kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); 644 if (!iwqp->kqp.rq_wrid_mem) { 645 kfree(iwqp->kqp.sq_wrid_mem); 646 iwqp->kqp.sq_wrid_mem = NULL; 647 return -ENOMEM; 648 } 649 650 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 651 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 652 653 size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE; 654 size += (IRDMA_SHADOW_AREA_SIZE << 3); 655 656 mem->size = ALIGN(size, 256); 657 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 658 &mem->pa, GFP_KERNEL); 659 if (!mem->va) { 660 kfree(iwqp->kqp.sq_wrid_mem); 661 iwqp->kqp.sq_wrid_mem = NULL; 662 kfree(iwqp->kqp.rq_wrid_mem); 663 iwqp->kqp.rq_wrid_mem = NULL; 664 return -ENOMEM; 665 } 666 667 ukinfo->sq = mem->va; 668 info->sq_pa = mem->pa; 669 ukinfo->rq = &ukinfo->sq[sqdepth]; 670 info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE); 671 ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; 672 info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE); 673 ukinfo->sq_size = sqdepth >> sqshift; 674 ukinfo->rq_size = rqdepth >> rqshift; 675 ukinfo->qp_id = iwqp->ibqp.qp_num; 676 677 init_attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift; 678 init_attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift; 679 680 return 0; 681 } 682 683 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 684 { 685 struct irdma_pci_f *rf = iwqp->iwdev->rf; 686 struct irdma_cqp_request *cqp_request; 687 struct cqp_cmds_info *cqp_info; 688 struct irdma_create_qp_info *qp_info; 689 int status; 690 691 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 692 if (!cqp_request) 693 return -ENOMEM; 694 695 cqp_info = &cqp_request->info; 696 qp_info = &cqp_request->info.in.u.qp_create.info; 697 memset(qp_info, 0, sizeof(*qp_info)); 698 qp_info->mac_valid = true; 699 qp_info->cq_num_valid = true; 700 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 701 702 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 703 cqp_info->post_sq = 1; 704 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 705 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 706 status = irdma_handle_cqp_op(rf, cqp_request); 707 irdma_put_cqp_request(&rf->cqp, cqp_request); 708 709 return status; 710 } 711 712 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 713 struct irdma_qp_host_ctx_info *ctx_info) 714 { 715 struct irdma_device *iwdev = iwqp->iwdev; 716 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 717 struct irdma_roce_offload_info *roce_info; 718 struct irdma_udp_offload_info *udp_info; 719 720 udp_info = &iwqp->udp_info; 721 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 722 udp_info->cwnd = iwdev->roce_cwnd; 723 udp_info->rexmit_thresh = 2; 724 udp_info->rnr_nak_thresh = 2; 725 udp_info->src_port = 0xc000; 726 udp_info->dst_port = ROCE_V2_UDP_DPORT; 727 roce_info = &iwqp->roce_info; 728 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 729 730 roce_info->rd_en = true; 731 roce_info->wr_rdresp_en = true; 732 roce_info->bind_en = true; 733 roce_info->dcqcn_en = false; 734 roce_info->rtomin = 5; 735 736 roce_info->ack_credits = iwdev->roce_ackcreds; 737 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 738 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 739 740 if (!iwqp->user_mode) { 741 roce_info->priv_mode_en = true; 742 roce_info->fast_reg_en = true; 743 roce_info->udprivcq_en = true; 744 } 745 roce_info->roce_tver = 0; 746 747 ctx_info->roce_info = &iwqp->roce_info; 748 ctx_info->udp_info = &iwqp->udp_info; 749 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 750 } 751 752 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 753 struct irdma_qp_host_ctx_info *ctx_info) 754 { 755 struct irdma_device *iwdev = iwqp->iwdev; 756 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 757 struct irdma_iwarp_offload_info *iwarp_info; 758 759 iwarp_info = &iwqp->iwarp_info; 760 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 761 iwarp_info->rd_en = true; 762 iwarp_info->wr_rdresp_en = true; 763 iwarp_info->bind_en = true; 764 iwarp_info->ecn_en = true; 765 iwarp_info->rtomin = 5; 766 767 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 768 iwarp_info->ib_rd_en = true; 769 if (!iwqp->user_mode) { 770 iwarp_info->priv_mode_en = true; 771 iwarp_info->fast_reg_en = true; 772 } 773 iwarp_info->ddp_ver = 1; 774 iwarp_info->rdmap_ver = 1; 775 776 ctx_info->iwarp_info = &iwqp->iwarp_info; 777 ctx_info->iwarp_info_valid = true; 778 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 779 ctx_info->iwarp_info_valid = false; 780 } 781 782 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 783 struct irdma_device *iwdev) 784 { 785 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 786 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 787 788 if (init_attr->create_flags) 789 return -EOPNOTSUPP; 790 791 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 792 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 793 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) 794 return -EINVAL; 795 796 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 797 if (init_attr->qp_type != IB_QPT_RC && 798 init_attr->qp_type != IB_QPT_UD && 799 init_attr->qp_type != IB_QPT_GSI) 800 return -EOPNOTSUPP; 801 } else { 802 if (init_attr->qp_type != IB_QPT_RC) 803 return -EOPNOTSUPP; 804 } 805 806 return 0; 807 } 808 809 static void irdma_flush_worker(struct work_struct *work) 810 { 811 struct delayed_work *dwork = to_delayed_work(work); 812 struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); 813 814 irdma_generate_flush_completions(iwqp); 815 } 816 817 /** 818 * irdma_create_qp - create qp 819 * @ibqp: ptr of qp 820 * @init_attr: attributes for qp 821 * @udata: user data for create qp 822 */ 823 static int irdma_create_qp(struct ib_qp *ibqp, 824 struct ib_qp_init_attr *init_attr, 825 struct ib_udata *udata) 826 { 827 #define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) 828 #define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) 829 struct ib_pd *ibpd = ibqp->pd; 830 struct irdma_pd *iwpd = to_iwpd(ibpd); 831 struct irdma_device *iwdev = to_iwdev(ibpd->device); 832 struct irdma_pci_f *rf = iwdev->rf; 833 struct irdma_qp *iwqp = to_iwqp(ibqp); 834 struct irdma_create_qp_req req = {}; 835 struct irdma_create_qp_resp uresp = {}; 836 u32 qp_num = 0; 837 int err_code; 838 int sq_size; 839 int rq_size; 840 struct irdma_sc_qp *qp; 841 struct irdma_sc_dev *dev = &rf->sc_dev; 842 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 843 struct irdma_qp_init_info init_info = {}; 844 struct irdma_qp_host_ctx_info *ctx_info; 845 unsigned long flags; 846 847 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 848 if (err_code) 849 return err_code; 850 851 if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || 852 udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) 853 return -EINVAL; 854 855 sq_size = init_attr->cap.max_send_wr; 856 rq_size = init_attr->cap.max_recv_wr; 857 858 init_info.vsi = &iwdev->vsi; 859 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 860 init_info.qp_uk_init_info.sq_size = sq_size; 861 init_info.qp_uk_init_info.rq_size = rq_size; 862 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 863 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 864 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 865 866 qp = &iwqp->sc_qp; 867 qp->qp_uk.back_qp = iwqp; 868 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 869 870 iwqp->iwdev = iwdev; 871 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 872 256); 873 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 874 iwqp->q2_ctx_mem.size, 875 &iwqp->q2_ctx_mem.pa, 876 GFP_KERNEL); 877 if (!iwqp->q2_ctx_mem.va) 878 return -ENOMEM; 879 880 init_info.q2 = iwqp->q2_ctx_mem.va; 881 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 882 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 883 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 884 885 if (init_attr->qp_type == IB_QPT_GSI) 886 qp_num = 1; 887 else 888 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 889 &qp_num, &rf->next_qp); 890 if (err_code) 891 goto error; 892 893 iwqp->iwpd = iwpd; 894 iwqp->ibqp.qp_num = qp_num; 895 qp = &iwqp->sc_qp; 896 iwqp->iwscq = to_iwcq(init_attr->send_cq); 897 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 898 iwqp->host_ctx.va = init_info.host_ctx; 899 iwqp->host_ctx.pa = init_info.host_ctx_pa; 900 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 901 902 init_info.pd = &iwpd->sc_pd; 903 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; 904 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 905 init_info.qp_uk_init_info.first_sq_wq = 1; 906 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 907 init_waitqueue_head(&iwqp->waitq); 908 init_waitqueue_head(&iwqp->mod_qp_waitq); 909 910 if (udata) { 911 err_code = ib_copy_from_udata(&req, udata, 912 min(sizeof(req), udata->inlen)); 913 if (err_code) { 914 ibdev_dbg(&iwdev->ibdev, 915 "VERBS: ib_copy_from_data fail\n"); 916 goto error; 917 } 918 919 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 920 iwqp->user_mode = 1; 921 if (req.user_wqe_bufs) { 922 struct irdma_ucontext *ucontext = 923 rdma_udata_to_drv_context(udata, 924 struct irdma_ucontext, 925 ibucontext); 926 927 init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; 928 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 929 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 930 &ucontext->qp_reg_mem_list); 931 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 932 933 if (!iwqp->iwpbl) { 934 err_code = -ENODATA; 935 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 936 goto error; 937 } 938 } 939 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 940 irdma_setup_virt_qp(iwdev, iwqp, &init_info); 941 } else { 942 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 943 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 944 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 945 } 946 947 if (err_code) { 948 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 949 goto error; 950 } 951 952 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 953 if (init_attr->qp_type == IB_QPT_RC) { 954 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 955 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 956 IRDMA_WRITE_WITH_IMM | 957 IRDMA_ROCE; 958 } else { 959 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 960 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 961 IRDMA_ROCE; 962 } 963 } else { 964 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 965 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 966 } 967 968 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 969 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 970 971 err_code = irdma_sc_qp_init(qp, &init_info); 972 if (err_code) { 973 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 974 goto error; 975 } 976 977 ctx_info = &iwqp->ctx_info; 978 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 979 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 980 981 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 982 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 983 else 984 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 985 986 err_code = irdma_cqp_create_qp_cmd(iwqp); 987 if (err_code) 988 goto error; 989 990 refcount_set(&iwqp->refcnt, 1); 991 spin_lock_init(&iwqp->lock); 992 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 993 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 994 rf->qp_table[qp_num] = iwqp; 995 iwqp->max_send_wr = sq_size; 996 iwqp->max_recv_wr = rq_size; 997 998 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 999 if (dev->ws_add(&iwdev->vsi, 0)) { 1000 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 1001 err_code = -EINVAL; 1002 goto error; 1003 } 1004 1005 irdma_qp_add_qos(&iwqp->sc_qp); 1006 } 1007 1008 if (udata) { 1009 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 1010 if (udata->outlen < sizeof(uresp)) { 1011 uresp.lsmm = 1; 1012 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 1013 } else { 1014 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 1015 uresp.lsmm = 1; 1016 } 1017 uresp.actual_sq_size = sq_size; 1018 uresp.actual_rq_size = rq_size; 1019 uresp.qp_id = qp_num; 1020 uresp.qp_caps = qp->qp_uk.qp_caps; 1021 1022 err_code = ib_copy_to_udata(udata, &uresp, 1023 min(sizeof(uresp), udata->outlen)); 1024 if (err_code) { 1025 ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); 1026 irdma_destroy_qp(&iwqp->ibqp, udata); 1027 return err_code; 1028 } 1029 } 1030 1031 init_completion(&iwqp->free_qp); 1032 return 0; 1033 1034 error: 1035 irdma_free_qp_rsrc(iwqp); 1036 return err_code; 1037 } 1038 1039 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1040 { 1041 int acc_flags = 0; 1042 1043 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1044 if (iwqp->roce_info.wr_rdresp_en) { 1045 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1046 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1047 } 1048 if (iwqp->roce_info.rd_en) 1049 acc_flags |= IB_ACCESS_REMOTE_READ; 1050 if (iwqp->roce_info.bind_en) 1051 acc_flags |= IB_ACCESS_MW_BIND; 1052 } else { 1053 if (iwqp->iwarp_info.wr_rdresp_en) { 1054 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1055 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1056 } 1057 if (iwqp->iwarp_info.rd_en) 1058 acc_flags |= IB_ACCESS_REMOTE_READ; 1059 if (iwqp->iwarp_info.bind_en) 1060 acc_flags |= IB_ACCESS_MW_BIND; 1061 } 1062 return acc_flags; 1063 } 1064 1065 /** 1066 * irdma_query_qp - query qp attributes 1067 * @ibqp: qp pointer 1068 * @attr: attributes pointer 1069 * @attr_mask: Not used 1070 * @init_attr: qp attributes to return 1071 */ 1072 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1073 int attr_mask, struct ib_qp_init_attr *init_attr) 1074 { 1075 struct irdma_qp *iwqp = to_iwqp(ibqp); 1076 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1077 1078 memset(attr, 0, sizeof(*attr)); 1079 memset(init_attr, 0, sizeof(*init_attr)); 1080 1081 attr->qp_state = iwqp->ibqp_state; 1082 attr->cur_qp_state = iwqp->ibqp_state; 1083 attr->cap.max_send_wr = iwqp->max_send_wr; 1084 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1085 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1086 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1087 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1088 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1089 attr->port_num = 1; 1090 if (rdma_protocol_roce(ibqp->device, 1)) { 1091 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1092 attr->qkey = iwqp->roce_info.qkey; 1093 attr->rq_psn = iwqp->udp_info.epsn; 1094 attr->sq_psn = iwqp->udp_info.psn_nxt; 1095 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1096 attr->pkey_index = iwqp->roce_info.p_key; 1097 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1098 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1099 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1100 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1101 } 1102 1103 init_attr->event_handler = iwqp->ibqp.event_handler; 1104 init_attr->qp_context = iwqp->ibqp.qp_context; 1105 init_attr->send_cq = iwqp->ibqp.send_cq; 1106 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1107 init_attr->cap = attr->cap; 1108 1109 return 0; 1110 } 1111 1112 /** 1113 * irdma_query_pkey - Query partition key 1114 * @ibdev: device pointer from stack 1115 * @port: port number 1116 * @index: index of pkey 1117 * @pkey: pointer to store the pkey 1118 */ 1119 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1120 u16 *pkey) 1121 { 1122 if (index >= IRDMA_PKEY_TBL_SZ) 1123 return -EINVAL; 1124 1125 *pkey = IRDMA_DEFAULT_PKEY; 1126 return 0; 1127 } 1128 1129 /** 1130 * irdma_modify_qp_roce - modify qp request 1131 * @ibqp: qp's pointer for modify 1132 * @attr: access attributes 1133 * @attr_mask: state mask 1134 * @udata: user data 1135 */ 1136 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1137 int attr_mask, struct ib_udata *udata) 1138 { 1139 #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) 1140 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1141 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1142 struct irdma_qp *iwqp = to_iwqp(ibqp); 1143 struct irdma_device *iwdev = iwqp->iwdev; 1144 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1145 struct irdma_qp_host_ctx_info *ctx_info; 1146 struct irdma_roce_offload_info *roce_info; 1147 struct irdma_udp_offload_info *udp_info; 1148 struct irdma_modify_qp_info info = {}; 1149 struct irdma_modify_qp_resp uresp = {}; 1150 struct irdma_modify_qp_req ureq = {}; 1151 unsigned long flags; 1152 u8 issue_modify_qp = 0; 1153 int ret = 0; 1154 1155 ctx_info = &iwqp->ctx_info; 1156 roce_info = &iwqp->roce_info; 1157 udp_info = &iwqp->udp_info; 1158 1159 if (udata) { 1160 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1161 if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || 1162 (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) 1163 return -EINVAL; 1164 } 1165 1166 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1167 return -EOPNOTSUPP; 1168 1169 if (attr_mask & IB_QP_DEST_QPN) 1170 roce_info->dest_qp = attr->dest_qp_num; 1171 1172 if (attr_mask & IB_QP_PKEY_INDEX) { 1173 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1174 &roce_info->p_key); 1175 if (ret) 1176 return ret; 1177 } 1178 1179 if (attr_mask & IB_QP_QKEY) 1180 roce_info->qkey = attr->qkey; 1181 1182 if (attr_mask & IB_QP_PATH_MTU) 1183 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1184 1185 if (attr_mask & IB_QP_SQ_PSN) { 1186 udp_info->psn_nxt = attr->sq_psn; 1187 udp_info->lsn = 0xffff; 1188 udp_info->psn_una = attr->sq_psn; 1189 udp_info->psn_max = attr->sq_psn; 1190 } 1191 1192 if (attr_mask & IB_QP_RQ_PSN) 1193 udp_info->epsn = attr->rq_psn; 1194 1195 if (attr_mask & IB_QP_RNR_RETRY) 1196 udp_info->rnr_nak_thresh = attr->rnr_retry; 1197 1198 if (attr_mask & IB_QP_RETRY_CNT) 1199 udp_info->rexmit_thresh = attr->retry_cnt; 1200 1201 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1202 1203 if (attr_mask & IB_QP_AV) { 1204 struct irdma_av *av = &iwqp->roce_ah.av; 1205 const struct ib_gid_attr *sgid_attr; 1206 u16 vlan_id = VLAN_N_VID; 1207 u32 local_ip[4]; 1208 1209 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1210 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1211 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1212 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1213 udp_info->tos = attr->ah_attr.grh.traffic_class; 1214 udp_info->src_port = 1215 rdma_get_udp_sport(udp_info->flow_label, 1216 ibqp->qp_num, 1217 roce_info->dest_qp); 1218 irdma_qp_rem_qos(&iwqp->sc_qp); 1219 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1220 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1221 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1222 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1223 return -ENOMEM; 1224 irdma_qp_add_qos(&iwqp->sc_qp); 1225 } 1226 sgid_attr = attr->ah_attr.grh.sgid_attr; 1227 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1228 ctx_info->roce_info->mac_addr); 1229 if (ret) 1230 return ret; 1231 1232 if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 1233 vlan_id = 0; 1234 if (vlan_id < VLAN_N_VID) { 1235 udp_info->insert_vlan_tag = true; 1236 udp_info->vlan_tag = vlan_id | 1237 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1238 } else { 1239 udp_info->insert_vlan_tag = false; 1240 } 1241 1242 av->attrs = attr->ah_attr; 1243 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1244 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1245 if (av->net_type == RDMA_NETWORK_IPV6) { 1246 __be32 *daddr = 1247 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1248 __be32 *saddr = 1249 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1250 1251 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1252 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1253 1254 udp_info->ipv4 = false; 1255 irdma_copy_ip_ntohl(local_ip, daddr); 1256 1257 udp_info->arp_idx = irdma_arp_table(iwdev->rf, 1258 &local_ip[0], 1259 false, NULL, 1260 IRDMA_ARP_RESOLVE); 1261 } else if (av->net_type == RDMA_NETWORK_IPV4) { 1262 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1263 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1264 1265 local_ip[0] = ntohl(daddr); 1266 1267 udp_info->ipv4 = true; 1268 udp_info->dest_ip_addr[0] = 0; 1269 udp_info->dest_ip_addr[1] = 0; 1270 udp_info->dest_ip_addr[2] = 0; 1271 udp_info->dest_ip_addr[3] = local_ip[0]; 1272 1273 udp_info->local_ipaddr[0] = 0; 1274 udp_info->local_ipaddr[1] = 0; 1275 udp_info->local_ipaddr[2] = 0; 1276 udp_info->local_ipaddr[3] = ntohl(saddr); 1277 } 1278 udp_info->arp_idx = 1279 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1280 attr->ah_attr.roce.dmac); 1281 } 1282 1283 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1284 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1285 ibdev_err(&iwdev->ibdev, 1286 "rd_atomic = %d, above max_hw_ord=%d\n", 1287 attr->max_rd_atomic, 1288 dev->hw_attrs.max_hw_ord); 1289 return -EINVAL; 1290 } 1291 if (attr->max_rd_atomic) 1292 roce_info->ord_size = attr->max_rd_atomic; 1293 info.ord_valid = true; 1294 } 1295 1296 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1297 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1298 ibdev_err(&iwdev->ibdev, 1299 "rd_atomic = %d, above max_hw_ird=%d\n", 1300 attr->max_rd_atomic, 1301 dev->hw_attrs.max_hw_ird); 1302 return -EINVAL; 1303 } 1304 if (attr->max_dest_rd_atomic) 1305 roce_info->ird_size = attr->max_dest_rd_atomic; 1306 } 1307 1308 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1309 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1310 roce_info->wr_rdresp_en = true; 1311 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1312 roce_info->wr_rdresp_en = true; 1313 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1314 roce_info->rd_en = true; 1315 } 1316 1317 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1318 1319 ibdev_dbg(&iwdev->ibdev, 1320 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1321 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1322 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1323 1324 spin_lock_irqsave(&iwqp->lock, flags); 1325 if (attr_mask & IB_QP_STATE) { 1326 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1327 iwqp->ibqp.qp_type, attr_mask)) { 1328 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1329 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1330 attr->qp_state); 1331 ret = -EINVAL; 1332 goto exit; 1333 } 1334 info.curr_iwarp_state = iwqp->iwarp_state; 1335 1336 switch (attr->qp_state) { 1337 case IB_QPS_INIT: 1338 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1339 ret = -EINVAL; 1340 goto exit; 1341 } 1342 1343 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1344 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1345 issue_modify_qp = 1; 1346 } 1347 break; 1348 case IB_QPS_RTR: 1349 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1350 ret = -EINVAL; 1351 goto exit; 1352 } 1353 info.arp_cache_idx_valid = true; 1354 info.cq_num_valid = true; 1355 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1356 issue_modify_qp = 1; 1357 break; 1358 case IB_QPS_RTS: 1359 if (iwqp->ibqp_state < IB_QPS_RTR || 1360 iwqp->ibqp_state == IB_QPS_ERR) { 1361 ret = -EINVAL; 1362 goto exit; 1363 } 1364 1365 info.arp_cache_idx_valid = true; 1366 info.cq_num_valid = true; 1367 info.ord_valid = true; 1368 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1369 issue_modify_qp = 1; 1370 if (iwdev->push_mode && udata && 1371 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1372 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1373 spin_unlock_irqrestore(&iwqp->lock, flags); 1374 irdma_alloc_push_page(iwqp); 1375 spin_lock_irqsave(&iwqp->lock, flags); 1376 } 1377 break; 1378 case IB_QPS_SQD: 1379 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1380 goto exit; 1381 1382 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1383 ret = -EINVAL; 1384 goto exit; 1385 } 1386 1387 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1388 issue_modify_qp = 1; 1389 break; 1390 case IB_QPS_SQE: 1391 case IB_QPS_ERR: 1392 case IB_QPS_RESET: 1393 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { 1394 spin_unlock_irqrestore(&iwqp->lock, flags); 1395 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1396 irdma_hw_modify_qp(iwdev, iwqp, &info, true); 1397 spin_lock_irqsave(&iwqp->lock, flags); 1398 } 1399 1400 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1401 spin_unlock_irqrestore(&iwqp->lock, flags); 1402 if (udata && udata->inlen) { 1403 if (ib_copy_from_udata(&ureq, udata, 1404 min(sizeof(ureq), udata->inlen))) 1405 return -EINVAL; 1406 1407 irdma_flush_wqes(iwqp, 1408 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1409 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1410 IRDMA_REFLUSH); 1411 } 1412 return 0; 1413 } 1414 1415 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1416 issue_modify_qp = 1; 1417 break; 1418 default: 1419 ret = -EINVAL; 1420 goto exit; 1421 } 1422 1423 iwqp->ibqp_state = attr->qp_state; 1424 } 1425 1426 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1427 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1428 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1429 spin_unlock_irqrestore(&iwqp->lock, flags); 1430 1431 if (attr_mask & IB_QP_STATE) { 1432 if (issue_modify_qp) { 1433 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1434 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1435 return -EINVAL; 1436 spin_lock_irqsave(&iwqp->lock, flags); 1437 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1438 iwqp->iwarp_state = info.next_iwarp_state; 1439 iwqp->ibqp_state = attr->qp_state; 1440 } 1441 if (iwqp->ibqp_state > IB_QPS_RTS && 1442 !iwqp->flush_issued) { 1443 spin_unlock_irqrestore(&iwqp->lock, flags); 1444 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1445 IRDMA_FLUSH_RQ | 1446 IRDMA_FLUSH_WAIT); 1447 iwqp->flush_issued = 1; 1448 } else { 1449 spin_unlock_irqrestore(&iwqp->lock, flags); 1450 } 1451 } else { 1452 iwqp->ibqp_state = attr->qp_state; 1453 } 1454 if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1455 struct irdma_ucontext *ucontext; 1456 1457 ucontext = rdma_udata_to_drv_context(udata, 1458 struct irdma_ucontext, ibucontext); 1459 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1460 !iwqp->push_wqe_mmap_entry && 1461 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1462 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1463 uresp.push_valid = 1; 1464 uresp.push_offset = iwqp->sc_qp.push_offset; 1465 } 1466 ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1467 udata->outlen)); 1468 if (ret) { 1469 irdma_remove_push_mmap_entries(iwqp); 1470 ibdev_dbg(&iwdev->ibdev, 1471 "VERBS: copy_to_udata failed\n"); 1472 return ret; 1473 } 1474 } 1475 } 1476 1477 return 0; 1478 exit: 1479 spin_unlock_irqrestore(&iwqp->lock, flags); 1480 1481 return ret; 1482 } 1483 1484 /** 1485 * irdma_modify_qp - modify qp request 1486 * @ibqp: qp's pointer for modify 1487 * @attr: access attributes 1488 * @attr_mask: state mask 1489 * @udata: user data 1490 */ 1491 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1492 struct ib_udata *udata) 1493 { 1494 #define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) 1495 #define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) 1496 struct irdma_qp *iwqp = to_iwqp(ibqp); 1497 struct irdma_device *iwdev = iwqp->iwdev; 1498 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1499 struct irdma_qp_host_ctx_info *ctx_info; 1500 struct irdma_tcp_offload_info *tcp_info; 1501 struct irdma_iwarp_offload_info *offload_info; 1502 struct irdma_modify_qp_info info = {}; 1503 struct irdma_modify_qp_resp uresp = {}; 1504 struct irdma_modify_qp_req ureq = {}; 1505 u8 issue_modify_qp = 0; 1506 u8 dont_wait = 0; 1507 int err; 1508 unsigned long flags; 1509 1510 if (udata) { 1511 /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ 1512 if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || 1513 (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) 1514 return -EINVAL; 1515 } 1516 1517 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1518 return -EOPNOTSUPP; 1519 1520 ctx_info = &iwqp->ctx_info; 1521 offload_info = &iwqp->iwarp_info; 1522 tcp_info = &iwqp->tcp_info; 1523 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1524 ibdev_dbg(&iwdev->ibdev, 1525 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1526 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1527 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1528 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1529 1530 spin_lock_irqsave(&iwqp->lock, flags); 1531 if (attr_mask & IB_QP_STATE) { 1532 info.curr_iwarp_state = iwqp->iwarp_state; 1533 switch (attr->qp_state) { 1534 case IB_QPS_INIT: 1535 case IB_QPS_RTR: 1536 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1537 err = -EINVAL; 1538 goto exit; 1539 } 1540 1541 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1542 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1543 issue_modify_qp = 1; 1544 } 1545 if (iwdev->push_mode && udata && 1546 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1547 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1548 spin_unlock_irqrestore(&iwqp->lock, flags); 1549 irdma_alloc_push_page(iwqp); 1550 spin_lock_irqsave(&iwqp->lock, flags); 1551 } 1552 break; 1553 case IB_QPS_RTS: 1554 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1555 !iwqp->cm_id) { 1556 err = -EINVAL; 1557 goto exit; 1558 } 1559 1560 issue_modify_qp = 1; 1561 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1562 iwqp->hte_added = 1; 1563 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1564 info.tcp_ctx_valid = true; 1565 info.ord_valid = true; 1566 info.arp_cache_idx_valid = true; 1567 info.cq_num_valid = true; 1568 break; 1569 case IB_QPS_SQD: 1570 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1571 err = 0; 1572 goto exit; 1573 } 1574 1575 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1576 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1577 err = 0; 1578 goto exit; 1579 } 1580 1581 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1582 err = -EINVAL; 1583 goto exit; 1584 } 1585 1586 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1587 issue_modify_qp = 1; 1588 break; 1589 case IB_QPS_SQE: 1590 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1591 err = -EINVAL; 1592 goto exit; 1593 } 1594 1595 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1596 issue_modify_qp = 1; 1597 break; 1598 case IB_QPS_ERR: 1599 case IB_QPS_RESET: 1600 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1601 spin_unlock_irqrestore(&iwqp->lock, flags); 1602 if (udata && udata->inlen) { 1603 if (ib_copy_from_udata(&ureq, udata, 1604 min(sizeof(ureq), udata->inlen))) 1605 return -EINVAL; 1606 1607 irdma_flush_wqes(iwqp, 1608 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1609 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1610 IRDMA_REFLUSH); 1611 } 1612 return 0; 1613 } 1614 1615 if (iwqp->sc_qp.term_flags) { 1616 spin_unlock_irqrestore(&iwqp->lock, flags); 1617 irdma_terminate_del_timer(&iwqp->sc_qp); 1618 spin_lock_irqsave(&iwqp->lock, flags); 1619 } 1620 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1621 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1622 iwdev->iw_status && 1623 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1624 info.reset_tcp_conn = true; 1625 else 1626 dont_wait = 1; 1627 1628 issue_modify_qp = 1; 1629 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1630 break; 1631 default: 1632 err = -EINVAL; 1633 goto exit; 1634 } 1635 1636 iwqp->ibqp_state = attr->qp_state; 1637 } 1638 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1639 ctx_info->iwarp_info_valid = true; 1640 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1641 offload_info->wr_rdresp_en = true; 1642 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1643 offload_info->wr_rdresp_en = true; 1644 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1645 offload_info->rd_en = true; 1646 } 1647 1648 if (ctx_info->iwarp_info_valid) { 1649 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1650 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1651 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1652 } 1653 spin_unlock_irqrestore(&iwqp->lock, flags); 1654 1655 if (attr_mask & IB_QP_STATE) { 1656 if (issue_modify_qp) { 1657 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1658 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1659 return -EINVAL; 1660 } 1661 1662 spin_lock_irqsave(&iwqp->lock, flags); 1663 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1664 iwqp->iwarp_state = info.next_iwarp_state; 1665 iwqp->ibqp_state = attr->qp_state; 1666 } 1667 spin_unlock_irqrestore(&iwqp->lock, flags); 1668 } 1669 1670 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1671 if (dont_wait) { 1672 if (iwqp->hw_tcp_state) { 1673 spin_lock_irqsave(&iwqp->lock, flags); 1674 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1675 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1676 spin_unlock_irqrestore(&iwqp->lock, flags); 1677 } 1678 irdma_cm_disconn(iwqp); 1679 } else { 1680 int close_timer_started; 1681 1682 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1683 1684 if (iwqp->cm_node) { 1685 refcount_inc(&iwqp->cm_node->refcnt); 1686 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1687 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1688 if (iwqp->cm_id && close_timer_started == 1) 1689 irdma_schedule_cm_timer(iwqp->cm_node, 1690 (struct irdma_puda_buf *)iwqp, 1691 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1692 1693 irdma_rem_ref_cm_node(iwqp->cm_node); 1694 } else { 1695 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1696 } 1697 } 1698 } 1699 if (attr_mask & IB_QP_STATE && udata && udata->outlen && 1700 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1701 struct irdma_ucontext *ucontext; 1702 1703 ucontext = rdma_udata_to_drv_context(udata, 1704 struct irdma_ucontext, ibucontext); 1705 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1706 !iwqp->push_wqe_mmap_entry && 1707 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1708 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1709 uresp.push_valid = 1; 1710 uresp.push_offset = iwqp->sc_qp.push_offset; 1711 } 1712 1713 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1714 udata->outlen)); 1715 if (err) { 1716 irdma_remove_push_mmap_entries(iwqp); 1717 ibdev_dbg(&iwdev->ibdev, 1718 "VERBS: copy_to_udata failed\n"); 1719 return err; 1720 } 1721 } 1722 1723 return 0; 1724 exit: 1725 spin_unlock_irqrestore(&iwqp->lock, flags); 1726 1727 return err; 1728 } 1729 1730 /** 1731 * irdma_cq_free_rsrc - free up resources for cq 1732 * @rf: RDMA PCI function 1733 * @iwcq: cq ptr 1734 */ 1735 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1736 { 1737 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1738 1739 if (!iwcq->user_mode) { 1740 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1741 iwcq->kmem.va, iwcq->kmem.pa); 1742 iwcq->kmem.va = NULL; 1743 dma_free_coherent(rf->sc_dev.hw->device, 1744 iwcq->kmem_shadow.size, 1745 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1746 iwcq->kmem_shadow.va = NULL; 1747 } 1748 1749 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1750 } 1751 1752 /** 1753 * irdma_free_cqbuf - worker to free a cq buffer 1754 * @work: provides access to the cq buffer to free 1755 */ 1756 static void irdma_free_cqbuf(struct work_struct *work) 1757 { 1758 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1759 1760 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1761 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1762 cq_buf->kmem_buf.va = NULL; 1763 kfree(cq_buf); 1764 } 1765 1766 /** 1767 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1768 * @iwcq: cq which owns the resize_list 1769 * @iwdev: irdma device 1770 * @lcqe_buf: the buffer where the last cqe is received 1771 */ 1772 static int irdma_process_resize_list(struct irdma_cq *iwcq, 1773 struct irdma_device *iwdev, 1774 struct irdma_cq_buf *lcqe_buf) 1775 { 1776 struct list_head *tmp_node, *list_node; 1777 struct irdma_cq_buf *cq_buf; 1778 int cnt = 0; 1779 1780 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 1781 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 1782 if (cq_buf == lcqe_buf) 1783 return cnt; 1784 1785 list_del(&cq_buf->list); 1786 queue_work(iwdev->cleanup_wq, &cq_buf->work); 1787 cnt++; 1788 } 1789 1790 return cnt; 1791 } 1792 1793 /** 1794 * irdma_destroy_cq - destroy cq 1795 * @ib_cq: cq pointer 1796 * @udata: user data 1797 */ 1798 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1799 { 1800 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1801 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1802 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1803 struct irdma_sc_dev *dev = cq->dev; 1804 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1805 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1806 unsigned long flags; 1807 1808 spin_lock_irqsave(&iwcq->lock, flags); 1809 if (!list_empty(&iwcq->cmpl_generated)) 1810 irdma_remove_cmpls_list(iwcq); 1811 if (!list_empty(&iwcq->resize_list)) 1812 irdma_process_resize_list(iwcq, iwdev, NULL); 1813 spin_unlock_irqrestore(&iwcq->lock, flags); 1814 1815 irdma_cq_wq_destroy(iwdev->rf, cq); 1816 1817 spin_lock_irqsave(&iwceq->ce_lock, flags); 1818 irdma_sc_cleanup_ceqes(cq, ceq); 1819 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1820 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1821 1822 return 0; 1823 } 1824 1825 /** 1826 * irdma_resize_cq - resize cq 1827 * @ibcq: cq to be resized 1828 * @entries: desired cq size 1829 * @udata: user data 1830 */ 1831 static int irdma_resize_cq(struct ib_cq *ibcq, int entries, 1832 struct ib_udata *udata) 1833 { 1834 #define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer) 1835 struct irdma_cq *iwcq = to_iwcq(ibcq); 1836 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 1837 struct irdma_cqp_request *cqp_request; 1838 struct cqp_cmds_info *cqp_info; 1839 struct irdma_modify_cq_info *m_info; 1840 struct irdma_modify_cq_info info = {}; 1841 struct irdma_dma_mem kmem_buf; 1842 struct irdma_cq_mr *cqmr_buf; 1843 struct irdma_pbl *iwpbl_buf; 1844 struct irdma_device *iwdev; 1845 struct irdma_pci_f *rf; 1846 struct irdma_cq_buf *cq_buf = NULL; 1847 unsigned long flags; 1848 int ret; 1849 1850 iwdev = to_iwdev(ibcq->device); 1851 rf = iwdev->rf; 1852 1853 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 1854 IRDMA_FEATURE_CQ_RESIZE)) 1855 return -EOPNOTSUPP; 1856 1857 if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN) 1858 return -EINVAL; 1859 1860 if (entries > rf->max_cqe) 1861 return -EINVAL; 1862 1863 if (!iwcq->user_mode) { 1864 entries++; 1865 if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1866 entries *= 2; 1867 } 1868 1869 info.cq_size = max(entries, 4); 1870 1871 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 1872 return 0; 1873 1874 if (udata) { 1875 struct irdma_resize_cq_req req = {}; 1876 struct irdma_ucontext *ucontext = 1877 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 1878 ibucontext); 1879 1880 /* CQ resize not supported with legacy GEN_1 libi40iw */ 1881 if (ucontext->legacy_mode) 1882 return -EOPNOTSUPP; 1883 1884 if (ib_copy_from_udata(&req, udata, 1885 min(sizeof(req), udata->inlen))) 1886 return -EINVAL; 1887 1888 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1889 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 1890 &ucontext->cq_reg_mem_list); 1891 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1892 1893 if (!iwpbl_buf) 1894 return -ENOMEM; 1895 1896 cqmr_buf = &iwpbl_buf->cq_mr; 1897 if (iwpbl_buf->pbl_allocated) { 1898 info.virtual_map = true; 1899 info.pbl_chunk_size = 1; 1900 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 1901 } else { 1902 info.cq_pa = cqmr_buf->cq_pbl.addr; 1903 } 1904 } else { 1905 /* Kmode CQ resize */ 1906 int rsize; 1907 1908 rsize = info.cq_size * sizeof(struct irdma_cqe); 1909 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 1910 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 1911 kmem_buf.size, &kmem_buf.pa, 1912 GFP_KERNEL); 1913 if (!kmem_buf.va) 1914 return -ENOMEM; 1915 1916 info.cq_base = kmem_buf.va; 1917 info.cq_pa = kmem_buf.pa; 1918 cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); 1919 if (!cq_buf) { 1920 ret = -ENOMEM; 1921 goto error; 1922 } 1923 } 1924 1925 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1926 if (!cqp_request) { 1927 ret = -ENOMEM; 1928 goto error; 1929 } 1930 1931 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 1932 info.cq_resize = true; 1933 1934 cqp_info = &cqp_request->info; 1935 m_info = &cqp_info->in.u.cq_modify.info; 1936 memcpy(m_info, &info, sizeof(*m_info)); 1937 1938 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 1939 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 1940 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 1941 cqp_info->post_sq = 1; 1942 ret = irdma_handle_cqp_op(rf, cqp_request); 1943 irdma_put_cqp_request(&rf->cqp, cqp_request); 1944 if (ret) 1945 goto error; 1946 1947 spin_lock_irqsave(&iwcq->lock, flags); 1948 if (cq_buf) { 1949 cq_buf->kmem_buf = iwcq->kmem; 1950 cq_buf->hw = dev->hw; 1951 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 1952 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 1953 list_add_tail(&cq_buf->list, &iwcq->resize_list); 1954 iwcq->kmem = kmem_buf; 1955 } 1956 1957 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 1958 ibcq->cqe = info.cq_size - 1; 1959 spin_unlock_irqrestore(&iwcq->lock, flags); 1960 1961 return 0; 1962 error: 1963 if (!udata) { 1964 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 1965 kmem_buf.pa); 1966 kmem_buf.va = NULL; 1967 } 1968 kfree(cq_buf); 1969 1970 return ret; 1971 } 1972 1973 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 1974 { 1975 /* GEN1 does not support CQ create flags */ 1976 if (hw_rev == IRDMA_GEN_1) 1977 return flags ? -EOPNOTSUPP : 0; 1978 1979 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 1980 } 1981 1982 /** 1983 * irdma_create_cq - create cq 1984 * @ibcq: CQ allocated 1985 * @attr: attributes for cq 1986 * @udata: user data 1987 */ 1988 static int irdma_create_cq(struct ib_cq *ibcq, 1989 const struct ib_cq_init_attr *attr, 1990 struct ib_udata *udata) 1991 { 1992 #define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) 1993 #define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) 1994 struct ib_device *ibdev = ibcq->device; 1995 struct irdma_device *iwdev = to_iwdev(ibdev); 1996 struct irdma_pci_f *rf = iwdev->rf; 1997 struct irdma_cq *iwcq = to_iwcq(ibcq); 1998 u32 cq_num = 0; 1999 struct irdma_sc_cq *cq; 2000 struct irdma_sc_dev *dev = &rf->sc_dev; 2001 struct irdma_cq_init_info info = {}; 2002 struct irdma_cqp_request *cqp_request; 2003 struct cqp_cmds_info *cqp_info; 2004 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 2005 unsigned long flags; 2006 int err_code; 2007 int entries = attr->cqe; 2008 2009 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 2010 if (err_code) 2011 return err_code; 2012 2013 if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || 2014 udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) 2015 return -EINVAL; 2016 2017 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 2018 &rf->next_cq); 2019 if (err_code) 2020 return err_code; 2021 2022 cq = &iwcq->sc_cq; 2023 cq->back_cq = iwcq; 2024 spin_lock_init(&iwcq->lock); 2025 INIT_LIST_HEAD(&iwcq->resize_list); 2026 INIT_LIST_HEAD(&iwcq->cmpl_generated); 2027 info.dev = dev; 2028 ukinfo->cq_size = max(entries, 4); 2029 ukinfo->cq_id = cq_num; 2030 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 2031 if (attr->comp_vector < rf->ceqs_count) 2032 info.ceq_id = attr->comp_vector; 2033 info.ceq_id_valid = true; 2034 info.ceqe_mask = 1; 2035 info.type = IRDMA_CQ_TYPE_IWARP; 2036 info.vsi = &iwdev->vsi; 2037 2038 if (udata) { 2039 struct irdma_ucontext *ucontext; 2040 struct irdma_create_cq_req req = {}; 2041 struct irdma_cq_mr *cqmr; 2042 struct irdma_pbl *iwpbl; 2043 struct irdma_pbl *iwpbl_shadow; 2044 struct irdma_cq_mr *cqmr_shadow; 2045 2046 iwcq->user_mode = true; 2047 ucontext = 2048 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2049 ibucontext); 2050 if (ib_copy_from_udata(&req, udata, 2051 min(sizeof(req), udata->inlen))) { 2052 err_code = -EFAULT; 2053 goto cq_free_rsrc; 2054 } 2055 2056 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2057 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 2058 &ucontext->cq_reg_mem_list); 2059 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2060 if (!iwpbl) { 2061 err_code = -EPROTO; 2062 goto cq_free_rsrc; 2063 } 2064 2065 iwcq->iwpbl = iwpbl; 2066 iwcq->cq_mem_size = 0; 2067 cqmr = &iwpbl->cq_mr; 2068 2069 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2070 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 2071 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2072 iwpbl_shadow = irdma_get_pbl( 2073 (unsigned long)req.user_shadow_area, 2074 &ucontext->cq_reg_mem_list); 2075 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2076 2077 if (!iwpbl_shadow) { 2078 err_code = -EPROTO; 2079 goto cq_free_rsrc; 2080 } 2081 iwcq->iwpbl_shadow = iwpbl_shadow; 2082 cqmr_shadow = &iwpbl_shadow->cq_mr; 2083 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2084 cqmr->split = true; 2085 } else { 2086 info.shadow_area_pa = cqmr->shadow; 2087 } 2088 if (iwpbl->pbl_allocated) { 2089 info.virtual_map = true; 2090 info.pbl_chunk_size = 1; 2091 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2092 } else { 2093 info.cq_base_pa = cqmr->cq_pbl.addr; 2094 } 2095 } else { 2096 /* Kmode allocations */ 2097 int rsize; 2098 2099 if (entries < 1 || entries > rf->max_cqe) { 2100 err_code = -EINVAL; 2101 goto cq_free_rsrc; 2102 } 2103 2104 entries++; 2105 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2106 entries *= 2; 2107 ukinfo->cq_size = entries; 2108 2109 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2110 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2111 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2112 iwcq->kmem.size, 2113 &iwcq->kmem.pa, GFP_KERNEL); 2114 if (!iwcq->kmem.va) { 2115 err_code = -ENOMEM; 2116 goto cq_free_rsrc; 2117 } 2118 2119 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2120 64); 2121 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2122 iwcq->kmem_shadow.size, 2123 &iwcq->kmem_shadow.pa, 2124 GFP_KERNEL); 2125 if (!iwcq->kmem_shadow.va) { 2126 err_code = -ENOMEM; 2127 goto cq_free_rsrc; 2128 } 2129 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2130 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2131 ukinfo->cq_base = iwcq->kmem.va; 2132 info.cq_base_pa = iwcq->kmem.pa; 2133 } 2134 2135 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2136 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2137 (u32)IRDMA_MAX_CQ_READ_THRESH); 2138 2139 if (irdma_sc_cq_init(cq, &info)) { 2140 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2141 err_code = -EPROTO; 2142 goto cq_free_rsrc; 2143 } 2144 2145 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2146 if (!cqp_request) { 2147 err_code = -ENOMEM; 2148 goto cq_free_rsrc; 2149 } 2150 2151 cqp_info = &cqp_request->info; 2152 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2153 cqp_info->post_sq = 1; 2154 cqp_info->in.u.cq_create.cq = cq; 2155 cqp_info->in.u.cq_create.check_overflow = true; 2156 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2157 err_code = irdma_handle_cqp_op(rf, cqp_request); 2158 irdma_put_cqp_request(&rf->cqp, cqp_request); 2159 if (err_code) 2160 goto cq_free_rsrc; 2161 2162 if (udata) { 2163 struct irdma_create_cq_resp resp = {}; 2164 2165 resp.cq_id = info.cq_uk_init_info.cq_id; 2166 resp.cq_size = info.cq_uk_init_info.cq_size; 2167 if (ib_copy_to_udata(udata, &resp, 2168 min(sizeof(resp), udata->outlen))) { 2169 ibdev_dbg(&iwdev->ibdev, 2170 "VERBS: copy to user data\n"); 2171 err_code = -EPROTO; 2172 goto cq_destroy; 2173 } 2174 } 2175 return 0; 2176 cq_destroy: 2177 irdma_cq_wq_destroy(rf, cq); 2178 cq_free_rsrc: 2179 irdma_cq_free_rsrc(rf, iwcq); 2180 2181 return err_code; 2182 } 2183 2184 /** 2185 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2186 * @access: IB access flags 2187 */ 2188 static inline u16 irdma_get_mr_access(int access) 2189 { 2190 u16 hw_access = 0; 2191 2192 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2193 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2194 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2195 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2196 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2197 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2198 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2199 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2200 hw_access |= (access & IB_ZERO_BASED) ? 2201 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2202 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2203 2204 return hw_access; 2205 } 2206 2207 /** 2208 * irdma_free_stag - free stag resource 2209 * @iwdev: irdma device 2210 * @stag: stag to free 2211 */ 2212 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2213 { 2214 u32 stag_idx; 2215 2216 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2217 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2218 } 2219 2220 /** 2221 * irdma_create_stag - create random stag 2222 * @iwdev: irdma device 2223 */ 2224 static u32 irdma_create_stag(struct irdma_device *iwdev) 2225 { 2226 u32 stag = 0; 2227 u32 stag_index = 0; 2228 u32 next_stag_index; 2229 u32 driver_key; 2230 u32 random; 2231 u8 consumer_key; 2232 int ret; 2233 2234 get_random_bytes(&random, sizeof(random)); 2235 consumer_key = (u8)random; 2236 2237 driver_key = random & ~iwdev->rf->mr_stagmask; 2238 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2239 next_stag_index %= iwdev->rf->max_mr; 2240 2241 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2242 iwdev->rf->max_mr, &stag_index, 2243 &next_stag_index); 2244 if (ret) 2245 return stag; 2246 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2247 stag |= driver_key; 2248 stag += (u32)consumer_key; 2249 2250 return stag; 2251 } 2252 2253 /** 2254 * irdma_next_pbl_addr - Get next pbl address 2255 * @pbl: pointer to a pble 2256 * @pinfo: info pointer 2257 * @idx: index 2258 */ 2259 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2260 u32 *idx) 2261 { 2262 *idx += 1; 2263 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2264 return ++pbl; 2265 *idx = 0; 2266 (*pinfo)++; 2267 2268 return (*pinfo)->addr; 2269 } 2270 2271 /** 2272 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2273 * @iwmr: iwmr for IB's user page addresses 2274 * @pbl: ple pointer to save 1 level or 0 level pble 2275 * @level: indicated level 0, 1 or 2 2276 */ 2277 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2278 enum irdma_pble_level level) 2279 { 2280 struct ib_umem *region = iwmr->region; 2281 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2282 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2283 struct irdma_pble_info *pinfo; 2284 struct ib_block_iter biter; 2285 u32 idx = 0; 2286 u32 pbl_cnt = 0; 2287 2288 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2289 2290 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2291 iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); 2292 2293 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2294 *pbl = rdma_block_iter_dma_address(&biter); 2295 if (++pbl_cnt == palloc->total_cnt) 2296 break; 2297 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2298 } 2299 } 2300 2301 /** 2302 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2303 * @arr: lvl1 pbl array 2304 * @npages: page count 2305 * @pg_size: page size 2306 * 2307 */ 2308 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2309 { 2310 u32 pg_idx; 2311 2312 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2313 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) 2314 return false; 2315 } 2316 2317 return true; 2318 } 2319 2320 /** 2321 * irdma_check_mr_contiguous - check if MR is physically contiguous 2322 * @palloc: pbl allocation struct 2323 * @pg_size: page size 2324 */ 2325 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, 2326 u32 pg_size) 2327 { 2328 struct irdma_pble_level2 *lvl2 = &palloc->level2; 2329 struct irdma_pble_info *leaf = lvl2->leaf; 2330 u64 *arr = NULL; 2331 u64 *start_addr = NULL; 2332 int i; 2333 bool ret; 2334 2335 if (palloc->level == PBLE_LEVEL_1) { 2336 arr = palloc->level1.addr; 2337 ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, 2338 pg_size); 2339 return ret; 2340 } 2341 2342 start_addr = leaf->addr; 2343 2344 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 2345 arr = leaf->addr; 2346 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) 2347 return false; 2348 ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); 2349 if (!ret) 2350 return false; 2351 } 2352 2353 return true; 2354 } 2355 2356 /** 2357 * irdma_setup_pbles - copy user pg address to pble's 2358 * @rf: RDMA PCI function 2359 * @iwmr: mr pointer for this memory registration 2360 * @use_pbles: flag if to use pble's 2361 */ 2362 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2363 bool use_pbles) 2364 { 2365 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2366 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2367 struct irdma_pble_info *pinfo; 2368 u64 *pbl; 2369 int status; 2370 enum irdma_pble_level level = PBLE_LEVEL_1; 2371 2372 if (use_pbles) { 2373 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2374 false); 2375 if (status) 2376 return status; 2377 2378 iwpbl->pbl_allocated = true; 2379 level = palloc->level; 2380 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2381 palloc->level2.leaf; 2382 pbl = pinfo->addr; 2383 } else { 2384 pbl = iwmr->pgaddrmem; 2385 } 2386 2387 irdma_copy_user_pgaddrs(iwmr, pbl, level); 2388 2389 if (use_pbles) 2390 iwmr->pgaddrmem[0] = *pbl; 2391 2392 return 0; 2393 } 2394 2395 /** 2396 * irdma_handle_q_mem - handle memory for qp and cq 2397 * @iwdev: irdma device 2398 * @req: information for q memory management 2399 * @iwpbl: pble struct 2400 * @use_pbles: flag to use pble 2401 */ 2402 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2403 struct irdma_mem_reg_req *req, 2404 struct irdma_pbl *iwpbl, bool use_pbles) 2405 { 2406 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2407 struct irdma_mr *iwmr = iwpbl->iwmr; 2408 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2409 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2410 struct irdma_hmc_pble *hmc_p; 2411 u64 *arr = iwmr->pgaddrmem; 2412 u32 pg_size, total; 2413 int err = 0; 2414 bool ret = true; 2415 2416 pg_size = iwmr->page_size; 2417 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2418 if (err) 2419 return err; 2420 2421 if (use_pbles && palloc->level != PBLE_LEVEL_1) { 2422 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2423 iwpbl->pbl_allocated = false; 2424 return -ENOMEM; 2425 } 2426 2427 if (use_pbles) 2428 arr = palloc->level1.addr; 2429 2430 switch (iwmr->type) { 2431 case IRDMA_MEMREG_TYPE_QP: 2432 total = req->sq_pages + req->rq_pages; 2433 hmc_p = &qpmr->sq_pbl; 2434 qpmr->shadow = (dma_addr_t)arr[total]; 2435 2436 if (use_pbles) { 2437 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 2438 pg_size); 2439 if (ret) 2440 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 2441 req->rq_pages, 2442 pg_size); 2443 } 2444 2445 if (!ret) { 2446 hmc_p->idx = palloc->level1.idx; 2447 hmc_p = &qpmr->rq_pbl; 2448 hmc_p->idx = palloc->level1.idx + req->sq_pages; 2449 } else { 2450 hmc_p->addr = arr[0]; 2451 hmc_p = &qpmr->rq_pbl; 2452 hmc_p->addr = arr[req->sq_pages]; 2453 } 2454 break; 2455 case IRDMA_MEMREG_TYPE_CQ: 2456 hmc_p = &cqmr->cq_pbl; 2457 2458 if (!cqmr->split) 2459 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 2460 2461 if (use_pbles) 2462 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 2463 pg_size); 2464 2465 if (!ret) 2466 hmc_p->idx = palloc->level1.idx; 2467 else 2468 hmc_p->addr = arr[0]; 2469 break; 2470 default: 2471 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 2472 err = -EINVAL; 2473 } 2474 2475 if (use_pbles && ret) { 2476 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2477 iwpbl->pbl_allocated = false; 2478 } 2479 2480 return err; 2481 } 2482 2483 /** 2484 * irdma_hw_alloc_mw - create the hw memory window 2485 * @iwdev: irdma device 2486 * @iwmr: pointer to memory window info 2487 */ 2488 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 2489 { 2490 struct irdma_mw_alloc_info *info; 2491 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2492 struct irdma_cqp_request *cqp_request; 2493 struct cqp_cmds_info *cqp_info; 2494 int status; 2495 2496 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2497 if (!cqp_request) 2498 return -ENOMEM; 2499 2500 cqp_info = &cqp_request->info; 2501 info = &cqp_info->in.u.mw_alloc.info; 2502 memset(info, 0, sizeof(*info)); 2503 if (iwmr->ibmw.type == IB_MW_TYPE_1) 2504 info->mw_wide = true; 2505 2506 info->page_size = PAGE_SIZE; 2507 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2508 info->pd_id = iwpd->sc_pd.pd_id; 2509 info->remote_access = true; 2510 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 2511 cqp_info->post_sq = 1; 2512 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 2513 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 2514 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2515 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2516 2517 return status; 2518 } 2519 2520 /** 2521 * irdma_alloc_mw - Allocate memory window 2522 * @ibmw: Memory Window 2523 * @udata: user data pointer 2524 */ 2525 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 2526 { 2527 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2528 struct irdma_mr *iwmr = to_iwmw(ibmw); 2529 int err_code; 2530 u32 stag; 2531 2532 stag = irdma_create_stag(iwdev); 2533 if (!stag) 2534 return -ENOMEM; 2535 2536 iwmr->stag = stag; 2537 ibmw->rkey = stag; 2538 2539 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 2540 if (err_code) { 2541 irdma_free_stag(iwdev, stag); 2542 return err_code; 2543 } 2544 2545 return 0; 2546 } 2547 2548 /** 2549 * irdma_dealloc_mw - Dealloc memory window 2550 * @ibmw: memory window structure. 2551 */ 2552 static int irdma_dealloc_mw(struct ib_mw *ibmw) 2553 { 2554 struct ib_pd *ibpd = ibmw->pd; 2555 struct irdma_pd *iwpd = to_iwpd(ibpd); 2556 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 2557 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2558 struct irdma_cqp_request *cqp_request; 2559 struct cqp_cmds_info *cqp_info; 2560 struct irdma_dealloc_stag_info *info; 2561 2562 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2563 if (!cqp_request) 2564 return -ENOMEM; 2565 2566 cqp_info = &cqp_request->info; 2567 info = &cqp_info->in.u.dealloc_stag.info; 2568 memset(info, 0, sizeof(*info)); 2569 info->pd_id = iwpd->sc_pd.pd_id; 2570 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 2571 info->mr = false; 2572 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 2573 cqp_info->post_sq = 1; 2574 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 2575 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 2576 irdma_handle_cqp_op(iwdev->rf, cqp_request); 2577 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2578 irdma_free_stag(iwdev, iwmr->stag); 2579 2580 return 0; 2581 } 2582 2583 /** 2584 * irdma_hw_alloc_stag - cqp command to allocate stag 2585 * @iwdev: irdma device 2586 * @iwmr: irdma mr pointer 2587 */ 2588 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 2589 struct irdma_mr *iwmr) 2590 { 2591 struct irdma_allocate_stag_info *info; 2592 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2593 int status; 2594 struct irdma_cqp_request *cqp_request; 2595 struct cqp_cmds_info *cqp_info; 2596 2597 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2598 if (!cqp_request) 2599 return -ENOMEM; 2600 2601 cqp_info = &cqp_request->info; 2602 info = &cqp_info->in.u.alloc_stag.info; 2603 memset(info, 0, sizeof(*info)); 2604 info->page_size = PAGE_SIZE; 2605 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2606 info->pd_id = iwpd->sc_pd.pd_id; 2607 info->total_len = iwmr->len; 2608 info->remote_access = true; 2609 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 2610 cqp_info->post_sq = 1; 2611 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 2612 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 2613 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2614 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2615 2616 return status; 2617 } 2618 2619 /** 2620 * irdma_alloc_mr - register stag for fast memory registration 2621 * @pd: ibpd pointer 2622 * @mr_type: memory for stag registrion 2623 * @max_num_sg: man number of pages 2624 */ 2625 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 2626 u32 max_num_sg) 2627 { 2628 struct irdma_device *iwdev = to_iwdev(pd->device); 2629 struct irdma_pble_alloc *palloc; 2630 struct irdma_pbl *iwpbl; 2631 struct irdma_mr *iwmr; 2632 u32 stag; 2633 int err_code; 2634 2635 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2636 if (!iwmr) 2637 return ERR_PTR(-ENOMEM); 2638 2639 stag = irdma_create_stag(iwdev); 2640 if (!stag) { 2641 err_code = -ENOMEM; 2642 goto err; 2643 } 2644 2645 iwmr->stag = stag; 2646 iwmr->ibmr.rkey = stag; 2647 iwmr->ibmr.lkey = stag; 2648 iwmr->ibmr.pd = pd; 2649 iwmr->ibmr.device = pd->device; 2650 iwpbl = &iwmr->iwpbl; 2651 iwpbl->iwmr = iwmr; 2652 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2653 palloc = &iwpbl->pble_alloc; 2654 iwmr->page_cnt = max_num_sg; 2655 err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 2656 false); 2657 if (err_code) 2658 goto err_get_pble; 2659 2660 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 2661 if (err_code) 2662 goto err_alloc_stag; 2663 2664 iwpbl->pbl_allocated = true; 2665 2666 return &iwmr->ibmr; 2667 err_alloc_stag: 2668 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2669 err_get_pble: 2670 irdma_free_stag(iwdev, stag); 2671 err: 2672 kfree(iwmr); 2673 2674 return ERR_PTR(err_code); 2675 } 2676 2677 /** 2678 * irdma_set_page - populate pbl list for fmr 2679 * @ibmr: ib mem to access iwarp mr pointer 2680 * @addr: page dma address fro pbl list 2681 */ 2682 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 2683 { 2684 struct irdma_mr *iwmr = to_iwmr(ibmr); 2685 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2686 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2687 u64 *pbl; 2688 2689 if (unlikely(iwmr->npages == iwmr->page_cnt)) 2690 return -ENOMEM; 2691 2692 if (palloc->level == PBLE_LEVEL_2) { 2693 struct irdma_pble_info *palloc_info = 2694 palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT); 2695 2696 palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr; 2697 } else { 2698 pbl = palloc->level1.addr; 2699 pbl[iwmr->npages] = addr; 2700 } 2701 iwmr->npages++; 2702 2703 return 0; 2704 } 2705 2706 /** 2707 * irdma_map_mr_sg - map of sg list for fmr 2708 * @ibmr: ib mem to access iwarp mr pointer 2709 * @sg: scatter gather list 2710 * @sg_nents: number of sg pages 2711 * @sg_offset: scatter gather list for fmr 2712 */ 2713 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2714 int sg_nents, unsigned int *sg_offset) 2715 { 2716 struct irdma_mr *iwmr = to_iwmr(ibmr); 2717 2718 iwmr->npages = 0; 2719 2720 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 2721 } 2722 2723 /** 2724 * irdma_hwreg_mr - send cqp command for memory registration 2725 * @iwdev: irdma device 2726 * @iwmr: irdma mr pointer 2727 * @access: access for MR 2728 */ 2729 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 2730 u16 access) 2731 { 2732 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2733 struct irdma_reg_ns_stag_info *stag_info; 2734 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2735 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2736 struct irdma_cqp_request *cqp_request; 2737 struct cqp_cmds_info *cqp_info; 2738 int ret; 2739 2740 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2741 if (!cqp_request) 2742 return -ENOMEM; 2743 2744 cqp_info = &cqp_request->info; 2745 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 2746 memset(stag_info, 0, sizeof(*stag_info)); 2747 stag_info->va = iwpbl->user_base; 2748 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2749 stag_info->stag_key = (u8)iwmr->stag; 2750 stag_info->total_len = iwmr->len; 2751 stag_info->access_rights = irdma_get_mr_access(access); 2752 stag_info->pd_id = iwpd->sc_pd.pd_id; 2753 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 2754 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 2755 else 2756 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 2757 stag_info->page_size = iwmr->page_size; 2758 2759 if (iwpbl->pbl_allocated) { 2760 if (palloc->level == PBLE_LEVEL_1) { 2761 stag_info->first_pm_pbl_index = palloc->level1.idx; 2762 stag_info->chunk_size = 1; 2763 } else { 2764 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 2765 stag_info->chunk_size = 3; 2766 } 2767 } else { 2768 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 2769 } 2770 2771 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 2772 cqp_info->post_sq = 1; 2773 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 2774 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 2775 ret = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2776 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2777 2778 return ret; 2779 } 2780 2781 /** 2782 * irdma_reg_user_mr - Register a user memory region 2783 * @pd: ptr of pd 2784 * @start: virtual start address 2785 * @len: length of mr 2786 * @virt: virtual address 2787 * @access: access of mr 2788 * @udata: user data 2789 */ 2790 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 2791 u64 virt, int access, 2792 struct ib_udata *udata) 2793 { 2794 #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) 2795 struct irdma_device *iwdev = to_iwdev(pd->device); 2796 struct irdma_ucontext *ucontext; 2797 struct irdma_pble_alloc *palloc; 2798 struct irdma_pbl *iwpbl; 2799 struct irdma_mr *iwmr; 2800 struct ib_umem *region; 2801 struct irdma_mem_reg_req req; 2802 u32 total, stag = 0; 2803 u8 shadow_pgcnt = 1; 2804 bool use_pbles = false; 2805 unsigned long flags; 2806 int err = -EINVAL; 2807 int ret; 2808 2809 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 2810 return ERR_PTR(-EINVAL); 2811 2812 if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) 2813 return ERR_PTR(-EINVAL); 2814 2815 region = ib_umem_get(pd->device, start, len, access); 2816 2817 if (IS_ERR(region)) { 2818 ibdev_dbg(&iwdev->ibdev, 2819 "VERBS: Failed to create ib_umem region\n"); 2820 return (struct ib_mr *)region; 2821 } 2822 2823 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 2824 ib_umem_release(region); 2825 return ERR_PTR(-EFAULT); 2826 } 2827 2828 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2829 if (!iwmr) { 2830 ib_umem_release(region); 2831 return ERR_PTR(-ENOMEM); 2832 } 2833 2834 iwpbl = &iwmr->iwpbl; 2835 iwpbl->iwmr = iwmr; 2836 iwmr->region = region; 2837 iwmr->ibmr.pd = pd; 2838 iwmr->ibmr.device = pd->device; 2839 iwmr->ibmr.iova = virt; 2840 iwmr->page_size = PAGE_SIZE; 2841 2842 if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) { 2843 iwmr->page_size = ib_umem_find_best_pgsz(region, 2844 iwdev->rf->sc_dev.hw_attrs.page_size_cap, 2845 virt); 2846 if (unlikely(!iwmr->page_size)) { 2847 kfree(iwmr); 2848 ib_umem_release(region); 2849 return ERR_PTR(-EOPNOTSUPP); 2850 } 2851 } 2852 iwmr->len = region->length; 2853 iwpbl->user_base = virt; 2854 palloc = &iwpbl->pble_alloc; 2855 iwmr->type = req.reg_type; 2856 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 2857 2858 switch (req.reg_type) { 2859 case IRDMA_MEMREG_TYPE_QP: 2860 total = req.sq_pages + req.rq_pages + shadow_pgcnt; 2861 if (total > iwmr->page_cnt) { 2862 err = -EINVAL; 2863 goto error; 2864 } 2865 total = req.sq_pages + req.rq_pages; 2866 use_pbles = (total > 2); 2867 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2868 if (err) 2869 goto error; 2870 2871 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2872 ibucontext); 2873 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2874 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 2875 iwpbl->on_list = true; 2876 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2877 break; 2878 case IRDMA_MEMREG_TYPE_CQ: 2879 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 2880 shadow_pgcnt = 0; 2881 total = req.cq_pages + shadow_pgcnt; 2882 if (total > iwmr->page_cnt) { 2883 err = -EINVAL; 2884 goto error; 2885 } 2886 2887 use_pbles = (req.cq_pages > 1); 2888 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2889 if (err) 2890 goto error; 2891 2892 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2893 ibucontext); 2894 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2895 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 2896 iwpbl->on_list = true; 2897 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2898 break; 2899 case IRDMA_MEMREG_TYPE_MEM: 2900 use_pbles = (iwmr->page_cnt != 1); 2901 2902 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2903 if (err) 2904 goto error; 2905 2906 if (use_pbles) { 2907 ret = irdma_check_mr_contiguous(palloc, 2908 iwmr->page_size); 2909 if (ret) { 2910 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2911 iwpbl->pbl_allocated = false; 2912 } 2913 } 2914 2915 stag = irdma_create_stag(iwdev); 2916 if (!stag) { 2917 err = -ENOMEM; 2918 goto error; 2919 } 2920 2921 iwmr->stag = stag; 2922 iwmr->ibmr.rkey = stag; 2923 iwmr->ibmr.lkey = stag; 2924 err = irdma_hwreg_mr(iwdev, iwmr, access); 2925 if (err) { 2926 irdma_free_stag(iwdev, stag); 2927 goto error; 2928 } 2929 2930 break; 2931 default: 2932 goto error; 2933 } 2934 2935 iwmr->type = req.reg_type; 2936 2937 return &iwmr->ibmr; 2938 2939 error: 2940 if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 2941 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2942 ib_umem_release(region); 2943 kfree(iwmr); 2944 2945 return ERR_PTR(err); 2946 } 2947 2948 /** 2949 * irdma_reg_phys_mr - register kernel physical memory 2950 * @pd: ibpd pointer 2951 * @addr: physical address of memory to register 2952 * @size: size of memory to register 2953 * @access: Access rights 2954 * @iova_start: start of virtual address for physical buffers 2955 */ 2956 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 2957 u64 *iova_start) 2958 { 2959 struct irdma_device *iwdev = to_iwdev(pd->device); 2960 struct irdma_pbl *iwpbl; 2961 struct irdma_mr *iwmr; 2962 u32 stag; 2963 int ret; 2964 2965 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2966 if (!iwmr) 2967 return ERR_PTR(-ENOMEM); 2968 2969 iwmr->ibmr.pd = pd; 2970 iwmr->ibmr.device = pd->device; 2971 iwpbl = &iwmr->iwpbl; 2972 iwpbl->iwmr = iwmr; 2973 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2974 iwpbl->user_base = *iova_start; 2975 stag = irdma_create_stag(iwdev); 2976 if (!stag) { 2977 ret = -ENOMEM; 2978 goto err; 2979 } 2980 2981 iwmr->stag = stag; 2982 iwmr->ibmr.iova = *iova_start; 2983 iwmr->ibmr.rkey = stag; 2984 iwmr->ibmr.lkey = stag; 2985 iwmr->page_cnt = 1; 2986 iwmr->pgaddrmem[0] = addr; 2987 iwmr->len = size; 2988 iwmr->page_size = SZ_4K; 2989 ret = irdma_hwreg_mr(iwdev, iwmr, access); 2990 if (ret) { 2991 irdma_free_stag(iwdev, stag); 2992 goto err; 2993 } 2994 2995 return &iwmr->ibmr; 2996 2997 err: 2998 kfree(iwmr); 2999 3000 return ERR_PTR(ret); 3001 } 3002 3003 /** 3004 * irdma_get_dma_mr - register physical mem 3005 * @pd: ptr of pd 3006 * @acc: access for memory 3007 */ 3008 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 3009 { 3010 u64 kva = 0; 3011 3012 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); 3013 } 3014 3015 /** 3016 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 3017 * @iwmr: iwmr for IB's user page addresses 3018 * @ucontext: ptr to user context 3019 */ 3020 static void irdma_del_memlist(struct irdma_mr *iwmr, 3021 struct irdma_ucontext *ucontext) 3022 { 3023 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3024 unsigned long flags; 3025 3026 switch (iwmr->type) { 3027 case IRDMA_MEMREG_TYPE_CQ: 3028 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 3029 if (iwpbl->on_list) { 3030 iwpbl->on_list = false; 3031 list_del(&iwpbl->list); 3032 } 3033 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 3034 break; 3035 case IRDMA_MEMREG_TYPE_QP: 3036 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 3037 if (iwpbl->on_list) { 3038 iwpbl->on_list = false; 3039 list_del(&iwpbl->list); 3040 } 3041 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 3042 break; 3043 default: 3044 break; 3045 } 3046 } 3047 3048 /** 3049 * irdma_dereg_mr - deregister mr 3050 * @ib_mr: mr ptr for dereg 3051 * @udata: user data 3052 */ 3053 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3054 { 3055 struct ib_pd *ibpd = ib_mr->pd; 3056 struct irdma_pd *iwpd = to_iwpd(ibpd); 3057 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3058 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3059 struct irdma_dealloc_stag_info *info; 3060 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3061 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3062 struct irdma_cqp_request *cqp_request; 3063 struct cqp_cmds_info *cqp_info; 3064 int status; 3065 3066 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 3067 if (iwmr->region) { 3068 struct irdma_ucontext *ucontext; 3069 3070 ucontext = rdma_udata_to_drv_context(udata, 3071 struct irdma_ucontext, 3072 ibucontext); 3073 irdma_del_memlist(iwmr, ucontext); 3074 } 3075 goto done; 3076 } 3077 3078 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3079 if (!cqp_request) 3080 return -ENOMEM; 3081 3082 cqp_info = &cqp_request->info; 3083 info = &cqp_info->in.u.dealloc_stag.info; 3084 memset(info, 0, sizeof(*info)); 3085 info->pd_id = iwpd->sc_pd.pd_id; 3086 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3087 info->mr = true; 3088 if (iwpbl->pbl_allocated) 3089 info->dealloc_pbl = true; 3090 3091 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3092 cqp_info->post_sq = 1; 3093 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3094 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3095 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3096 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3097 if (status) 3098 return status; 3099 3100 irdma_free_stag(iwdev, iwmr->stag); 3101 done: 3102 if (iwpbl->pbl_allocated) 3103 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3104 ib_umem_release(iwmr->region); 3105 kfree(iwmr); 3106 3107 return 0; 3108 } 3109 3110 /** 3111 * irdma_post_send - kernel application wr 3112 * @ibqp: qp ptr for wr 3113 * @ib_wr: work request ptr 3114 * @bad_wr: return of bad wr if err 3115 */ 3116 static int irdma_post_send(struct ib_qp *ibqp, 3117 const struct ib_send_wr *ib_wr, 3118 const struct ib_send_wr **bad_wr) 3119 { 3120 struct irdma_qp *iwqp; 3121 struct irdma_qp_uk *ukqp; 3122 struct irdma_sc_dev *dev; 3123 struct irdma_post_sq_info info; 3124 int err = 0; 3125 unsigned long flags; 3126 bool inv_stag; 3127 struct irdma_ah *ah; 3128 3129 iwqp = to_iwqp(ibqp); 3130 ukqp = &iwqp->sc_qp.qp_uk; 3131 dev = &iwqp->iwdev->rf->sc_dev; 3132 3133 spin_lock_irqsave(&iwqp->lock, flags); 3134 while (ib_wr) { 3135 memset(&info, 0, sizeof(info)); 3136 inv_stag = false; 3137 info.wr_id = (ib_wr->wr_id); 3138 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 3139 info.signaled = true; 3140 if (ib_wr->send_flags & IB_SEND_FENCE) 3141 info.read_fence = true; 3142 switch (ib_wr->opcode) { 3143 case IB_WR_SEND_WITH_IMM: 3144 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 3145 info.imm_data_valid = true; 3146 info.imm_data = ntohl(ib_wr->ex.imm_data); 3147 } else { 3148 err = -EINVAL; 3149 break; 3150 } 3151 fallthrough; 3152 case IB_WR_SEND: 3153 case IB_WR_SEND_WITH_INV: 3154 if (ib_wr->opcode == IB_WR_SEND || 3155 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 3156 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3157 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 3158 else 3159 info.op_type = IRDMA_OP_TYPE_SEND; 3160 } else { 3161 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3162 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 3163 else 3164 info.op_type = IRDMA_OP_TYPE_SEND_INV; 3165 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 3166 } 3167 3168 if (ib_wr->send_flags & IB_SEND_INLINE) { 3169 info.op.inline_send.data = (void *)(unsigned long) 3170 ib_wr->sg_list[0].addr; 3171 info.op.inline_send.len = ib_wr->sg_list[0].length; 3172 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3173 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3174 ah = to_iwah(ud_wr(ib_wr)->ah); 3175 info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx; 3176 info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey; 3177 info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3178 } 3179 err = irdma_uk_inline_send(ukqp, &info, false); 3180 } else { 3181 info.op.send.num_sges = ib_wr->num_sge; 3182 info.op.send.sg_list = ib_wr->sg_list; 3183 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3184 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3185 ah = to_iwah(ud_wr(ib_wr)->ah); 3186 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 3187 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 3188 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3189 } 3190 err = irdma_uk_send(ukqp, &info, false); 3191 } 3192 break; 3193 case IB_WR_RDMA_WRITE_WITH_IMM: 3194 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 3195 info.imm_data_valid = true; 3196 info.imm_data = ntohl(ib_wr->ex.imm_data); 3197 } else { 3198 err = -EINVAL; 3199 break; 3200 } 3201 fallthrough; 3202 case IB_WR_RDMA_WRITE: 3203 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3204 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 3205 else 3206 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 3207 3208 if (ib_wr->send_flags & IB_SEND_INLINE) { 3209 info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr; 3210 info.op.inline_rdma_write.len = 3211 ib_wr->sg_list[0].length; 3212 info.op.inline_rdma_write.rem_addr.addr = 3213 rdma_wr(ib_wr)->remote_addr; 3214 info.op.inline_rdma_write.rem_addr.lkey = 3215 rdma_wr(ib_wr)->rkey; 3216 err = irdma_uk_inline_rdma_write(ukqp, &info, false); 3217 } else { 3218 info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; 3219 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 3220 info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 3221 info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 3222 err = irdma_uk_rdma_write(ukqp, &info, false); 3223 } 3224 break; 3225 case IB_WR_RDMA_READ_WITH_INV: 3226 inv_stag = true; 3227 fallthrough; 3228 case IB_WR_RDMA_READ: 3229 if (ib_wr->num_sge > 3230 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 3231 err = -EINVAL; 3232 break; 3233 } 3234 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 3235 info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 3236 info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 3237 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 3238 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 3239 err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 3240 break; 3241 case IB_WR_LOCAL_INV: 3242 info.op_type = IRDMA_OP_TYPE_INV_STAG; 3243 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 3244 err = irdma_uk_stag_local_invalidate(ukqp, &info, true); 3245 break; 3246 case IB_WR_REG_MR: { 3247 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 3248 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 3249 struct irdma_fast_reg_stag_info stag_info = {}; 3250 3251 stag_info.signaled = info.signaled; 3252 stag_info.read_fence = info.read_fence; 3253 stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); 3254 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 3255 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 3256 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 3257 stag_info.wr_id = ib_wr->wr_id; 3258 stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3259 stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; 3260 stag_info.total_len = iwmr->ibmr.length; 3261 stag_info.reg_addr_pa = *palloc->level1.addr; 3262 stag_info.first_pm_pbl_index = palloc->level1.idx; 3263 stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; 3264 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) 3265 stag_info.chunk_size = 1; 3266 err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, 3267 true); 3268 break; 3269 } 3270 default: 3271 err = -EINVAL; 3272 ibdev_dbg(&iwqp->iwdev->ibdev, 3273 "VERBS: upost_send bad opcode = 0x%x\n", 3274 ib_wr->opcode); 3275 break; 3276 } 3277 3278 if (err) 3279 break; 3280 ib_wr = ib_wr->next; 3281 } 3282 3283 if (!iwqp->flush_issued) { 3284 if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) 3285 irdma_uk_qp_post_wr(ukqp); 3286 spin_unlock_irqrestore(&iwqp->lock, flags); 3287 } else { 3288 spin_unlock_irqrestore(&iwqp->lock, flags); 3289 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 3290 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 3291 } 3292 if (err) 3293 *bad_wr = ib_wr; 3294 3295 return err; 3296 } 3297 3298 /** 3299 * irdma_post_recv - post receive wr for kernel application 3300 * @ibqp: ib qp pointer 3301 * @ib_wr: work request for receive 3302 * @bad_wr: bad wr caused an error 3303 */ 3304 static int irdma_post_recv(struct ib_qp *ibqp, 3305 const struct ib_recv_wr *ib_wr, 3306 const struct ib_recv_wr **bad_wr) 3307 { 3308 struct irdma_qp *iwqp; 3309 struct irdma_qp_uk *ukqp; 3310 struct irdma_post_rq_info post_recv = {}; 3311 unsigned long flags; 3312 int err = 0; 3313 3314 iwqp = to_iwqp(ibqp); 3315 ukqp = &iwqp->sc_qp.qp_uk; 3316 3317 spin_lock_irqsave(&iwqp->lock, flags); 3318 while (ib_wr) { 3319 post_recv.num_sges = ib_wr->num_sge; 3320 post_recv.wr_id = ib_wr->wr_id; 3321 post_recv.sg_list = ib_wr->sg_list; 3322 err = irdma_uk_post_receive(ukqp, &post_recv); 3323 if (err) { 3324 ibdev_dbg(&iwqp->iwdev->ibdev, 3325 "VERBS: post_recv err %d\n", err); 3326 goto out; 3327 } 3328 3329 ib_wr = ib_wr->next; 3330 } 3331 3332 out: 3333 spin_unlock_irqrestore(&iwqp->lock, flags); 3334 if (iwqp->flush_issued) 3335 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 3336 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 3337 3338 if (err) 3339 *bad_wr = ib_wr; 3340 3341 return err; 3342 } 3343 3344 /** 3345 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 3346 * @opcode: iwarp flush code 3347 */ 3348 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 3349 { 3350 switch (opcode) { 3351 case FLUSH_PROT_ERR: 3352 return IB_WC_LOC_PROT_ERR; 3353 case FLUSH_REM_ACCESS_ERR: 3354 return IB_WC_REM_ACCESS_ERR; 3355 case FLUSH_LOC_QP_OP_ERR: 3356 return IB_WC_LOC_QP_OP_ERR; 3357 case FLUSH_REM_OP_ERR: 3358 return IB_WC_REM_OP_ERR; 3359 case FLUSH_LOC_LEN_ERR: 3360 return IB_WC_LOC_LEN_ERR; 3361 case FLUSH_GENERAL_ERR: 3362 return IB_WC_WR_FLUSH_ERR; 3363 case FLUSH_RETRY_EXC_ERR: 3364 return IB_WC_RETRY_EXC_ERR; 3365 case FLUSH_MW_BIND_ERR: 3366 return IB_WC_MW_BIND_ERR; 3367 case FLUSH_REM_INV_REQ_ERR: 3368 return IB_WC_REM_INV_REQ_ERR; 3369 case FLUSH_FATAL_ERR: 3370 default: 3371 return IB_WC_FATAL_ERR; 3372 } 3373 } 3374 3375 /** 3376 * irdma_process_cqe - process cqe info 3377 * @entry: processed cqe 3378 * @cq_poll_info: cqe info 3379 */ 3380 static void irdma_process_cqe(struct ib_wc *entry, 3381 struct irdma_cq_poll_info *cq_poll_info) 3382 { 3383 struct irdma_qp *iwqp; 3384 struct irdma_sc_qp *qp; 3385 3386 entry->wc_flags = 0; 3387 entry->pkey_index = 0; 3388 entry->wr_id = cq_poll_info->wr_id; 3389 3390 qp = cq_poll_info->qp_handle; 3391 iwqp = qp->qp_uk.back_qp; 3392 entry->qp = qp->qp_uk.back_qp; 3393 3394 if (cq_poll_info->error) { 3395 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 3396 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 3397 3398 entry->vendor_err = cq_poll_info->major_err << 16 | 3399 cq_poll_info->minor_err; 3400 } else { 3401 entry->status = IB_WC_SUCCESS; 3402 if (cq_poll_info->imm_valid) { 3403 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 3404 entry->wc_flags |= IB_WC_WITH_IMM; 3405 } 3406 if (cq_poll_info->ud_smac_valid) { 3407 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 3408 entry->wc_flags |= IB_WC_WITH_SMAC; 3409 } 3410 3411 if (cq_poll_info->ud_vlan_valid) { 3412 u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK; 3413 3414 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 3415 if (vlan) { 3416 entry->vlan_id = vlan; 3417 entry->wc_flags |= IB_WC_WITH_VLAN; 3418 } 3419 } else { 3420 entry->sl = 0; 3421 } 3422 } 3423 3424 switch (cq_poll_info->op_type) { 3425 case IRDMA_OP_TYPE_RDMA_WRITE: 3426 case IRDMA_OP_TYPE_RDMA_WRITE_SOL: 3427 entry->opcode = IB_WC_RDMA_WRITE; 3428 break; 3429 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: 3430 case IRDMA_OP_TYPE_RDMA_READ: 3431 entry->opcode = IB_WC_RDMA_READ; 3432 break; 3433 case IRDMA_OP_TYPE_SEND_INV: 3434 case IRDMA_OP_TYPE_SEND_SOL: 3435 case IRDMA_OP_TYPE_SEND_SOL_INV: 3436 case IRDMA_OP_TYPE_SEND: 3437 entry->opcode = IB_WC_SEND; 3438 break; 3439 case IRDMA_OP_TYPE_FAST_REG_NSMR: 3440 entry->opcode = IB_WC_REG_MR; 3441 break; 3442 case IRDMA_OP_TYPE_INV_STAG: 3443 entry->opcode = IB_WC_LOCAL_INV; 3444 break; 3445 case IRDMA_OP_TYPE_REC_IMM: 3446 case IRDMA_OP_TYPE_REC: 3447 entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ? 3448 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; 3449 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 3450 cq_poll_info->stag_invalid_set) { 3451 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 3452 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 3453 } 3454 break; 3455 default: 3456 ibdev_err(&iwqp->iwdev->ibdev, 3457 "Invalid opcode = %d in CQE\n", cq_poll_info->op_type); 3458 entry->status = IB_WC_GENERAL_ERR; 3459 return; 3460 } 3461 3462 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 3463 entry->src_qp = cq_poll_info->ud_src_qpn; 3464 entry->slid = 0; 3465 entry->wc_flags |= 3466 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 3467 entry->network_hdr_type = cq_poll_info->ipv4 ? 3468 RDMA_NETWORK_IPV4 : 3469 RDMA_NETWORK_IPV6; 3470 } else { 3471 entry->src_qp = cq_poll_info->qp_id; 3472 } 3473 3474 entry->byte_len = cq_poll_info->bytes_xfered; 3475 } 3476 3477 /** 3478 * irdma_poll_one - poll one entry of the CQ 3479 * @ukcq: ukcq to poll 3480 * @cur_cqe: current CQE info to be filled in 3481 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 3482 * 3483 * Returns the internal irdma device error code or 0 on success 3484 */ 3485 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 3486 struct irdma_cq_poll_info *cur_cqe, 3487 struct ib_wc *entry) 3488 { 3489 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 3490 3491 if (ret) 3492 return ret; 3493 3494 irdma_process_cqe(entry, cur_cqe); 3495 3496 return 0; 3497 } 3498 3499 /** 3500 * __irdma_poll_cq - poll cq for completion (kernel apps) 3501 * @iwcq: cq to poll 3502 * @num_entries: number of entries to poll 3503 * @entry: wr of a completed entry 3504 */ 3505 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 3506 { 3507 struct list_head *tmp_node, *list_node; 3508 struct irdma_cq_buf *last_buf = NULL; 3509 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 3510 struct irdma_cq_buf *cq_buf; 3511 int ret; 3512 struct irdma_device *iwdev; 3513 struct irdma_cq_uk *ukcq; 3514 bool cq_new_cqe = false; 3515 int resized_bufs = 0; 3516 int npolled = 0; 3517 3518 iwdev = to_iwdev(iwcq->ibcq.device); 3519 ukcq = &iwcq->sc_cq.cq_uk; 3520 3521 /* go through the list of previously resized CQ buffers */ 3522 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 3523 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 3524 while (npolled < num_entries) { 3525 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 3526 if (!ret) { 3527 ++npolled; 3528 cq_new_cqe = true; 3529 continue; 3530 } 3531 if (ret == -ENOENT) 3532 break; 3533 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3534 if (ret == -EFAULT) { 3535 cq_new_cqe = true; 3536 continue; 3537 } 3538 goto error; 3539 } 3540 3541 /* save the resized CQ buffer which received the last cqe */ 3542 if (cq_new_cqe) 3543 last_buf = cq_buf; 3544 cq_new_cqe = false; 3545 } 3546 3547 /* check the current CQ for new cqes */ 3548 while (npolled < num_entries) { 3549 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 3550 if (ret == -ENOENT) { 3551 ret = irdma_generated_cmpls(iwcq, cur_cqe); 3552 if (!ret) 3553 irdma_process_cqe(entry + npolled, cur_cqe); 3554 } 3555 if (!ret) { 3556 ++npolled; 3557 cq_new_cqe = true; 3558 continue; 3559 } 3560 3561 if (ret == -ENOENT) 3562 break; 3563 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3564 if (ret == -EFAULT) { 3565 cq_new_cqe = true; 3566 continue; 3567 } 3568 goto error; 3569 } 3570 3571 if (cq_new_cqe) 3572 /* all previous CQ resizes are complete */ 3573 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 3574 else if (last_buf) 3575 /* only CQ resizes up to the last_buf are complete */ 3576 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 3577 if (resized_bufs) 3578 /* report to the HW the number of complete CQ resizes */ 3579 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 3580 3581 return npolled; 3582 error: 3583 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 3584 __func__, ret); 3585 3586 return ret; 3587 } 3588 3589 /** 3590 * irdma_poll_cq - poll cq for completion (kernel apps) 3591 * @ibcq: cq to poll 3592 * @num_entries: number of entries to poll 3593 * @entry: wr of a completed entry 3594 */ 3595 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 3596 struct ib_wc *entry) 3597 { 3598 struct irdma_cq *iwcq; 3599 unsigned long flags; 3600 int ret; 3601 3602 iwcq = to_iwcq(ibcq); 3603 3604 spin_lock_irqsave(&iwcq->lock, flags); 3605 ret = __irdma_poll_cq(iwcq, num_entries, entry); 3606 spin_unlock_irqrestore(&iwcq->lock, flags); 3607 3608 return ret; 3609 } 3610 3611 /** 3612 * irdma_req_notify_cq - arm cq kernel application 3613 * @ibcq: cq to arm 3614 * @notify_flags: notofication flags 3615 */ 3616 static int irdma_req_notify_cq(struct ib_cq *ibcq, 3617 enum ib_cq_notify_flags notify_flags) 3618 { 3619 struct irdma_cq *iwcq; 3620 struct irdma_cq_uk *ukcq; 3621 unsigned long flags; 3622 enum irdma_cmpl_notify cq_notify; 3623 bool promo_event = false; 3624 int ret = 0; 3625 3626 cq_notify = notify_flags == IB_CQ_SOLICITED ? 3627 IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT; 3628 iwcq = to_iwcq(ibcq); 3629 ukcq = &iwcq->sc_cq.cq_uk; 3630 3631 spin_lock_irqsave(&iwcq->lock, flags); 3632 /* Only promote to arm the CQ for any event if the last arm event was solicited. */ 3633 if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED) 3634 promo_event = true; 3635 3636 if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) { 3637 iwcq->last_notify = cq_notify; 3638 irdma_uk_cq_request_notification(ukcq, cq_notify); 3639 } 3640 3641 if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3642 (!irdma_cq_empty(iwcq) || !list_empty(&iwcq->cmpl_generated))) 3643 ret = 1; 3644 spin_unlock_irqrestore(&iwcq->lock, flags); 3645 3646 return ret; 3647 } 3648 3649 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 3650 struct ib_port_immutable *immutable) 3651 { 3652 struct ib_port_attr attr; 3653 int err; 3654 3655 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3656 err = ib_query_port(ibdev, port_num, &attr); 3657 if (err) 3658 return err; 3659 3660 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3661 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3662 immutable->gid_tbl_len = attr.gid_tbl_len; 3663 3664 return 0; 3665 } 3666 3667 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 3668 struct ib_port_immutable *immutable) 3669 { 3670 struct ib_port_attr attr; 3671 int err; 3672 3673 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 3674 err = ib_query_port(ibdev, port_num, &attr); 3675 if (err) 3676 return err; 3677 immutable->gid_tbl_len = attr.gid_tbl_len; 3678 3679 return 0; 3680 } 3681 3682 static const struct rdma_stat_desc irdma_hw_stat_descs[] = { 3683 /* 32bit names */ 3684 [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors", 3685 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards", 3686 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts", 3687 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes", 3688 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards", 3689 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts", 3690 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes", 3691 [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "tcpRetransSegs", 3692 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "tcpInOptErrors", 3693 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "tcpInProtoErrors", 3694 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled", 3695 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored", 3696 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent", 3697 3698 /* 64bit names */ 3699 [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3700 "ip4InOctets", 3701 [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3702 "ip4InPkts", 3703 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3704 "ip4InReasmRqd", 3705 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3706 "ip4InMcastOctets", 3707 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3708 "ip4InMcastPkts", 3709 [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3710 "ip4OutOctets", 3711 [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3712 "ip4OutPkts", 3713 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3714 "ip4OutSegRqd", 3715 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3716 "ip4OutMcastOctets", 3717 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3718 "ip4OutMcastPkts", 3719 [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3720 "ip6InOctets", 3721 [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3722 "ip6InPkts", 3723 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3724 "ip6InReasmRqd", 3725 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3726 "ip6InMcastOctets", 3727 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3728 "ip6InMcastPkts", 3729 [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3730 "ip6OutOctets", 3731 [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3732 "ip6OutPkts", 3733 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3734 "ip6OutSegRqd", 3735 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3736 "ip6OutMcastOctets", 3737 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3738 "ip6OutMcastPkts", 3739 [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3740 "tcpInSegs", 3741 [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32].name = 3742 "tcpOutSegs", 3743 [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3744 "iwInRdmaReads", 3745 [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3746 "iwInRdmaSends", 3747 [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3748 "iwInRdmaWrites", 3749 [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3750 "iwOutRdmaReads", 3751 [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3752 "iwOutRdmaSends", 3753 [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3754 "iwOutRdmaWrites", 3755 [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32].name = 3756 "iwRdmaBnd", 3757 [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32].name = 3758 "iwRdmaInv", 3759 [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3760 "RxUDP", 3761 [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3762 "TxUDP", 3763 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32] 3764 .name = "RxECNMrkd", 3765 }; 3766 3767 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 3768 { 3769 struct irdma_device *iwdev = to_iwdev(dev); 3770 3771 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 3772 irdma_fw_major_ver(&iwdev->rf->sc_dev), 3773 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 3774 } 3775 3776 /** 3777 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 3778 * @ibdev: device pointer from stack 3779 * @port_num: port number 3780 */ 3781 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 3782 u32 port_num) 3783 { 3784 int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 + 3785 IRDMA_HW_STAT_INDEX_MAX_64; 3786 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 3787 3788 BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_descs) != 3789 (IRDMA_HW_STAT_INDEX_MAX_32 + IRDMA_HW_STAT_INDEX_MAX_64)); 3790 3791 return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters, 3792 lifespan); 3793 } 3794 3795 /** 3796 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 3797 * @ibdev: device pointer from stack 3798 * @stats: stats pointer from stack 3799 * @port_num: port number 3800 * @index: which hw counter the stack is requesting we update 3801 */ 3802 static int irdma_get_hw_stats(struct ib_device *ibdev, 3803 struct rdma_hw_stats *stats, u32 port_num, 3804 int index) 3805 { 3806 struct irdma_device *iwdev = to_iwdev(ibdev); 3807 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 3808 3809 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 3810 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 3811 else 3812 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 3813 3814 memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); 3815 3816 return stats->num_counters; 3817 } 3818 3819 /** 3820 * irdma_query_gid - Query port GID 3821 * @ibdev: device pointer from stack 3822 * @port: port number 3823 * @index: Entry index 3824 * @gid: Global ID 3825 */ 3826 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 3827 union ib_gid *gid) 3828 { 3829 struct irdma_device *iwdev = to_iwdev(ibdev); 3830 3831 memset(gid->raw, 0, sizeof(gid->raw)); 3832 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 3833 3834 return 0; 3835 } 3836 3837 /** 3838 * mcast_list_add - Add a new mcast item to list 3839 * @rf: RDMA PCI function 3840 * @new_elem: pointer to element to add 3841 */ 3842 static void mcast_list_add(struct irdma_pci_f *rf, 3843 struct mc_table_list *new_elem) 3844 { 3845 list_add(&new_elem->list, &rf->mc_qht_list.list); 3846 } 3847 3848 /** 3849 * mcast_list_del - Remove an mcast item from list 3850 * @mc_qht_elem: pointer to mcast table list element 3851 */ 3852 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 3853 { 3854 if (mc_qht_elem) 3855 list_del(&mc_qht_elem->list); 3856 } 3857 3858 /** 3859 * mcast_list_lookup_ip - Search mcast list for address 3860 * @rf: RDMA PCI function 3861 * @ip_mcast: pointer to mcast IP address 3862 */ 3863 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 3864 u32 *ip_mcast) 3865 { 3866 struct mc_table_list *mc_qht_el; 3867 struct list_head *pos, *q; 3868 3869 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 3870 mc_qht_el = list_entry(pos, struct mc_table_list, list); 3871 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 3872 sizeof(mc_qht_el->mc_info.dest_ip))) 3873 return mc_qht_el; 3874 } 3875 3876 return NULL; 3877 } 3878 3879 /** 3880 * irdma_mcast_cqp_op - perform a mcast cqp operation 3881 * @iwdev: irdma device 3882 * @mc_grp_ctx: mcast group info 3883 * @op: operation 3884 * 3885 * returns error status 3886 */ 3887 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 3888 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 3889 { 3890 struct cqp_cmds_info *cqp_info; 3891 struct irdma_cqp_request *cqp_request; 3892 int status; 3893 3894 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3895 if (!cqp_request) 3896 return -ENOMEM; 3897 3898 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 3899 cqp_info = &cqp_request->info; 3900 cqp_info->cqp_cmd = op; 3901 cqp_info->post_sq = 1; 3902 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 3903 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 3904 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3905 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3906 3907 return status; 3908 } 3909 3910 /** 3911 * irdma_mcast_mac - Get the multicast MAC for an IP address 3912 * @ip_addr: IPv4 or IPv6 address 3913 * @mac: pointer to result MAC address 3914 * @ipv4: flag indicating IPv4 or IPv6 3915 * 3916 */ 3917 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 3918 { 3919 u8 *ip = (u8 *)ip_addr; 3920 3921 if (ipv4) { 3922 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 3923 0x00, 0x00}; 3924 3925 mac4[3] = ip[2] & 0x7F; 3926 mac4[4] = ip[1]; 3927 mac4[5] = ip[0]; 3928 ether_addr_copy(mac, mac4); 3929 } else { 3930 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 3931 0x00, 0x00}; 3932 3933 mac6[2] = ip[3]; 3934 mac6[3] = ip[2]; 3935 mac6[4] = ip[1]; 3936 mac6[5] = ip[0]; 3937 ether_addr_copy(mac, mac6); 3938 } 3939 } 3940 3941 /** 3942 * irdma_attach_mcast - attach a qp to a multicast group 3943 * @ibqp: ptr to qp 3944 * @ibgid: pointer to global ID 3945 * @lid: local ID 3946 * 3947 * returns error status 3948 */ 3949 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 3950 { 3951 struct irdma_qp *iwqp = to_iwqp(ibqp); 3952 struct irdma_device *iwdev = iwqp->iwdev; 3953 struct irdma_pci_f *rf = iwdev->rf; 3954 struct mc_table_list *mc_qht_elem; 3955 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 3956 unsigned long flags; 3957 u32 ip_addr[4] = {}; 3958 u32 mgn; 3959 u32 no_mgs; 3960 int ret = 0; 3961 bool ipv4; 3962 u16 vlan_id; 3963 union irdma_sockaddr sgid_addr; 3964 unsigned char dmac[ETH_ALEN]; 3965 3966 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 3967 3968 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 3969 irdma_copy_ip_ntohl(ip_addr, 3970 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 3971 irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); 3972 ipv4 = false; 3973 ibdev_dbg(&iwdev->ibdev, 3974 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 3975 ip_addr); 3976 irdma_mcast_mac(ip_addr, dmac, false); 3977 } else { 3978 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 3979 ipv4 = true; 3980 vlan_id = irdma_get_vlan_ipv4(ip_addr); 3981 irdma_mcast_mac(ip_addr, dmac, true); 3982 ibdev_dbg(&iwdev->ibdev, 3983 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 3984 ibqp->qp_num, ip_addr, dmac); 3985 } 3986 3987 spin_lock_irqsave(&rf->qh_list_lock, flags); 3988 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 3989 if (!mc_qht_elem) { 3990 struct irdma_dma_mem *dma_mem_mc; 3991 3992 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3993 mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); 3994 if (!mc_qht_elem) 3995 return -ENOMEM; 3996 3997 mc_qht_elem->mc_info.ipv4_valid = ipv4; 3998 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 3999 sizeof(mc_qht_elem->mc_info.dest_ip)); 4000 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 4001 &mgn, &rf->next_mcg); 4002 if (ret) { 4003 kfree(mc_qht_elem); 4004 return -ENOMEM; 4005 } 4006 4007 mc_qht_elem->mc_info.mgn = mgn; 4008 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 4009 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 4010 IRDMA_HW_PAGE_SIZE); 4011 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 4012 dma_mem_mc->size, 4013 &dma_mem_mc->pa, 4014 GFP_KERNEL); 4015 if (!dma_mem_mc->va) { 4016 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 4017 kfree(mc_qht_elem); 4018 return -ENOMEM; 4019 } 4020 4021 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 4022 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 4023 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 4024 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 4025 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 4026 if (vlan_id < VLAN_N_VID) 4027 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 4028 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id; 4029 mc_qht_elem->mc_grp_ctx.qs_handle = 4030 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 4031 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 4032 4033 spin_lock_irqsave(&rf->qh_list_lock, flags); 4034 mcast_list_add(rf, mc_qht_elem); 4035 } else { 4036 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 4037 IRDMA_MAX_MGS_PER_CTX) { 4038 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4039 return -ENOMEM; 4040 } 4041 } 4042 4043 mcg_info.qp_id = iwqp->ibqp.qp_num; 4044 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 4045 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4046 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4047 4048 /* Only if there is a change do we need to modify or create */ 4049 if (!no_mgs) { 4050 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4051 IRDMA_OP_MC_CREATE); 4052 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4053 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4054 IRDMA_OP_MC_MODIFY); 4055 } else { 4056 return 0; 4057 } 4058 4059 if (ret) 4060 goto error; 4061 4062 return 0; 4063 4064 error: 4065 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4066 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4067 mcast_list_del(mc_qht_elem); 4068 dma_free_coherent(rf->hw.device, 4069 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4070 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4071 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4072 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4073 irdma_free_rsrc(rf, rf->allocated_mcgs, 4074 mc_qht_elem->mc_grp_ctx.mg_id); 4075 kfree(mc_qht_elem); 4076 } 4077 4078 return ret; 4079 } 4080 4081 /** 4082 * irdma_detach_mcast - detach a qp from a multicast group 4083 * @ibqp: ptr to qp 4084 * @ibgid: pointer to global ID 4085 * @lid: local ID 4086 * 4087 * returns error status 4088 */ 4089 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4090 { 4091 struct irdma_qp *iwqp = to_iwqp(ibqp); 4092 struct irdma_device *iwdev = iwqp->iwdev; 4093 struct irdma_pci_f *rf = iwdev->rf; 4094 u32 ip_addr[4] = {}; 4095 struct mc_table_list *mc_qht_elem; 4096 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4097 int ret; 4098 unsigned long flags; 4099 union irdma_sockaddr sgid_addr; 4100 4101 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4102 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 4103 irdma_copy_ip_ntohl(ip_addr, 4104 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4105 else 4106 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4107 4108 spin_lock_irqsave(&rf->qh_list_lock, flags); 4109 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4110 if (!mc_qht_elem) { 4111 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4112 ibdev_dbg(&iwdev->ibdev, 4113 "VERBS: address not found MCG\n"); 4114 return 0; 4115 } 4116 4117 mcg_info.qp_id = iwqp->ibqp.qp_num; 4118 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4119 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4120 mcast_list_del(mc_qht_elem); 4121 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4122 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4123 IRDMA_OP_MC_DESTROY); 4124 if (ret) { 4125 ibdev_dbg(&iwdev->ibdev, 4126 "VERBS: failed MC_DESTROY MCG\n"); 4127 spin_lock_irqsave(&rf->qh_list_lock, flags); 4128 mcast_list_add(rf, mc_qht_elem); 4129 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4130 return -EAGAIN; 4131 } 4132 4133 dma_free_coherent(rf->hw.device, 4134 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4135 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4136 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4137 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4138 irdma_free_rsrc(rf, rf->allocated_mcgs, 4139 mc_qht_elem->mc_grp_ctx.mg_id); 4140 kfree(mc_qht_elem); 4141 } else { 4142 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4143 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4144 IRDMA_OP_MC_MODIFY); 4145 if (ret) { 4146 ibdev_dbg(&iwdev->ibdev, 4147 "VERBS: failed Modify MCG\n"); 4148 return ret; 4149 } 4150 } 4151 4152 return 0; 4153 } 4154 4155 static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep) 4156 { 4157 struct irdma_pci_f *rf = iwdev->rf; 4158 int err; 4159 4160 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah->sc_ah.ah_info.ah_idx, 4161 &rf->next_ah); 4162 if (err) 4163 return err; 4164 4165 err = irdma_ah_cqp_op(rf, &ah->sc_ah, IRDMA_OP_AH_CREATE, sleep, 4166 irdma_gsi_ud_qp_ah_cb, &ah->sc_ah); 4167 4168 if (err) { 4169 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail"); 4170 goto err_ah_create; 4171 } 4172 4173 if (!sleep) { 4174 int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; 4175 4176 do { 4177 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 4178 mdelay(1); 4179 } while (!ah->sc_ah.ah_info.ah_valid && --cnt); 4180 4181 if (!cnt) { 4182 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP create AH timed out"); 4183 err = -ETIMEDOUT; 4184 goto err_ah_create; 4185 } 4186 } 4187 return 0; 4188 4189 err_ah_create: 4190 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); 4191 4192 return err; 4193 } 4194 4195 static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr) 4196 { 4197 struct irdma_pd *pd = to_iwpd(ibah->pd); 4198 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4199 struct rdma_ah_attr *ah_attr = attr->ah_attr; 4200 const struct ib_gid_attr *sgid_attr; 4201 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4202 struct irdma_pci_f *rf = iwdev->rf; 4203 struct irdma_sc_ah *sc_ah; 4204 struct irdma_ah_info *ah_info; 4205 union irdma_sockaddr sgid_addr, dgid_addr; 4206 int err; 4207 u8 dmac[ETH_ALEN]; 4208 4209 ah->pd = pd; 4210 sc_ah = &ah->sc_ah; 4211 sc_ah->ah_info.vsi = &iwdev->vsi; 4212 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 4213 ah->sgid_index = ah_attr->grh.sgid_index; 4214 sgid_attr = ah_attr->grh.sgid_attr; 4215 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 4216 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 4217 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 4218 ah->av.attrs = *ah_attr; 4219 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 4220 ah_info = &sc_ah->ah_info; 4221 ah_info->pd_idx = pd->sc_pd.pd_id; 4222 if (ah_attr->ah_flags & IB_AH_GRH) { 4223 ah_info->flow_label = ah_attr->grh.flow_label; 4224 ah_info->hop_ttl = ah_attr->grh.hop_limit; 4225 ah_info->tc_tos = ah_attr->grh.traffic_class; 4226 } 4227 4228 ether_addr_copy(dmac, ah_attr->roce.dmac); 4229 if (ah->av.net_type == RDMA_NETWORK_IPV4) { 4230 ah_info->ipv4_valid = true; 4231 ah_info->dest_ip_addr[0] = 4232 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 4233 ah_info->src_ip_addr[0] = 4234 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4235 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 4236 ah_info->dest_ip_addr[0]); 4237 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 4238 ah_info->do_lpbk = true; 4239 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 4240 } 4241 } else { 4242 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 4243 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4244 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 4245 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4246 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 4247 ah_info->dest_ip_addr); 4248 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 4249 ah_info->do_lpbk = true; 4250 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 4251 } 4252 } 4253 4254 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 4255 ah_info->mac_addr); 4256 if (err) 4257 return err; 4258 4259 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 4260 ah_info->ipv4_valid, dmac); 4261 4262 if (ah_info->dst_arpindex == -1) 4263 return -EINVAL; 4264 4265 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 4266 ah_info->vlan_tag = 0; 4267 4268 if (ah_info->vlan_tag < VLAN_N_VID) { 4269 ah_info->insert_vlan_tag = true; 4270 ah_info->vlan_tag |= 4271 rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; 4272 } 4273 4274 return 0; 4275 } 4276 4277 /** 4278 * irdma_ah_exists - Check for existing identical AH 4279 * @iwdev: irdma device 4280 * @new_ah: AH to check for 4281 * 4282 * returns true if AH is found, false if not found. 4283 */ 4284 static bool irdma_ah_exists(struct irdma_device *iwdev, 4285 struct irdma_ah *new_ah) 4286 { 4287 struct irdma_ah *ah; 4288 u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 4289 new_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 4290 new_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 4291 new_ah->sc_ah.ah_info.dest_ip_addr[3]; 4292 4293 hash_for_each_possible(iwdev->ah_hash_tbl, ah, list, key) { 4294 /* Set ah_valid and ah_id the same so memcmp can work */ 4295 new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx; 4296 new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid; 4297 if (!memcmp(&ah->sc_ah.ah_info, &new_ah->sc_ah.ah_info, 4298 sizeof(ah->sc_ah.ah_info))) { 4299 refcount_inc(&ah->refcnt); 4300 new_ah->parent_ah = ah; 4301 return true; 4302 } 4303 } 4304 4305 return false; 4306 } 4307 4308 /** 4309 * irdma_destroy_ah - Destroy address handle 4310 * @ibah: pointer to address handle 4311 * @ah_flags: flags for sleepable 4312 */ 4313 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 4314 { 4315 struct irdma_device *iwdev = to_iwdev(ibah->device); 4316 struct irdma_ah *ah = to_iwah(ibah); 4317 4318 if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) { 4319 mutex_lock(&iwdev->ah_tbl_lock); 4320 if (!refcount_dec_and_test(&ah->parent_ah->refcnt)) { 4321 mutex_unlock(&iwdev->ah_tbl_lock); 4322 return 0; 4323 } 4324 hash_del(&ah->parent_ah->list); 4325 kfree(ah->parent_ah); 4326 mutex_unlock(&iwdev->ah_tbl_lock); 4327 } 4328 4329 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 4330 false, NULL, ah); 4331 4332 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 4333 ah->sc_ah.ah_info.ah_idx); 4334 4335 return 0; 4336 } 4337 4338 /** 4339 * irdma_create_user_ah - create user address handle 4340 * @ibah: address handle 4341 * @attr: address handle attributes 4342 * @udata: User data 4343 * 4344 * returns 0 on success, error otherwise 4345 */ 4346 static int irdma_create_user_ah(struct ib_ah *ibah, 4347 struct rdma_ah_init_attr *attr, 4348 struct ib_udata *udata) 4349 { 4350 #define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) 4351 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4352 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4353 struct irdma_create_ah_resp uresp; 4354 struct irdma_ah *parent_ah; 4355 int err; 4356 4357 if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) 4358 return -EINVAL; 4359 4360 err = irdma_setup_ah(ibah, attr); 4361 if (err) 4362 return err; 4363 mutex_lock(&iwdev->ah_tbl_lock); 4364 if (!irdma_ah_exists(iwdev, ah)) { 4365 err = irdma_create_hw_ah(iwdev, ah, true); 4366 if (err) { 4367 mutex_unlock(&iwdev->ah_tbl_lock); 4368 return err; 4369 } 4370 /* Add new AH to list */ 4371 parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL); 4372 if (parent_ah) { 4373 u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 4374 parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 4375 parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 4376 parent_ah->sc_ah.ah_info.dest_ip_addr[3]; 4377 4378 ah->parent_ah = parent_ah; 4379 hash_add(iwdev->ah_hash_tbl, &parent_ah->list, key); 4380 refcount_set(&parent_ah->refcnt, 1); 4381 } 4382 } 4383 mutex_unlock(&iwdev->ah_tbl_lock); 4384 4385 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 4386 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); 4387 if (err) 4388 irdma_destroy_ah(ibah, attr->flags); 4389 4390 return err; 4391 } 4392 4393 /** 4394 * irdma_create_ah - create address handle 4395 * @ibah: address handle 4396 * @attr: address handle attributes 4397 * @udata: NULL 4398 * 4399 * returns 0 on success, error otherwise 4400 */ 4401 static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, 4402 struct ib_udata *udata) 4403 { 4404 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4405 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4406 int err; 4407 4408 err = irdma_setup_ah(ibah, attr); 4409 if (err) 4410 return err; 4411 err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE); 4412 4413 return err; 4414 } 4415 4416 /** 4417 * irdma_query_ah - Query address handle 4418 * @ibah: pointer to address handle 4419 * @ah_attr: address handle attributes 4420 */ 4421 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 4422 { 4423 struct irdma_ah *ah = to_iwah(ibah); 4424 4425 memset(ah_attr, 0, sizeof(*ah_attr)); 4426 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 4427 ah_attr->ah_flags = IB_AH_GRH; 4428 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 4429 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 4430 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 4431 ah_attr->grh.sgid_index = ah->sgid_index; 4432 ah_attr->grh.sgid_index = ah->sgid_index; 4433 memcpy(&ah_attr->grh.dgid, &ah->dgid, 4434 sizeof(ah_attr->grh.dgid)); 4435 } 4436 4437 return 0; 4438 } 4439 4440 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 4441 u32 port_num) 4442 { 4443 return IB_LINK_LAYER_ETHERNET; 4444 } 4445 4446 static const struct ib_device_ops irdma_roce_dev_ops = { 4447 .attach_mcast = irdma_attach_mcast, 4448 .create_ah = irdma_create_ah, 4449 .create_user_ah = irdma_create_user_ah, 4450 .destroy_ah = irdma_destroy_ah, 4451 .detach_mcast = irdma_detach_mcast, 4452 .get_link_layer = irdma_get_link_layer, 4453 .get_port_immutable = irdma_roce_port_immutable, 4454 .modify_qp = irdma_modify_qp_roce, 4455 .query_ah = irdma_query_ah, 4456 .query_pkey = irdma_query_pkey, 4457 }; 4458 4459 static const struct ib_device_ops irdma_iw_dev_ops = { 4460 .modify_qp = irdma_modify_qp, 4461 .get_port_immutable = irdma_iw_port_immutable, 4462 .query_gid = irdma_query_gid, 4463 }; 4464 4465 static const struct ib_device_ops irdma_dev_ops = { 4466 .owner = THIS_MODULE, 4467 .driver_id = RDMA_DRIVER_IRDMA, 4468 .uverbs_abi_ver = IRDMA_ABI_VER, 4469 4470 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 4471 .alloc_mr = irdma_alloc_mr, 4472 .alloc_mw = irdma_alloc_mw, 4473 .alloc_pd = irdma_alloc_pd, 4474 .alloc_ucontext = irdma_alloc_ucontext, 4475 .create_cq = irdma_create_cq, 4476 .create_qp = irdma_create_qp, 4477 .dealloc_driver = irdma_ib_dealloc_device, 4478 .dealloc_mw = irdma_dealloc_mw, 4479 .dealloc_pd = irdma_dealloc_pd, 4480 .dealloc_ucontext = irdma_dealloc_ucontext, 4481 .dereg_mr = irdma_dereg_mr, 4482 .destroy_cq = irdma_destroy_cq, 4483 .destroy_qp = irdma_destroy_qp, 4484 .disassociate_ucontext = irdma_disassociate_ucontext, 4485 .get_dev_fw_str = irdma_get_dev_fw_str, 4486 .get_dma_mr = irdma_get_dma_mr, 4487 .get_hw_stats = irdma_get_hw_stats, 4488 .map_mr_sg = irdma_map_mr_sg, 4489 .mmap = irdma_mmap, 4490 .mmap_free = irdma_mmap_free, 4491 .poll_cq = irdma_poll_cq, 4492 .post_recv = irdma_post_recv, 4493 .post_send = irdma_post_send, 4494 .query_device = irdma_query_device, 4495 .query_port = irdma_query_port, 4496 .query_qp = irdma_query_qp, 4497 .reg_user_mr = irdma_reg_user_mr, 4498 .req_notify_cq = irdma_req_notify_cq, 4499 .resize_cq = irdma_resize_cq, 4500 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 4501 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 4502 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 4503 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 4504 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 4505 INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), 4506 }; 4507 4508 /** 4509 * irdma_init_roce_device - initialization of roce rdma device 4510 * @iwdev: irdma device 4511 */ 4512 static void irdma_init_roce_device(struct irdma_device *iwdev) 4513 { 4514 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 4515 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 4516 iwdev->netdev->dev_addr); 4517 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 4518 } 4519 4520 /** 4521 * irdma_init_iw_device - initialization of iwarp rdma device 4522 * @iwdev: irdma device 4523 */ 4524 static int irdma_init_iw_device(struct irdma_device *iwdev) 4525 { 4526 struct net_device *netdev = iwdev->netdev; 4527 4528 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 4529 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 4530 netdev->dev_addr); 4531 iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref; 4532 iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref; 4533 iwdev->ibdev.ops.iw_get_qp = irdma_get_qp; 4534 iwdev->ibdev.ops.iw_connect = irdma_connect; 4535 iwdev->ibdev.ops.iw_accept = irdma_accept; 4536 iwdev->ibdev.ops.iw_reject = irdma_reject; 4537 iwdev->ibdev.ops.iw_create_listen = irdma_create_listen; 4538 iwdev->ibdev.ops.iw_destroy_listen = irdma_destroy_listen; 4539 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 4540 sizeof(iwdev->ibdev.iw_ifname)); 4541 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 4542 4543 return 0; 4544 } 4545 4546 /** 4547 * irdma_init_rdma_device - initialization of rdma device 4548 * @iwdev: irdma device 4549 */ 4550 static int irdma_init_rdma_device(struct irdma_device *iwdev) 4551 { 4552 struct pci_dev *pcidev = iwdev->rf->pcidev; 4553 int ret; 4554 4555 if (iwdev->roce_mode) { 4556 irdma_init_roce_device(iwdev); 4557 } else { 4558 ret = irdma_init_iw_device(iwdev); 4559 if (ret) 4560 return ret; 4561 } 4562 iwdev->ibdev.phys_port_cnt = 1; 4563 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 4564 iwdev->ibdev.dev.parent = &pcidev->dev; 4565 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 4566 4567 return 0; 4568 } 4569 4570 /** 4571 * irdma_port_ibevent - indicate port event 4572 * @iwdev: irdma device 4573 */ 4574 void irdma_port_ibevent(struct irdma_device *iwdev) 4575 { 4576 struct ib_event event; 4577 4578 event.device = &iwdev->ibdev; 4579 event.element.port_num = 1; 4580 event.event = 4581 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 4582 ib_dispatch_event(&event); 4583 } 4584 4585 /** 4586 * irdma_ib_unregister_device - unregister rdma device from IB 4587 * core 4588 * @iwdev: irdma device 4589 */ 4590 void irdma_ib_unregister_device(struct irdma_device *iwdev) 4591 { 4592 iwdev->iw_status = 0; 4593 irdma_port_ibevent(iwdev); 4594 ib_unregister_device(&iwdev->ibdev); 4595 } 4596 4597 /** 4598 * irdma_ib_register_device - register irdma device to IB core 4599 * @iwdev: irdma device 4600 */ 4601 int irdma_ib_register_device(struct irdma_device *iwdev) 4602 { 4603 int ret; 4604 4605 ret = irdma_init_rdma_device(iwdev); 4606 if (ret) 4607 return ret; 4608 4609 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 4610 if (ret) 4611 goto error; 4612 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 4613 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 4614 if (ret) 4615 goto error; 4616 4617 iwdev->iw_status = 1; 4618 irdma_port_ibevent(iwdev); 4619 4620 return 0; 4621 4622 error: 4623 if (ret) 4624 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 4625 4626 return ret; 4627 } 4628 4629 /** 4630 * irdma_ib_dealloc_device 4631 * @ibdev: ib device 4632 * 4633 * callback from ibdev dealloc_driver to deallocate resources 4634 * unber irdma device 4635 */ 4636 void irdma_ib_dealloc_device(struct ib_device *ibdev) 4637 { 4638 struct irdma_device *iwdev = to_iwdev(ibdev); 4639 4640 irdma_rt_deinit_hw(iwdev); 4641 irdma_ctrl_deinit_hw(iwdev->rf); 4642 kfree(iwdev->rf); 4643 } 4644