1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2018 - 2022 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 /*$FreeBSD$*/ 35 36 #include "irdma_main.h" 37 38 void 39 irdma_get_dev_fw_str(struct ib_device *dev, 40 char *str, 41 size_t str_len) 42 { 43 struct irdma_device *iwdev = to_iwdev(dev); 44 45 snprintf(str, str_len, "%u.%u", 46 irdma_fw_major_ver(&iwdev->rf->sc_dev), 47 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 48 } 49 50 int 51 irdma_add_gid(struct ib_device *device, 52 u8 port_num, 53 unsigned int index, 54 const union ib_gid *gid, 55 const struct ib_gid_attr *attr, 56 void **context) 57 { 58 return 0; 59 } 60 61 int 62 irdma_del_gid(struct ib_device *device, 63 u8 port_num, 64 unsigned int index, 65 void **context) 66 { 67 return 0; 68 } 69 70 /** 71 * irdma_alloc_mr - register stag for fast memory registration 72 * @pd: ibpd pointer 73 * @mr_type: memory for stag registrion 74 * @max_num_sg: man number of pages 75 * @udata: user data 76 */ 77 struct ib_mr * 78 irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 79 u32 max_num_sg, struct ib_udata *udata) 80 { 81 struct irdma_device *iwdev = to_iwdev(pd->device); 82 struct irdma_pble_alloc *palloc; 83 struct irdma_pbl *iwpbl; 84 struct irdma_mr *iwmr; 85 int status; 86 u32 stag; 87 int err_code = -ENOMEM; 88 89 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 90 if (!iwmr) 91 return ERR_PTR(-ENOMEM); 92 93 stag = irdma_create_stag(iwdev); 94 if (!stag) { 95 err_code = -ENOMEM; 96 goto err; 97 } 98 99 iwmr->stag = stag; 100 iwmr->ibmr.rkey = stag; 101 iwmr->ibmr.lkey = stag; 102 iwmr->ibmr.pd = pd; 103 iwmr->ibmr.device = pd->device; 104 iwpbl = &iwmr->iwpbl; 105 iwpbl->iwmr = iwmr; 106 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 107 palloc = &iwpbl->pble_alloc; 108 iwmr->page_cnt = max_num_sg; 109 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 110 true); 111 if (status) 112 goto err_get_pble; 113 114 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 115 if (err_code) 116 goto err_alloc_stag; 117 118 iwpbl->pbl_allocated = true; 119 120 return &iwmr->ibmr; 121 err_alloc_stag: 122 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 123 err_get_pble: 124 irdma_free_stag(iwdev, stag); 125 err: 126 kfree(iwmr); 127 128 return ERR_PTR(err_code); 129 } 130 131 /** 132 * irdma_alloc_ucontext - Allocate the user context data structure 133 * @uctx: context 134 * @udata: user data 135 * 136 * This keeps track of all objects associated with a particular 137 * user-mode client. 138 */ 139 int 140 irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 141 { 142 struct ib_device *ibdev = uctx->device; 143 struct irdma_device *iwdev = to_iwdev(ibdev); 144 struct irdma_alloc_ucontext_req req; 145 struct irdma_alloc_ucontext_resp uresp = {0}; 146 struct irdma_ucontext *ucontext = to_ucontext(uctx); 147 struct irdma_uk_attrs *uk_attrs; 148 149 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 150 return -EINVAL; 151 152 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 153 goto ver_error; 154 155 ucontext->iwdev = iwdev; 156 ucontext->abi_ver = req.userspace_ver; 157 158 uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 159 /* GEN_1 support for libi40iw */ 160 if (udata->outlen < sizeof(uresp)) { 161 if (uk_attrs->hw_rev != IRDMA_GEN_1) 162 return -EOPNOTSUPP; 163 164 ucontext->legacy_mode = true; 165 uresp.max_qps = iwdev->rf->max_qp; 166 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 167 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 168 uresp.kernel_ver = req.userspace_ver; 169 if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) 170 return -EFAULT; 171 } else { 172 u64 bar_off = 173 (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 174 ucontext->db_mmap_entry = 175 irdma_user_mmap_entry_insert(ucontext, bar_off, 176 IRDMA_MMAP_IO_NC, 177 &uresp.db_mmap_key); 178 if (!ucontext->db_mmap_entry) { 179 return -ENOMEM; 180 } 181 uresp.kernel_ver = IRDMA_ABI_VER; 182 uresp.feature_flags = uk_attrs->feature_flags; 183 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 184 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 185 uresp.max_hw_inline = uk_attrs->max_hw_inline; 186 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 187 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 188 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 189 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 190 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 191 uresp.hw_rev = uk_attrs->hw_rev; 192 if (ib_copy_to_udata(udata, &uresp, 193 min(sizeof(uresp), udata->outlen))) { 194 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 195 return -EFAULT; 196 } 197 } 198 199 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 200 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 201 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 202 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 203 INIT_LIST_HEAD(&ucontext->vma_list); 204 mutex_init(&ucontext->vma_list_mutex); 205 206 return 0; 207 208 ver_error: 209 irdma_dev_err(&iwdev->rf->sc_dev, 210 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 211 req.userspace_ver, IRDMA_ABI_VER); 212 return -EINVAL; 213 } 214 215 /** 216 * irdma_dealloc_ucontext - deallocate the user context data structure 217 * @context: user context created during alloc 218 */ 219 void 220 irdma_dealloc_ucontext(struct ib_ucontext *context) 221 { 222 struct irdma_ucontext *ucontext = to_ucontext(context); 223 224 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 225 226 return; 227 } 228 229 /** 230 * irdma_alloc_pd - allocate protection domain 231 * @pd: protection domain 232 * @udata: user data 233 */ 234 int 235 irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 236 { 237 struct irdma_pd *iwpd = to_iwpd(pd); 238 struct irdma_device *iwdev = to_iwdev(pd->device); 239 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 240 struct irdma_pci_f *rf = iwdev->rf; 241 struct irdma_alloc_pd_resp uresp = {0}; 242 struct irdma_sc_pd *sc_pd; 243 u32 pd_id = 0; 244 int err; 245 246 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 247 &rf->next_pd); 248 if (err) 249 return err; 250 251 sc_pd = &iwpd->sc_pd; 252 if (udata) { 253 struct irdma_ucontext *ucontext = 254 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 255 ibucontext); 256 257 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 258 uresp.pd_id = pd_id; 259 if (ib_copy_to_udata(udata, &uresp, 260 min(sizeof(uresp), udata->outlen))) { 261 err = -EFAULT; 262 goto error; 263 } 264 } else { 265 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 266 } 267 268 return 0; 269 270 error: 271 272 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 273 274 return err; 275 } 276 277 void 278 irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 279 { 280 struct irdma_pd *iwpd = to_iwpd(ibpd); 281 struct irdma_device *iwdev = to_iwdev(ibpd->device); 282 283 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 284 } 285 286 static void 287 irdma_fill_ah_info(struct vnet *vnet, 288 struct irdma_ah_info *ah_info, 289 const struct ib_gid_attr *sgid_attr, 290 struct sockaddr *sgid_addr, struct sockaddr *dgid_addr, 291 u8 *dmac, u8 net_type) 292 { 293 if (net_type == RDMA_NETWORK_IPV4) { 294 ah_info->ipv4_valid = true; 295 ah_info->dest_ip_addr[0] = 296 ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr); 297 ah_info->src_ip_addr[0] = 298 ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr); 299 ah_info->do_lpbk = irdma_ipv4_is_lpb(vnet, 300 ah_info->src_ip_addr[0], 301 ah_info->dest_ip_addr[0]); 302 if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) { 303 irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); 304 } 305 } else { 306 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 307 ((struct sockaddr_in6 *)dgid_addr)->sin6_addr.__u6_addr.__u6_addr32); 308 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 309 ((struct sockaddr_in6 *)sgid_addr)->sin6_addr.__u6_addr.__u6_addr32); 310 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 311 ah_info->dest_ip_addr); 312 if (rdma_is_multicast_addr(&((struct sockaddr_in6 *)dgid_addr)->sin6_addr)) { 313 irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac); 314 } 315 } 316 } 317 318 static int 319 irdma_create_ah_vlan_tag(struct irdma_device *iwdev, 320 struct irdma_ah_info *ah_info, 321 const struct ib_gid_attr *sgid_attr, 322 u8 *dmac) 323 { 324 if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev)) 325 ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev); 326 else 327 ah_info->vlan_tag = VLAN_N_VID; 328 329 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, dmac); 330 331 if (ah_info->dst_arpindex == -1) 332 return -EINVAL; 333 334 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 335 ah_info->vlan_tag = 0; 336 337 if (ah_info->vlan_tag < VLAN_N_VID) { 338 ah_info->insert_vlan_tag = true; 339 ah_info->vlan_tag |= 340 rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; 341 } 342 return 0; 343 } 344 345 static int 346 irdma_create_ah_wait(struct irdma_pci_f *rf, 347 struct irdma_sc_ah *sc_ah, bool sleep) 348 { 349 if (!sleep) { 350 int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; 351 352 do { 353 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 354 mdelay(1); 355 } while (!sc_ah->ah_info.ah_valid && --cnt); 356 357 if (!cnt) 358 return -ETIMEDOUT; 359 } 360 return 0; 361 } 362 363 /** 364 * irdma_create_ah - create address handle 365 * @ib_ah: ptr to AH 366 * @attr: address handle attributes 367 * @flags: AH flags to wait 368 * @udata: user data 369 * 370 * returns 0 on success, error otherwise 371 */ 372 int 373 irdma_create_ah(struct ib_ah *ib_ah, 374 struct ib_ah_attr *attr, u32 flags, 375 struct ib_udata *udata) 376 { 377 struct irdma_pd *pd = to_iwpd(ib_ah->pd); 378 struct irdma_ah *ah = container_of(ib_ah, struct irdma_ah, ibah); 379 struct irdma_device *iwdev = to_iwdev(ib_ah->pd->device); 380 union ib_gid sgid; 381 struct ib_gid_attr sgid_attr; 382 struct irdma_pci_f *rf = iwdev->rf; 383 struct irdma_sc_ah *sc_ah; 384 u32 ah_id = 0; 385 struct irdma_ah_info *ah_info; 386 struct irdma_create_ah_resp uresp; 387 union { 388 struct sockaddr saddr; 389 struct sockaddr_in saddr_in; 390 struct sockaddr_in6 saddr_in6; 391 } sgid_addr, dgid_addr; 392 int err; 393 u8 dmac[ETH_ALEN]; 394 bool sleep; 395 396 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, 397 rf->max_ah, &ah_id, &rf->next_ah); 398 399 if (err) 400 return err; 401 402 ah->pd = pd; 403 sc_ah = &ah->sc_ah; 404 sc_ah->ah_info.ah_idx = ah_id; 405 sc_ah->ah_info.vsi = &iwdev->vsi; 406 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 407 ah->sgid_index = attr->grh.sgid_index; 408 memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid)); 409 rcu_read_lock(); 410 err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num, 411 attr->grh.sgid_index, &sgid, &sgid_attr); 412 rcu_read_unlock(); 413 if (err) { 414 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 415 "GID lookup at idx=%d with port=%d failed\n", 416 attr->grh.sgid_index, attr->port_num); 417 err = -EINVAL; 418 goto error; 419 } 420 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); 421 rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); 422 ah->av.attrs = *attr; 423 ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, 424 sgid_attr.gid_type, 425 &sgid); 426 427 if (sgid_attr.ndev) 428 dev_put(sgid_attr.ndev); 429 430 ah->av.sgid_addr.saddr = sgid_addr.saddr; 431 ah->av.dgid_addr.saddr = dgid_addr.saddr; 432 ah_info = &sc_ah->ah_info; 433 ah_info->ah_idx = ah_id; 434 ah_info->pd_idx = pd->sc_pd.pd_id; 435 ether_addr_copy(ah_info->mac_addr, IF_LLADDR(iwdev->netdev)); 436 437 if (attr->ah_flags & IB_AH_GRH) { 438 ah_info->flow_label = attr->grh.flow_label; 439 ah_info->hop_ttl = attr->grh.hop_limit; 440 ah_info->tc_tos = attr->grh.traffic_class; 441 } 442 443 ether_addr_copy(dmac, attr->dmac); 444 445 irdma_fill_ah_info(iwdev->netdev->if_vnet, 446 ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, 447 dmac, ah->av.net_type); 448 449 err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac); 450 if (err) 451 goto error; 452 453 sleep = flags & RDMA_CREATE_AH_SLEEPABLE; 454 455 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, 456 sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); 457 if (err) { 458 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 459 "CQP-OP Create AH fail"); 460 goto error; 461 } 462 463 err = irdma_create_ah_wait(rf, sc_ah, sleep); 464 if (err) { 465 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 466 "CQP create AH timed out"); 467 goto error; 468 } 469 470 if (udata) { 471 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 472 err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 473 } 474 return 0; 475 error: 476 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); 477 return err; 478 } 479 480 void 481 irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr) 482 { 483 ether_addr_copy(dmac, attr->dmac); 484 } 485 486 int 487 irdma_create_ah_stub(struct ib_ah *ib_ah, 488 struct ib_ah_attr *attr, u32 flags, 489 struct ib_udata *udata) 490 { 491 return -ENOSYS; 492 } 493 494 void 495 irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags) 496 { 497 return; 498 } 499 500 /** 501 * irdma_free_qp_rsrc - free up memory resources for qp 502 * @iwqp: qp ptr (user or kernel) 503 */ 504 void 505 irdma_free_qp_rsrc(struct irdma_qp *iwqp) 506 { 507 struct irdma_device *iwdev = iwqp->iwdev; 508 struct irdma_pci_f *rf = iwdev->rf; 509 u32 qp_num = iwqp->ibqp.qp_num; 510 511 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp); 512 irdma_dealloc_push_page(rf, &iwqp->sc_qp); 513 if (iwqp->sc_qp.vsi) { 514 irdma_qp_rem_qos(&iwqp->sc_qp); 515 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, 516 iwqp->sc_qp.user_pri); 517 } 518 519 if (qp_num > 2) 520 irdma_free_rsrc(rf, rf->allocated_qps, qp_num); 521 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem); 522 irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem); 523 kfree(iwqp->kqp.sig_trk_mem); 524 iwqp->kqp.sig_trk_mem = NULL; 525 kfree(iwqp->kqp.sq_wrid_mem); 526 kfree(iwqp->kqp.rq_wrid_mem); 527 kfree(iwqp->sg_list); 528 kfree(iwqp); 529 } 530 531 /** 532 * irdma_create_qp - create qp 533 * @ibpd: ptr of pd 534 * @init_attr: attributes for qp 535 * @udata: user data for create qp 536 */ 537 struct ib_qp * 538 irdma_create_qp(struct ib_pd *ibpd, 539 struct ib_qp_init_attr *init_attr, 540 struct ib_udata *udata) 541 { 542 struct irdma_pd *iwpd = to_iwpd(ibpd); 543 struct irdma_device *iwdev = to_iwdev(ibpd->device); 544 struct irdma_pci_f *rf = iwdev->rf; 545 struct irdma_qp *iwqp; 546 struct irdma_create_qp_req req; 547 struct irdma_create_qp_resp uresp = {0}; 548 u32 qp_num = 0; 549 int ret; 550 int err_code; 551 int sq_size; 552 int rq_size; 553 struct irdma_sc_qp *qp; 554 struct irdma_sc_dev *dev = &rf->sc_dev; 555 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 556 struct irdma_qp_init_info init_info = {{0}}; 557 struct irdma_qp_host_ctx_info *ctx_info; 558 unsigned long flags; 559 560 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 561 if (err_code) 562 return ERR_PTR(err_code); 563 564 sq_size = init_attr->cap.max_send_wr; 565 rq_size = init_attr->cap.max_recv_wr; 566 567 init_info.vsi = &iwdev->vsi; 568 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 569 init_info.qp_uk_init_info.sq_size = sq_size; 570 init_info.qp_uk_init_info.rq_size = rq_size; 571 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 572 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 573 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 574 575 iwqp = kzalloc(sizeof(*iwqp), GFP_KERNEL); 576 if (!iwqp) 577 return ERR_PTR(-ENOMEM); 578 579 iwqp->sg_list = kcalloc(uk_attrs->max_hw_wq_frags, sizeof(*iwqp->sg_list), 580 GFP_KERNEL); 581 if (!iwqp->sg_list) { 582 kfree(iwqp); 583 return ERR_PTR(-ENOMEM); 584 } 585 586 qp = &iwqp->sc_qp; 587 qp->qp_uk.back_qp = iwqp; 588 qp->qp_uk.lock = &iwqp->lock; 589 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 590 591 iwqp->iwdev = iwdev; 592 iwqp->q2_ctx_mem.size = IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE; 593 iwqp->q2_ctx_mem.va = irdma_allocate_dma_mem(dev->hw, &iwqp->q2_ctx_mem, 594 iwqp->q2_ctx_mem.size, 595 256); 596 if (!iwqp->q2_ctx_mem.va) { 597 kfree(iwqp->sg_list); 598 kfree(iwqp); 599 return ERR_PTR(-ENOMEM); 600 } 601 602 init_info.q2 = iwqp->q2_ctx_mem.va; 603 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 604 init_info.host_ctx = (__le64 *) (init_info.q2 + IRDMA_Q2_BUF_SIZE); 605 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 606 607 if (init_attr->qp_type == IB_QPT_GSI) 608 qp_num = 1; 609 else 610 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 611 &qp_num, &rf->next_qp); 612 if (err_code) 613 goto error; 614 615 iwqp->iwpd = iwpd; 616 iwqp->ibqp.qp_num = qp_num; 617 qp = &iwqp->sc_qp; 618 iwqp->iwscq = to_iwcq(init_attr->send_cq); 619 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 620 iwqp->host_ctx.va = init_info.host_ctx; 621 iwqp->host_ctx.pa = init_info.host_ctx_pa; 622 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 623 624 init_info.pd = &iwpd->sc_pd; 625 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; 626 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 627 init_info.qp_uk_init_info.first_sq_wq = 1; 628 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 629 init_waitqueue_head(&iwqp->waitq); 630 init_waitqueue_head(&iwqp->mod_qp_waitq); 631 632 if (udata) { 633 err_code = ib_copy_from_udata(&req, udata, 634 min(sizeof(req), udata->inlen)); 635 if (err_code) { 636 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 637 "ib_copy_from_data fail\n"); 638 goto error; 639 } 640 641 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 642 iwqp->user_mode = 1; 643 if (req.user_wqe_bufs) { 644 struct irdma_ucontext *ucontext = to_ucontext(ibpd->uobject->context); 645 646 init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; 647 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 648 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 649 &ucontext->qp_reg_mem_list); 650 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 651 652 if (!iwqp->iwpbl) { 653 err_code = -ENODATA; 654 irdma_debug(iwdev_to_idev(iwdev), 655 IRDMA_DEBUG_VERBS, 656 "no pbl info\n"); 657 goto error; 658 } 659 } 660 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 661 irdma_setup_virt_qp(iwdev, iwqp, &init_info); 662 } else { 663 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 664 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 665 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 666 } 667 668 if (err_code) { 669 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 670 "setup qp failed\n"); 671 goto error; 672 } 673 674 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 675 if (init_attr->qp_type == IB_QPT_RC) { 676 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 677 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 678 IRDMA_WRITE_WITH_IMM | 679 IRDMA_ROCE; 680 } else { 681 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 682 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 683 IRDMA_ROCE; 684 } 685 } else { 686 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 687 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 688 } 689 690 ret = irdma_sc_qp_init(qp, &init_info); 691 if (ret) { 692 err_code = -EPROTO; 693 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 694 "qp_init fail\n"); 695 goto error; 696 } 697 698 ctx_info = &iwqp->ctx_info; 699 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 700 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 701 702 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 703 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 704 else 705 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 706 707 err_code = irdma_cqp_create_qp_cmd(iwqp); 708 if (err_code) 709 goto error; 710 711 atomic_set(&iwqp->refcnt, 1); 712 spin_lock_init(&iwqp->lock); 713 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 714 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 715 rf->qp_table[qp_num] = iwqp; 716 iwqp->max_send_wr = sq_size; 717 iwqp->max_recv_wr = rq_size; 718 719 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 720 if (dev->ws_add(&iwdev->vsi, 0)) { 721 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 722 err_code = -EINVAL; 723 goto error; 724 } 725 726 irdma_qp_add_qos(&iwqp->sc_qp); 727 } 728 729 if (udata) { 730 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 731 if (udata->outlen < sizeof(uresp)) { 732 uresp.lsmm = 1; 733 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 734 } else { 735 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 736 uresp.lsmm = 1; 737 } 738 uresp.actual_sq_size = sq_size; 739 uresp.actual_rq_size = rq_size; 740 uresp.qp_id = qp_num; 741 uresp.qp_caps = qp->qp_uk.qp_caps; 742 743 err_code = ib_copy_to_udata(udata, &uresp, 744 min(sizeof(uresp), udata->outlen)); 745 if (err_code) { 746 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 747 "copy_to_udata failed\n"); 748 kc_irdma_destroy_qp(&iwqp->ibqp, udata); 749 return ERR_PTR(err_code); 750 } 751 } 752 753 init_completion(&iwqp->free_qp); 754 return &iwqp->ibqp; 755 756 error: 757 irdma_free_qp_rsrc(iwqp); 758 759 return ERR_PTR(err_code); 760 } 761 762 /** 763 * irdma_destroy_qp - destroy qp 764 * @ibqp: qp's ib pointer also to get to device's qp address 765 * @udata: user data 766 */ 767 int 768 irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 769 { 770 struct irdma_qp *iwqp = to_iwqp(ibqp); 771 struct irdma_device *iwdev = iwqp->iwdev; 772 773 if (iwqp->sc_qp.qp_uk.destroy_pending) 774 goto free_rsrc; 775 iwqp->sc_qp.qp_uk.destroy_pending = true; 776 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) 777 irdma_modify_qp_to_err(&iwqp->sc_qp); 778 779 if (!iwqp->user_mode) 780 cancel_delayed_work_sync(&iwqp->dwork_flush); 781 782 irdma_qp_rem_ref(&iwqp->ibqp); 783 wait_for_completion(&iwqp->free_qp); 784 irdma_free_lsmm_rsrc(iwqp); 785 if (!iwdev->rf->reset && 786 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) 787 return -ENOTRECOVERABLE; 788 free_rsrc: 789 if (!iwqp->user_mode) { 790 if (iwqp->iwscq) { 791 irdma_clean_cqes(iwqp, iwqp->iwscq); 792 if (iwqp->iwrcq != iwqp->iwscq) 793 irdma_clean_cqes(iwqp, iwqp->iwrcq); 794 } 795 } 796 irdma_remove_push_mmap_entries(iwqp); 797 irdma_free_qp_rsrc(iwqp); 798 799 return 0; 800 } 801 802 /** 803 * irdma_create_cq - create cq 804 * @ibcq: CQ allocated 805 * @attr: attributes for cq 806 * @udata: user data 807 */ 808 int 809 irdma_create_cq(struct ib_cq *ibcq, 810 const struct ib_cq_init_attr *attr, 811 struct ib_udata *udata) 812 { 813 struct ib_device *ibdev = ibcq->device; 814 struct irdma_device *iwdev = to_iwdev(ibdev); 815 struct irdma_pci_f *rf = iwdev->rf; 816 struct irdma_cq *iwcq = to_iwcq(ibcq); 817 u32 cq_num = 0; 818 struct irdma_sc_cq *cq; 819 struct irdma_sc_dev *dev = &rf->sc_dev; 820 struct irdma_cq_init_info info = {0}; 821 int status; 822 struct irdma_cqp_request *cqp_request; 823 struct cqp_cmds_info *cqp_info; 824 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 825 unsigned long flags; 826 int err_code; 827 int entries = attr->cqe; 828 829 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 830 if (err_code) 831 return err_code; 832 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 833 &rf->next_cq); 834 if (err_code) 835 return err_code; 836 cq = &iwcq->sc_cq; 837 cq->back_cq = iwcq; 838 atomic_set(&iwcq->refcnt, 1); 839 spin_lock_init(&iwcq->lock); 840 INIT_LIST_HEAD(&iwcq->resize_list); 841 INIT_LIST_HEAD(&iwcq->cmpl_generated); 842 info.dev = dev; 843 ukinfo->cq_size = max(entries, 4); 844 ukinfo->cq_id = cq_num; 845 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 846 if (attr->comp_vector < rf->ceqs_count) 847 info.ceq_id = attr->comp_vector; 848 info.ceq_id_valid = true; 849 info.ceqe_mask = 1; 850 info.type = IRDMA_CQ_TYPE_IWARP; 851 info.vsi = &iwdev->vsi; 852 853 if (udata) { 854 struct irdma_ucontext *ucontext; 855 struct irdma_create_cq_req req = {0}; 856 struct irdma_cq_mr *cqmr; 857 struct irdma_pbl *iwpbl; 858 struct irdma_pbl *iwpbl_shadow; 859 struct irdma_cq_mr *cqmr_shadow; 860 861 iwcq->user_mode = true; 862 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 863 if (ib_copy_from_udata(&req, udata, 864 min(sizeof(req), udata->inlen))) { 865 err_code = -EFAULT; 866 goto cq_free_rsrc; 867 } 868 869 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 870 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 871 &ucontext->cq_reg_mem_list); 872 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 873 if (!iwpbl) { 874 err_code = -EPROTO; 875 goto cq_free_rsrc; 876 } 877 iwcq->iwpbl = iwpbl; 878 iwcq->cq_mem_size = 0; 879 cqmr = &iwpbl->cq_mr; 880 881 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 882 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 883 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 884 iwpbl_shadow = irdma_get_pbl((unsigned long)req.user_shadow_area, 885 &ucontext->cq_reg_mem_list); 886 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 887 888 if (!iwpbl_shadow) { 889 err_code = -EPROTO; 890 goto cq_free_rsrc; 891 } 892 iwcq->iwpbl_shadow = iwpbl_shadow; 893 cqmr_shadow = &iwpbl_shadow->cq_mr; 894 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 895 cqmr->split = true; 896 } else { 897 info.shadow_area_pa = cqmr->shadow; 898 } 899 if (iwpbl->pbl_allocated) { 900 info.virtual_map = true; 901 info.pbl_chunk_size = 1; 902 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 903 } else { 904 info.cq_base_pa = cqmr->cq_pbl.addr; 905 } 906 } else { 907 /* Kmode allocations */ 908 int rsize; 909 910 if (entries < 1 || entries > rf->max_cqe) { 911 err_code = -EINVAL; 912 goto cq_free_rsrc; 913 } 914 915 entries++; 916 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 917 entries *= 2; 918 ukinfo->cq_size = entries; 919 920 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 921 iwcq->kmem.size = round_up(rsize, 256); 922 iwcq->kmem.va = irdma_allocate_dma_mem(dev->hw, &iwcq->kmem, 923 iwcq->kmem.size, 256); 924 if (!iwcq->kmem.va) { 925 err_code = -ENOMEM; 926 goto cq_free_rsrc; 927 } 928 929 iwcq->kmem_shadow.size = IRDMA_SHADOW_AREA_SIZE << 3; 930 iwcq->kmem_shadow.va = irdma_allocate_dma_mem(dev->hw, 931 &iwcq->kmem_shadow, 932 iwcq->kmem_shadow.size, 933 64); 934 935 if (!iwcq->kmem_shadow.va) { 936 err_code = -ENOMEM; 937 goto cq_free_rsrc; 938 } 939 info.shadow_area_pa = iwcq->kmem_shadow.pa; 940 ukinfo->shadow_area = iwcq->kmem_shadow.va; 941 ukinfo->cq_base = iwcq->kmem.va; 942 info.cq_base_pa = iwcq->kmem.pa; 943 } 944 945 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 946 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 947 (u32)IRDMA_MAX_CQ_READ_THRESH); 948 if (irdma_sc_cq_init(cq, &info)) { 949 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 950 "init cq fail\n"); 951 err_code = -EPROTO; 952 goto cq_free_rsrc; 953 } 954 955 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 956 if (!cqp_request) { 957 err_code = -ENOMEM; 958 goto cq_free_rsrc; 959 } 960 cqp_info = &cqp_request->info; 961 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 962 cqp_info->post_sq = 1; 963 cqp_info->in.u.cq_create.cq = cq; 964 cqp_info->in.u.cq_create.check_overflow = true; 965 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 966 status = irdma_handle_cqp_op(rf, cqp_request); 967 irdma_put_cqp_request(&rf->cqp, cqp_request); 968 if (status) { 969 err_code = -ENOMEM; 970 goto cq_free_rsrc; 971 } 972 973 if (udata) { 974 struct irdma_create_cq_resp resp = {0}; 975 976 resp.cq_id = info.cq_uk_init_info.cq_id; 977 resp.cq_size = info.cq_uk_init_info.cq_size; 978 if (ib_copy_to_udata(udata, &resp, 979 min(sizeof(resp), udata->outlen))) { 980 irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, 981 "copy to user data\n"); 982 err_code = -EPROTO; 983 goto cq_destroy; 984 } 985 } 986 987 rf->cq_table[cq_num] = iwcq; 988 init_completion(&iwcq->free_cq); 989 990 return 0; 991 cq_destroy: 992 irdma_cq_wq_destroy(rf, cq); 993 cq_free_rsrc: 994 irdma_cq_free_rsrc(rf, iwcq); 995 return err_code; 996 } 997 998 /** 999 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 1000 * @iwmr: iwmr for IB's user page addresses 1001 * @pbl: ple pointer to save 1 level or 0 level pble 1002 * @level: indicated level 0, 1 or 2 1003 */ 1004 1005 void 1006 irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 1007 enum irdma_pble_level level) 1008 { 1009 struct ib_umem *region = iwmr->region; 1010 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1011 int chunk_pages, entry, i; 1012 struct scatterlist *sg; 1013 u64 pg_addr = 0; 1014 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 1015 struct irdma_pble_info *pinfo; 1016 u32 idx = 0; 1017 u32 pbl_cnt = 0; 1018 1019 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 1020 for_each_sg(region->sg_head.sgl, sg, region->nmap, entry) { 1021 chunk_pages = DIV_ROUND_UP(sg_dma_len(sg), iwmr->page_size); 1022 if (iwmr->type == IRDMA_MEMREG_TYPE_QP && !iwpbl->qp_mr.sq_page) 1023 iwpbl->qp_mr.sq_page = sg_page(sg); 1024 for (i = 0; i < chunk_pages; i++) { 1025 pg_addr = sg_dma_address(sg) + (i * iwmr->page_size); 1026 if ((entry + i) == 0) 1027 *pbl = pg_addr & iwmr->page_msk; 1028 else if (!(pg_addr & ~iwmr->page_msk)) 1029 *pbl = pg_addr; 1030 else 1031 continue; 1032 if (++pbl_cnt == palloc->total_cnt) 1033 break; 1034 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 1035 } 1036 } 1037 } 1038 1039 /** 1040 * irdma_destroy_ah - Destroy address handle 1041 * @ibah: pointer to address handle 1042 * @ah_flags: destroy flags 1043 */ 1044 1045 void 1046 irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 1047 { 1048 struct irdma_device *iwdev = to_iwdev(ibah->device); 1049 struct irdma_ah *ah = to_iwah(ibah); 1050 1051 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 1052 false, NULL, ah); 1053 1054 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 1055 ah->sc_ah.ah_info.ah_idx); 1056 } 1057 1058 int 1059 irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 1060 { 1061 struct ib_pd *ibpd = ib_mr->pd; 1062 struct irdma_pd *iwpd = to_iwpd(ibpd); 1063 struct irdma_mr *iwmr = to_iwmr(ib_mr); 1064 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 1065 struct irdma_dealloc_stag_info *info; 1066 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 1067 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 1068 struct irdma_cqp_request *cqp_request; 1069 struct cqp_cmds_info *cqp_info; 1070 int status; 1071 1072 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 1073 if (iwmr->region) { 1074 struct irdma_ucontext *ucontext; 1075 1076 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); 1077 irdma_del_memlist(iwmr, ucontext); 1078 } 1079 goto done; 1080 } 1081 1082 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 1083 if (!cqp_request) 1084 return -ENOMEM; 1085 1086 cqp_info = &cqp_request->info; 1087 info = &cqp_info->in.u.dealloc_stag.info; 1088 memset(info, 0, sizeof(*info)); 1089 info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; 1090 info->stag_idx = RS_64_1(ib_mr->rkey, IRDMA_CQPSQ_STAG_IDX_S); 1091 info->mr = true; 1092 if (iwpbl->pbl_allocated) 1093 info->dealloc_pbl = true; 1094 1095 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 1096 cqp_info->post_sq = 1; 1097 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 1098 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 1099 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 1100 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 1101 if (status) 1102 return status; 1103 1104 irdma_free_stag(iwdev, iwmr->stag); 1105 done: 1106 if (iwpbl->pbl_allocated) 1107 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 1108 1109 if (iwmr->region) 1110 ib_umem_release(iwmr->region); 1111 kfree(iwmr); 1112 1113 return 0; 1114 } 1115 1116 int 1117 kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, 1118 u16 *vlan_id) 1119 { 1120 int ret; 1121 union ib_gid sgid; 1122 struct ib_gid_attr sgid_attr; 1123 struct irdma_av *av = &iwqp->roce_ah.av; 1124 1125 ret = ib_get_cached_gid(iwqp->ibqp.device, attr->ah_attr.port_num, 1126 attr->ah_attr.grh.sgid_index, &sgid, 1127 &sgid_attr); 1128 if (ret) 1129 return ret; 1130 1131 if (sgid_attr.ndev) { 1132 *vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev); 1133 ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, IF_LLADDR(sgid_attr.ndev)); 1134 } 1135 1136 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); 1137 1138 dev_put(sgid_attr.ndev); 1139 1140 return 0; 1141 } 1142 1143 /** 1144 * irdma_destroy_cq - destroy cq 1145 * @ib_cq: cq pointer 1146 * @udata: user data 1147 */ 1148 void 1149 irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1150 { 1151 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1152 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1153 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1154 struct irdma_sc_dev *dev = cq->dev; 1155 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1156 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1157 unsigned long flags; 1158 1159 spin_lock_irqsave(&iwcq->lock, flags); 1160 if (!list_empty(&iwcq->cmpl_generated)) 1161 irdma_remove_cmpls_list(iwcq); 1162 if (!list_empty(&iwcq->resize_list)) 1163 irdma_process_resize_list(iwcq, iwdev, NULL); 1164 spin_unlock_irqrestore(&iwcq->lock, flags); 1165 1166 irdma_cq_rem_ref(ib_cq); 1167 wait_for_completion(&iwcq->free_cq); 1168 1169 irdma_cq_wq_destroy(iwdev->rf, cq); 1170 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1171 1172 spin_lock_irqsave(&iwceq->ce_lock, flags); 1173 irdma_sc_cleanup_ceqes(cq, ceq); 1174 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1175 } 1176 1177 /** 1178 * irdma_alloc_mw - Allocate memory window 1179 * @pd: Protection domain 1180 * @type: Window type 1181 * @udata: user data pointer 1182 */ 1183 struct ib_mw * 1184 irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 1185 struct ib_udata *udata) 1186 { 1187 struct irdma_device *iwdev = to_iwdev(pd->device); 1188 struct irdma_mr *iwmr; 1189 int err_code; 1190 u32 stag; 1191 1192 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 1193 if (!iwmr) 1194 return ERR_PTR(-ENOMEM); 1195 1196 stag = irdma_create_stag(iwdev); 1197 if (!stag) { 1198 kfree(iwmr); 1199 return ERR_PTR(-ENOMEM); 1200 } 1201 1202 iwmr->stag = stag; 1203 iwmr->ibmw.rkey = stag; 1204 iwmr->ibmw.pd = pd; 1205 iwmr->ibmw.type = type; 1206 iwmr->ibmw.device = pd->device; 1207 1208 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 1209 if (err_code) { 1210 irdma_free_stag(iwdev, stag); 1211 kfree(iwmr); 1212 return ERR_PTR(err_code); 1213 } 1214 1215 return &iwmr->ibmw; 1216 } 1217 1218 /** 1219 * kc_set_loc_seq_num_mss - Set local seq number and mss 1220 * @cm_node: cm node info 1221 */ 1222 void 1223 kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node) 1224 { 1225 struct timespec ts; 1226 1227 getnanotime(&ts); 1228 cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec; 1229 if (cm_node->iwdev->vsi.mtu > 1500 && 1230 2 * cm_node->iwdev->vsi.mtu > cm_node->iwdev->rcv_wnd) 1231 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? 1232 (1500 - IRDMA_MTU_TO_MSS_IPV4) : 1233 (1500 - IRDMA_MTU_TO_MSS_IPV6); 1234 else 1235 cm_node->tcp_cntxt.mss = (cm_node->ipv4) ? 1236 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4) : 1237 (cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6); 1238 } 1239 1240 /** 1241 * irdma_disassociate_ucontext - Disassociate user context 1242 * @context: ib user context 1243 */ 1244 void 1245 irdma_disassociate_ucontext(struct ib_ucontext *context) 1246 { 1247 } 1248 1249 struct ib_device * 1250 ib_device_get_by_netdev(struct ifnet *netdev, int driver_id) 1251 { 1252 struct irdma_device *iwdev; 1253 struct irdma_handler *hdl; 1254 unsigned long flags; 1255 1256 spin_lock_irqsave(&irdma_handler_lock, flags); 1257 list_for_each_entry(hdl, &irdma_handlers, list) { 1258 iwdev = hdl->iwdev; 1259 if (netdev == iwdev->netdev) { 1260 spin_unlock_irqrestore(&irdma_handler_lock, 1261 flags); 1262 return &iwdev->ibdev; 1263 } 1264 } 1265 spin_unlock_irqrestore(&irdma_handler_lock, flags); 1266 1267 return NULL; 1268 } 1269 1270 void 1271 ib_unregister_device_put(struct ib_device *device) 1272 { 1273 ib_unregister_device(device); 1274 } 1275 1276 /** 1277 * irdma_query_gid_roce - Query port GID for Roce 1278 * @ibdev: device pointer from stack 1279 * @port: port number 1280 * @index: Entry index 1281 * @gid: Global ID 1282 */ 1283 int 1284 irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index, 1285 union ib_gid *gid) 1286 { 1287 int ret; 1288 1289 ret = rdma_query_gid(ibdev, port, index, gid); 1290 if (ret == -EAGAIN) { 1291 memcpy(gid, &zgid, sizeof(*gid)); 1292 return 0; 1293 } 1294 1295 return ret; 1296 } 1297 1298 /** 1299 * irdma_modify_port - modify port attributes 1300 * @ibdev: device pointer from stack 1301 * @port: port number for query 1302 * @mask: Property mask 1303 * @props: returning device attributes 1304 */ 1305 int 1306 irdma_modify_port(struct ib_device *ibdev, u8 port, int mask, 1307 struct ib_port_modify *props) 1308 { 1309 if (port > 1) 1310 return -EINVAL; 1311 1312 return 0; 1313 } 1314 1315 /** 1316 * irdma_query_pkey - Query partition key 1317 * @ibdev: device pointer from stack 1318 * @port: port number 1319 * @index: index of pkey 1320 * @pkey: pointer to store the pkey 1321 */ 1322 int 1323 irdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, 1324 u16 *pkey) 1325 { 1326 if (index >= IRDMA_PKEY_TBL_SZ) 1327 return -EINVAL; 1328 1329 *pkey = IRDMA_DEFAULT_PKEY; 1330 return 0; 1331 } 1332 1333 int 1334 irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, 1335 struct ib_port_immutable *immutable) 1336 { 1337 struct ib_port_attr attr; 1338 int err; 1339 1340 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 1341 err = ib_query_port(ibdev, port_num, &attr); 1342 if (err) 1343 return err; 1344 1345 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 1346 immutable->pkey_tbl_len = attr.pkey_tbl_len; 1347 immutable->gid_tbl_len = attr.gid_tbl_len; 1348 1349 return 0; 1350 } 1351 1352 int 1353 irdma_iw_port_immutable(struct ib_device *ibdev, u8 port_num, 1354 struct ib_port_immutable *immutable) 1355 { 1356 struct ib_port_attr attr; 1357 int err; 1358 1359 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 1360 err = ib_query_port(ibdev, port_num, &attr); 1361 if (err) 1362 return err; 1363 immutable->gid_tbl_len = 1; 1364 1365 return 0; 1366 } 1367 1368 /** 1369 * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed 1370 * @link_speed: netdev phy link speed 1371 * @active_speed: IB port speed 1372 * @active_width: IB port width 1373 */ 1374 void 1375 irdma_get_eth_speed_and_width(u32 link_speed, u8 *active_speed, 1376 u8 *active_width) 1377 { 1378 if (link_speed <= SPEED_1000) { 1379 *active_width = IB_WIDTH_1X; 1380 *active_speed = IB_SPEED_SDR; 1381 } else if (link_speed <= SPEED_10000) { 1382 *active_width = IB_WIDTH_1X; 1383 *active_speed = IB_SPEED_FDR10; 1384 } else if (link_speed <= SPEED_20000) { 1385 *active_width = IB_WIDTH_4X; 1386 *active_speed = IB_SPEED_DDR; 1387 } else if (link_speed <= SPEED_25000) { 1388 *active_width = IB_WIDTH_1X; 1389 *active_speed = IB_SPEED_EDR; 1390 } else if (link_speed <= SPEED_40000) { 1391 *active_width = IB_WIDTH_4X; 1392 *active_speed = IB_SPEED_FDR10; 1393 } else { 1394 *active_width = IB_WIDTH_4X; 1395 *active_speed = IB_SPEED_EDR; 1396 } 1397 } 1398 1399 /** 1400 * irdma_query_port - get port attributes 1401 * @ibdev: device pointer from stack 1402 * @port: port number for query 1403 * @props: returning device attributes 1404 */ 1405 int 1406 irdma_query_port(struct ib_device *ibdev, u8 port, 1407 struct ib_port_attr *props) 1408 { 1409 struct irdma_device *iwdev = to_iwdev(ibdev); 1410 struct ifnet *netdev = iwdev->netdev; 1411 1412 /* no need to zero out pros here. done by caller */ 1413 1414 props->max_mtu = IB_MTU_4096; 1415 props->active_mtu = ib_mtu_int_to_enum(netdev->if_mtu); 1416 props->lid = 1; 1417 props->lmc = 0; 1418 props->sm_lid = 0; 1419 props->sm_sl = 0; 1420 if ((netdev->if_link_state == LINK_STATE_UP) && (netdev->if_drv_flags & IFF_DRV_RUNNING)) { 1421 props->state = IB_PORT_ACTIVE; 1422 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 1423 } else { 1424 props->state = IB_PORT_DOWN; 1425 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 1426 } 1427 irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, 1428 &props->active_width); 1429 1430 if (rdma_protocol_roce(ibdev, 1)) { 1431 props->gid_tbl_len = 32; 1432 kc_set_props_ip_gid_caps(props); 1433 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 1434 } else { 1435 props->gid_tbl_len = 1; 1436 } 1437 props->qkey_viol_cntr = 0; 1438 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 1439 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 1440 1441 return 0; 1442 } 1443 1444 extern const char *const irdma_hw_stat_names[]; 1445 1446 /** 1447 * irdma_alloc_hw_stats - Allocate a hw stats structure 1448 * @ibdev: device pointer from stack 1449 * @port_num: port number 1450 */ 1451 struct rdma_hw_stats * 1452 irdma_alloc_hw_stats(struct ib_device *ibdev, 1453 u8 port_num) 1454 { 1455 struct irdma_device *iwdev = to_iwdev(ibdev); 1456 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1457 1458 int num_counters = dev->hw_attrs.max_stat_idx; 1459 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 1460 1461 return rdma_alloc_hw_stats_struct(irdma_hw_stat_names, num_counters, 1462 lifespan); 1463 } 1464 1465 /** 1466 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 1467 * @ibdev: device pointer from stack 1468 * @stats: stats pointer from stack 1469 * @port_num: port number 1470 * @index: which hw counter the stack is requesting we update 1471 */ 1472 int 1473 irdma_get_hw_stats(struct ib_device *ibdev, 1474 struct rdma_hw_stats *stats, u8 port_num, 1475 int index) 1476 { 1477 struct irdma_device *iwdev = to_iwdev(ibdev); 1478 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 1479 1480 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 1481 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 1482 1483 memcpy(&stats->value[0], hw_stats, sizeof(u64)* stats->num_counters); 1484 1485 return stats->num_counters; 1486 } 1487 1488 /** 1489 * irdma_query_gid - Query port GID 1490 * @ibdev: device pointer from stack 1491 * @port: port number 1492 * @index: Entry index 1493 * @gid: Global ID 1494 */ 1495 int 1496 irdma_query_gid(struct ib_device *ibdev, u8 port, int index, 1497 union ib_gid *gid) 1498 { 1499 struct irdma_device *iwdev = to_iwdev(ibdev); 1500 1501 memset(gid->raw, 0, sizeof(gid->raw)); 1502 ether_addr_copy(gid->raw, IF_LLADDR(iwdev->netdev)); 1503 1504 return 0; 1505 } 1506 1507 enum rdma_link_layer 1508 irdma_get_link_layer(struct ib_device *ibdev, 1509 u8 port_num) 1510 { 1511 return IB_LINK_LAYER_ETHERNET; 1512 } 1513 1514 inline enum ib_mtu 1515 ib_mtu_int_to_enum(int mtu) 1516 { 1517 if (mtu >= 4096) 1518 return IB_MTU_4096; 1519 else if (mtu >= 2048) 1520 return IB_MTU_2048; 1521 else if (mtu >= 1024) 1522 return IB_MTU_1024; 1523 else if (mtu >= 512) 1524 return IB_MTU_512; 1525 else 1526 return IB_MTU_256; 1527 } 1528 1529 inline void 1530 kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev) 1531 { 1532 iwdev->ibdev.uverbs_cmd_mask |= 1533 BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST) | 1534 BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH) | 1535 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH) | 1536 BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST); 1537 } 1538 1539 inline void 1540 kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev) 1541 { 1542 iwdev->ibdev.uverbs_cmd_mask = 1543 BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT) | 1544 BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE) | 1545 BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT) | 1546 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD) | 1547 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD) | 1548 BIT_ULL(IB_USER_VERBS_CMD_REG_MR) | 1549 BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR) | 1550 BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | 1551 BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ) | 1552 BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ) | 1553 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ) | 1554 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | 1555 BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP) | 1556 BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP) | 1557 BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) | 1558 BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) | 1559 BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) | 1560 BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) | 1561 BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) | 1562 BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) | 1563 BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) | 1564 BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); 1565 iwdev->ibdev.uverbs_ex_cmd_mask = 1566 BIT_ULL(IB_USER_VERBS_EX_CMD_MODIFY_QP) | 1567 BIT_ULL(IB_USER_VERBS_EX_CMD_QUERY_DEVICE); 1568 1569 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 1570 iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ); 1571 } 1572