1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/pci.h> 35 #include <rdma/ib_addr.h> 36 #include <rdma/ib_umem.h> 37 #include <rdma/uverbs_ioctl.h> 38 #include "hns_roce_common.h" 39 #include "hns_roce_device.h" 40 #include "hns_roce_hem.h" 41 42 static void flush_work_handle(struct work_struct *work) 43 { 44 struct hns_roce_work *flush_work = container_of(work, 45 struct hns_roce_work, work); 46 struct hns_roce_qp *hr_qp = container_of(flush_work, 47 struct hns_roce_qp, flush_work); 48 struct device *dev = flush_work->hr_dev->dev; 49 struct ib_qp_attr attr; 50 int attr_mask; 51 int ret; 52 53 attr_mask = IB_QP_STATE; 54 attr.qp_state = IB_QPS_ERR; 55 56 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { 57 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); 58 if (ret) 59 dev_err(dev, "modify QP to error state failed(%d) during CQE flush\n", 60 ret); 61 } 62 63 /* 64 * make sure we signal QP destroy leg that flush QP was completed 65 * so that it can safely proceed ahead now and destroy QP 66 */ 67 if (refcount_dec_and_test(&hr_qp->refcount)) 68 complete(&hr_qp->free); 69 } 70 71 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 72 { 73 struct hns_roce_work *flush_work = &hr_qp->flush_work; 74 75 flush_work->hr_dev = hr_dev; 76 INIT_WORK(&flush_work->work, flush_work_handle); 77 refcount_inc(&hr_qp->refcount); 78 queue_work(hr_dev->irq_workq, &flush_work->work); 79 } 80 81 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp) 82 { 83 /* 84 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state 85 * gets into errored mode. Hence, as a workaround to this 86 * hardware limitation, driver needs to assist in flushing. But 87 * the flushing operation uses mailbox to convey the QP state to 88 * the hardware and which can sleep due to the mutex protection 89 * around the mailbox calls. Hence, use the deferred flush for 90 * now. 91 */ 92 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag)) 93 init_flush_work(dev, qp); 94 } 95 96 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) 97 { 98 struct device *dev = hr_dev->dev; 99 struct hns_roce_qp *qp; 100 101 xa_lock(&hr_dev->qp_table_xa); 102 qp = __hns_roce_qp_lookup(hr_dev, qpn); 103 if (qp) 104 refcount_inc(&qp->refcount); 105 xa_unlock(&hr_dev->qp_table_xa); 106 107 if (!qp) { 108 dev_warn(dev, "async event for bogus QP %08x\n", qpn); 109 return; 110 } 111 112 if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR || 113 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR || 114 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR || 115 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION || 116 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) { 117 qp->state = IB_QPS_ERR; 118 119 flush_cqe(hr_dev, qp); 120 } 121 122 qp->event(qp, (enum hns_roce_event)event_type); 123 124 if (refcount_dec_and_test(&qp->refcount)) 125 complete(&qp->free); 126 } 127 128 static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, 129 enum hns_roce_event type) 130 { 131 struct ib_qp *ibqp = &hr_qp->ibqp; 132 struct ib_event event; 133 134 if (ibqp->event_handler) { 135 event.device = ibqp->device; 136 event.element.qp = ibqp; 137 switch (type) { 138 case HNS_ROCE_EVENT_TYPE_PATH_MIG: 139 event.event = IB_EVENT_PATH_MIG; 140 break; 141 case HNS_ROCE_EVENT_TYPE_COMM_EST: 142 event.event = IB_EVENT_COMM_EST; 143 break; 144 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: 145 event.event = IB_EVENT_SQ_DRAINED; 146 break; 147 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH: 148 event.event = IB_EVENT_QP_LAST_WQE_REACHED; 149 break; 150 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: 151 event.event = IB_EVENT_QP_FATAL; 152 break; 153 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: 154 event.event = IB_EVENT_PATH_MIG_ERR; 155 break; 156 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: 157 event.event = IB_EVENT_QP_REQ_ERR; 158 break; 159 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: 160 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: 161 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH: 162 event.event = IB_EVENT_QP_ACCESS_ERR; 163 break; 164 default: 165 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n", 166 type, hr_qp->qpn); 167 return; 168 } 169 ibqp->event_handler(&event, ibqp->qp_context); 170 } 171 } 172 173 static u8 get_least_load_bankid_for_qp(struct hns_roce_bank *bank) 174 { 175 u32 least_load = bank[0].inuse; 176 u8 bankid = 0; 177 u32 bankcnt; 178 u8 i; 179 180 for (i = 1; i < HNS_ROCE_QP_BANK_NUM; i++) { 181 bankcnt = bank[i].inuse; 182 if (bankcnt < least_load) { 183 least_load = bankcnt; 184 bankid = i; 185 } 186 } 187 188 return bankid; 189 } 190 191 static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid, 192 unsigned long *qpn) 193 { 194 int id; 195 196 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL); 197 if (id < 0) { 198 id = ida_alloc_range(&bank->ida, bank->min, bank->max, 199 GFP_KERNEL); 200 if (id < 0) 201 return id; 202 } 203 204 /* the QPN should keep increasing until the max value is reached. */ 205 bank->next = (id + 1) > bank->max ? bank->min : id + 1; 206 207 /* the lower 3 bits is bankid */ 208 *qpn = (id << 3) | bankid; 209 210 return 0; 211 } 212 static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 213 { 214 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 215 unsigned long num = 0; 216 u8 bankid; 217 int ret; 218 219 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { 220 num = 1; 221 } else { 222 mutex_lock(&qp_table->bank_mutex); 223 bankid = get_least_load_bankid_for_qp(qp_table->bank); 224 225 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid, 226 &num); 227 if (ret) { 228 ibdev_err(&hr_dev->ib_dev, 229 "failed to alloc QPN, ret = %d\n", ret); 230 mutex_unlock(&qp_table->bank_mutex); 231 return ret; 232 } 233 234 qp_table->bank[bankid].inuse++; 235 mutex_unlock(&qp_table->bank_mutex); 236 } 237 238 hr_qp->qpn = num; 239 240 return 0; 241 } 242 243 static void add_qp_to_list(struct hns_roce_dev *hr_dev, 244 struct hns_roce_qp *hr_qp, 245 struct ib_cq *send_cq, struct ib_cq *recv_cq) 246 { 247 struct hns_roce_cq *hr_send_cq, *hr_recv_cq; 248 unsigned long flags; 249 250 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL; 251 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL; 252 253 spin_lock_irqsave(&hr_dev->qp_list_lock, flags); 254 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq); 255 256 list_add_tail(&hr_qp->node, &hr_dev->qp_list); 257 if (hr_send_cq) 258 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list); 259 if (hr_recv_cq) 260 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list); 261 262 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq); 263 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); 264 } 265 266 static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, 267 struct hns_roce_qp *hr_qp, 268 struct ib_qp_init_attr *init_attr) 269 { 270 struct xarray *xa = &hr_dev->qp_table_xa; 271 int ret; 272 273 if (!hr_qp->qpn) 274 return -EINVAL; 275 276 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); 277 if (ret) 278 dev_err(hr_dev->dev, "failed to xa store for QPC\n"); 279 else 280 /* add QP to device's QP list for softwc */ 281 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, 282 init_attr->recv_cq); 283 284 return ret; 285 } 286 287 static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 288 { 289 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 290 struct device *dev = hr_dev->dev; 291 int ret; 292 293 if (!hr_qp->qpn) 294 return -EINVAL; 295 296 /* Alloc memory for QPC */ 297 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); 298 if (ret) { 299 dev_err(dev, "failed to get QPC table\n"); 300 goto err_out; 301 } 302 303 /* Alloc memory for IRRL */ 304 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 305 if (ret) { 306 dev_err(dev, "failed to get IRRL table\n"); 307 goto err_put_qp; 308 } 309 310 if (hr_dev->caps.trrl_entry_sz) { 311 /* Alloc memory for TRRL */ 312 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, 313 hr_qp->qpn); 314 if (ret) { 315 dev_err(dev, "failed to get TRRL table\n"); 316 goto err_put_irrl; 317 } 318 } 319 320 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 321 /* Alloc memory for SCC CTX */ 322 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, 323 hr_qp->qpn); 324 if (ret) { 325 dev_err(dev, "failed to get SCC CTX table\n"); 326 goto err_put_trrl; 327 } 328 } 329 330 return 0; 331 332 err_put_trrl: 333 if (hr_dev->caps.trrl_entry_sz) 334 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 335 336 err_put_irrl: 337 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 338 339 err_put_qp: 340 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); 341 342 err_out: 343 return ret; 344 } 345 346 static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp) 347 { 348 rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry); 349 } 350 351 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 352 { 353 struct xarray *xa = &hr_dev->qp_table_xa; 354 unsigned long flags; 355 356 list_del(&hr_qp->node); 357 358 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 359 list_del(&hr_qp->sq_node); 360 361 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI && 362 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT) 363 list_del(&hr_qp->rq_node); 364 365 xa_lock_irqsave(xa, flags); 366 __xa_erase(xa, hr_qp->qpn); 367 xa_unlock_irqrestore(xa, flags); 368 } 369 370 static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 371 { 372 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 373 374 if (hr_dev->caps.trrl_entry_sz) 375 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); 376 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); 377 } 378 379 static inline u8 get_qp_bankid(unsigned long qpn) 380 { 381 /* The lower 3 bits of QPN are used to hash to different banks */ 382 return (u8)(qpn & GENMASK(2, 0)); 383 } 384 385 static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 386 { 387 u8 bankid; 388 389 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) 390 return; 391 392 if (hr_qp->qpn < hr_dev->caps.reserved_qps) 393 return; 394 395 bankid = get_qp_bankid(hr_qp->qpn); 396 397 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3); 398 399 mutex_lock(&hr_dev->qp_table.bank_mutex); 400 hr_dev->qp_table.bank[bankid].inuse--; 401 mutex_unlock(&hr_dev->qp_table.bank_mutex); 402 } 403 404 static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp, 405 bool user) 406 { 407 u32 max_sge = dev->caps.max_rq_sg; 408 409 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 410 return max_sge; 411 412 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will 413 * calculate number of max_sge with reserved SGEs when allocating wqe 414 * buf, so there is no need to do this again in kernel. But the number 415 * may exceed the capacity of SGEs recorded in the firmware, so the 416 * kernel driver should just adapt the value accordingly. 417 */ 418 if (user) 419 max_sge = roundup_pow_of_two(max_sge + 1); 420 else 421 hr_qp->rq.rsv_sge = 1; 422 423 return max_sge; 424 } 425 426 static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, 427 struct hns_roce_qp *hr_qp, int has_rq, bool user) 428 { 429 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user); 430 u32 cnt; 431 432 /* If srq exist, set zero for relative number of rq */ 433 if (!has_rq) { 434 hr_qp->rq.wqe_cnt = 0; 435 hr_qp->rq.max_gs = 0; 436 hr_qp->rq_inl_buf.wqe_cnt = 0; 437 cap->max_recv_wr = 0; 438 cap->max_recv_sge = 0; 439 440 return 0; 441 } 442 443 /* Check the validity of QP support capacity */ 444 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes || 445 cap->max_recv_sge > max_sge) { 446 ibdev_err(&hr_dev->ib_dev, 447 "RQ config error, depth = %u, sge = %u\n", 448 cap->max_recv_wr, cap->max_recv_sge); 449 return -EINVAL; 450 } 451 452 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes)); 453 if (cnt > hr_dev->caps.max_wqes) { 454 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n", 455 cap->max_recv_wr); 456 return -EINVAL; 457 } 458 459 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) + 460 hr_qp->rq.rsv_sge); 461 462 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * 463 hr_qp->rq.max_gs); 464 465 hr_qp->rq.wqe_cnt = cnt; 466 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && 467 hr_qp->ibqp.qp_type != IB_QPT_UD && 468 hr_qp->ibqp.qp_type != IB_QPT_GSI) 469 hr_qp->rq_inl_buf.wqe_cnt = cnt; 470 else 471 hr_qp->rq_inl_buf.wqe_cnt = 0; 472 473 cap->max_recv_wr = cnt; 474 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge; 475 476 return 0; 477 } 478 479 static u32 get_wqe_ext_sge_cnt(struct hns_roce_qp *qp) 480 { 481 /* GSI/UD QP only has extended sge */ 482 if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) 483 return qp->sq.max_gs; 484 485 if (qp->sq.max_gs > HNS_ROCE_SGE_IN_WQE) 486 return qp->sq.max_gs - HNS_ROCE_SGE_IN_WQE; 487 488 return 0; 489 } 490 491 static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt, 492 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap) 493 { 494 u32 total_sge_cnt; 495 u32 wqe_sge_cnt; 496 497 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT; 498 499 hr_qp->sq.max_gs = max(1U, cap->max_send_sge); 500 501 wqe_sge_cnt = get_wqe_ext_sge_cnt(hr_qp); 502 503 /* If the number of extended sge is not zero, they MUST use the 504 * space of HNS_HW_PAGE_SIZE at least. 505 */ 506 if (wqe_sge_cnt) { 507 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * wqe_sge_cnt); 508 hr_qp->sge.sge_cnt = max(total_sge_cnt, 509 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE); 510 } 511 } 512 513 static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev, 514 struct ib_qp_cap *cap, 515 struct hns_roce_ib_create_qp *ucmd) 516 { 517 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz); 518 u8 max_sq_stride = ilog2(roundup_sq_stride); 519 520 /* Sanity check SQ size before proceeding */ 521 if (ucmd->log_sq_stride > max_sq_stride || 522 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) { 523 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n"); 524 return -EINVAL; 525 } 526 527 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) { 528 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n", 529 cap->max_send_sge); 530 return -EINVAL; 531 } 532 533 return 0; 534 } 535 536 static int set_user_sq_size(struct hns_roce_dev *hr_dev, 537 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp, 538 struct hns_roce_ib_create_qp *ucmd) 539 { 540 struct ib_device *ibdev = &hr_dev->ib_dev; 541 u32 cnt = 0; 542 int ret; 543 544 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) || 545 cnt > hr_dev->caps.max_wqes) 546 return -EINVAL; 547 548 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd); 549 if (ret) { 550 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n", 551 ret); 552 return ret; 553 } 554 555 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 556 557 hr_qp->sq.wqe_shift = ucmd->log_sq_stride; 558 hr_qp->sq.wqe_cnt = cnt; 559 560 return 0; 561 } 562 563 static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev, 564 struct hns_roce_qp *hr_qp, 565 struct hns_roce_buf_attr *buf_attr) 566 { 567 int buf_size; 568 int idx = 0; 569 570 hr_qp->buff_size = 0; 571 572 /* SQ WQE */ 573 hr_qp->sq.offset = 0; 574 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt, 575 hr_qp->sq.wqe_shift); 576 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 577 buf_attr->region[idx].size = buf_size; 578 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num; 579 idx++; 580 hr_qp->buff_size += buf_size; 581 } 582 583 /* extend SGE WQE in SQ */ 584 hr_qp->sge.offset = hr_qp->buff_size; 585 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt, 586 hr_qp->sge.sge_shift); 587 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 588 buf_attr->region[idx].size = buf_size; 589 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num; 590 idx++; 591 hr_qp->buff_size += buf_size; 592 } 593 594 /* RQ WQE */ 595 hr_qp->rq.offset = hr_qp->buff_size; 596 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt, 597 hr_qp->rq.wqe_shift); 598 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) { 599 buf_attr->region[idx].size = buf_size; 600 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num; 601 idx++; 602 hr_qp->buff_size += buf_size; 603 } 604 605 if (hr_qp->buff_size < 1) 606 return -EINVAL; 607 608 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; 609 buf_attr->region_count = idx; 610 611 return 0; 612 } 613 614 static int set_kernel_sq_size(struct hns_roce_dev *hr_dev, 615 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) 616 { 617 struct ib_device *ibdev = &hr_dev->ib_dev; 618 u32 cnt; 619 620 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes || 621 cap->max_send_sge > hr_dev->caps.max_sq_sg) { 622 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n"); 623 return -EINVAL; 624 } 625 626 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes)); 627 if (cnt > hr_dev->caps.max_wqes) { 628 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n", 629 cnt); 630 return -EINVAL; 631 } 632 633 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz); 634 hr_qp->sq.wqe_cnt = cnt; 635 636 set_ext_sge_param(hr_dev, cnt, hr_qp, cap); 637 638 /* sync the parameters of kernel QP to user's configuration */ 639 cap->max_send_wr = cnt; 640 cap->max_send_sge = hr_qp->sq.max_gs; 641 642 return 0; 643 } 644 645 static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr) 646 { 647 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr) 648 return 0; 649 650 return 1; 651 } 652 653 static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr) 654 { 655 if (attr->qp_type == IB_QPT_XRC_INI || 656 attr->qp_type == IB_QPT_XRC_TGT || attr->srq || 657 !attr->cap.max_recv_wr) 658 return 0; 659 660 return 1; 661 } 662 663 static int alloc_rq_inline_buf(struct hns_roce_qp *hr_qp, 664 struct ib_qp_init_attr *init_attr) 665 { 666 u32 max_recv_sge = init_attr->cap.max_recv_sge; 667 u32 wqe_cnt = hr_qp->rq_inl_buf.wqe_cnt; 668 struct hns_roce_rinl_wqe *wqe_list; 669 int i; 670 671 /* allocate recv inline buf */ 672 wqe_list = kcalloc(wqe_cnt, sizeof(struct hns_roce_rinl_wqe), 673 GFP_KERNEL); 674 if (!wqe_list) 675 goto err; 676 677 /* Allocate a continuous buffer for all inline sge we need */ 678 wqe_list[0].sg_list = kcalloc(wqe_cnt, (max_recv_sge * 679 sizeof(struct hns_roce_rinl_sge)), 680 GFP_KERNEL); 681 if (!wqe_list[0].sg_list) 682 goto err_wqe_list; 683 684 /* Assign buffers of sg_list to each inline wqe */ 685 for (i = 1; i < wqe_cnt; i++) 686 wqe_list[i].sg_list = &wqe_list[0].sg_list[i * max_recv_sge]; 687 688 hr_qp->rq_inl_buf.wqe_list = wqe_list; 689 690 return 0; 691 692 err_wqe_list: 693 kfree(wqe_list); 694 695 err: 696 return -ENOMEM; 697 } 698 699 static void free_rq_inline_buf(struct hns_roce_qp *hr_qp) 700 { 701 if (hr_qp->rq_inl_buf.wqe_list) 702 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list); 703 kfree(hr_qp->rq_inl_buf.wqe_list); 704 } 705 706 static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 707 struct ib_qp_init_attr *init_attr, 708 struct ib_udata *udata, unsigned long addr) 709 { 710 struct ib_device *ibdev = &hr_dev->ib_dev; 711 struct hns_roce_buf_attr buf_attr = {}; 712 int ret; 713 714 if (!udata && hr_qp->rq_inl_buf.wqe_cnt) { 715 ret = alloc_rq_inline_buf(hr_qp, init_attr); 716 if (ret) { 717 ibdev_err(ibdev, 718 "failed to alloc inline buf, ret = %d.\n", 719 ret); 720 return ret; 721 } 722 } else { 723 hr_qp->rq_inl_buf.wqe_list = NULL; 724 } 725 726 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr); 727 if (ret) { 728 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret); 729 goto err_inline; 730 } 731 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, 732 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, 733 udata, addr); 734 if (ret) { 735 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); 736 goto err_inline; 737 } 738 739 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE) 740 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE; 741 742 return 0; 743 744 err_inline: 745 free_rq_inline_buf(hr_qp); 746 747 return ret; 748 } 749 750 static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) 751 { 752 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr); 753 free_rq_inline_buf(hr_qp); 754 } 755 756 static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev, 757 struct ib_qp_init_attr *init_attr, 758 struct ib_udata *udata, 759 struct hns_roce_ib_create_qp_resp *resp, 760 struct hns_roce_ib_create_qp *ucmd) 761 { 762 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 763 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 764 hns_roce_qp_has_sq(init_attr) && 765 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr)); 766 } 767 768 static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev, 769 struct ib_qp_init_attr *init_attr, 770 struct ib_udata *udata, 771 struct hns_roce_ib_create_qp_resp *resp) 772 { 773 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 774 udata->outlen >= offsetofend(typeof(*resp), cap_flags) && 775 hns_roce_qp_has_rq(init_attr)); 776 } 777 778 static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev, 779 struct ib_qp_init_attr *init_attr) 780 { 781 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) && 782 hns_roce_qp_has_rq(init_attr)); 783 } 784 785 static int qp_mmap_entry(struct hns_roce_qp *hr_qp, 786 struct hns_roce_dev *hr_dev, 787 struct ib_udata *udata, 788 struct hns_roce_ib_create_qp_resp *resp) 789 { 790 struct hns_roce_ucontext *uctx = 791 rdma_udata_to_drv_context(udata, 792 struct hns_roce_ucontext, ibucontext); 793 struct rdma_user_mmap_entry *rdma_entry; 794 u64 address; 795 796 address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE; 797 798 hr_qp->dwqe_mmap_entry = 799 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address, 800 HNS_ROCE_DWQE_SIZE, 801 HNS_ROCE_MMAP_TYPE_DWQE); 802 803 if (!hr_qp->dwqe_mmap_entry) { 804 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n"); 805 return -ENOMEM; 806 } 807 808 rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry; 809 resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry); 810 811 return 0; 812 } 813 814 static int alloc_user_qp_db(struct hns_roce_dev *hr_dev, 815 struct hns_roce_qp *hr_qp, 816 struct ib_qp_init_attr *init_attr, 817 struct ib_udata *udata, 818 struct hns_roce_ib_create_qp *ucmd, 819 struct hns_roce_ib_create_qp_resp *resp) 820 { 821 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata, 822 struct hns_roce_ucontext, ibucontext); 823 struct ib_device *ibdev = &hr_dev->ib_dev; 824 int ret; 825 826 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) { 827 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb); 828 if (ret) { 829 ibdev_err(ibdev, 830 "failed to map user SQ doorbell, ret = %d.\n", 831 ret); 832 goto err_out; 833 } 834 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB; 835 } 836 837 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) { 838 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb); 839 if (ret) { 840 ibdev_err(ibdev, 841 "failed to map user RQ doorbell, ret = %d.\n", 842 ret); 843 goto err_sdb; 844 } 845 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 846 } 847 848 return 0; 849 850 err_sdb: 851 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 852 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 853 err_out: 854 return ret; 855 } 856 857 static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev, 858 struct hns_roce_qp *hr_qp, 859 struct ib_qp_init_attr *init_attr) 860 { 861 struct ib_device *ibdev = &hr_dev->ib_dev; 862 int ret; 863 864 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) 865 hr_qp->sq.db_reg = hr_dev->mem_base + 866 HNS_ROCE_DWQE_SIZE * hr_qp->qpn; 867 else 868 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset + 869 DB_REG_OFFSET * hr_dev->priv_uar.index; 870 871 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset + 872 DB_REG_OFFSET * hr_dev->priv_uar.index; 873 874 if (kernel_qp_has_rdb(hr_dev, init_attr)) { 875 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0); 876 if (ret) { 877 ibdev_err(ibdev, 878 "failed to alloc kernel RQ doorbell, ret = %d.\n", 879 ret); 880 return ret; 881 } 882 *hr_qp->rdb.db_record = 0; 883 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB; 884 } 885 886 return 0; 887 } 888 889 static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 890 struct ib_qp_init_attr *init_attr, 891 struct ib_udata *udata, 892 struct hns_roce_ib_create_qp *ucmd, 893 struct hns_roce_ib_create_qp_resp *resp) 894 { 895 int ret; 896 897 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE) 898 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB; 899 900 if (udata) { 901 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) { 902 ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp); 903 if (ret) 904 return ret; 905 } 906 907 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd, 908 resp); 909 if (ret) 910 goto err_remove_qp; 911 } else { 912 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr); 913 if (ret) 914 return ret; 915 } 916 917 return 0; 918 919 err_remove_qp: 920 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) 921 qp_user_mmap_entry_remove(hr_qp); 922 923 return ret; 924 } 925 926 static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 927 struct ib_udata *udata) 928 { 929 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( 930 udata, struct hns_roce_ucontext, ibucontext); 931 932 if (udata) { 933 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 934 hns_roce_db_unmap_user(uctx, &hr_qp->rdb); 935 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) 936 hns_roce_db_unmap_user(uctx, &hr_qp->sdb); 937 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) 938 qp_user_mmap_entry_remove(hr_qp); 939 } else { 940 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 941 hns_roce_free_db(hr_dev, &hr_qp->rdb); 942 } 943 } 944 945 static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev, 946 struct hns_roce_qp *hr_qp) 947 { 948 struct ib_device *ibdev = &hr_dev->ib_dev; 949 u64 *sq_wrid = NULL; 950 u64 *rq_wrid = NULL; 951 int ret; 952 953 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL); 954 if (ZERO_OR_NULL_PTR(sq_wrid)) { 955 ibdev_err(ibdev, "failed to alloc SQ wrid.\n"); 956 return -ENOMEM; 957 } 958 959 if (hr_qp->rq.wqe_cnt) { 960 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL); 961 if (ZERO_OR_NULL_PTR(rq_wrid)) { 962 ibdev_err(ibdev, "failed to alloc RQ wrid.\n"); 963 ret = -ENOMEM; 964 goto err_sq; 965 } 966 } 967 968 hr_qp->sq.wrid = sq_wrid; 969 hr_qp->rq.wrid = rq_wrid; 970 return 0; 971 err_sq: 972 kfree(sq_wrid); 973 974 return ret; 975 } 976 977 static void free_kernel_wrid(struct hns_roce_qp *hr_qp) 978 { 979 kfree(hr_qp->rq.wrid); 980 kfree(hr_qp->sq.wrid); 981 } 982 983 static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 984 struct ib_qp_init_attr *init_attr, 985 struct ib_udata *udata, 986 struct hns_roce_ib_create_qp *ucmd) 987 { 988 struct ib_device *ibdev = &hr_dev->ib_dev; 989 int ret; 990 991 if (init_attr->cap.max_inline_data > hr_dev->caps.max_sq_inline) 992 init_attr->cap.max_inline_data = hr_dev->caps.max_sq_inline; 993 994 hr_qp->max_inline_data = init_attr->cap.max_inline_data; 995 996 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) 997 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR; 998 else 999 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR; 1000 1001 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp, 1002 hns_roce_qp_has_rq(init_attr), !!udata); 1003 if (ret) { 1004 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n", 1005 ret); 1006 return ret; 1007 } 1008 1009 if (udata) { 1010 ret = ib_copy_from_udata(ucmd, udata, 1011 min(udata->inlen, sizeof(*ucmd))); 1012 if (ret) { 1013 ibdev_err(ibdev, 1014 "failed to copy QP ucmd, ret = %d\n", ret); 1015 return ret; 1016 } 1017 1018 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd); 1019 if (ret) 1020 ibdev_err(ibdev, 1021 "failed to set user SQ size, ret = %d.\n", 1022 ret); 1023 } else { 1024 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp); 1025 if (ret) 1026 ibdev_err(ibdev, 1027 "failed to set kernel SQ size, ret = %d.\n", 1028 ret); 1029 } 1030 1031 return ret; 1032 } 1033 1034 static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, 1035 struct ib_pd *ib_pd, 1036 struct ib_qp_init_attr *init_attr, 1037 struct ib_udata *udata, 1038 struct hns_roce_qp *hr_qp) 1039 { 1040 struct hns_roce_ib_create_qp_resp resp = {}; 1041 struct ib_device *ibdev = &hr_dev->ib_dev; 1042 struct hns_roce_ib_create_qp ucmd; 1043 int ret; 1044 1045 mutex_init(&hr_qp->mutex); 1046 spin_lock_init(&hr_qp->sq.lock); 1047 spin_lock_init(&hr_qp->rq.lock); 1048 1049 hr_qp->state = IB_QPS_RESET; 1050 hr_qp->flush_flag = 0; 1051 1052 if (init_attr->create_flags) 1053 return -EOPNOTSUPP; 1054 1055 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd); 1056 if (ret) { 1057 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret); 1058 return ret; 1059 } 1060 1061 if (!udata) { 1062 ret = alloc_kernel_wrid(hr_dev, hr_qp); 1063 if (ret) { 1064 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n", 1065 ret); 1066 return ret; 1067 } 1068 } 1069 1070 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr); 1071 if (ret) { 1072 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret); 1073 goto err_buf; 1074 } 1075 1076 ret = alloc_qpn(hr_dev, hr_qp); 1077 if (ret) { 1078 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret); 1079 goto err_qpn; 1080 } 1081 1082 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp); 1083 if (ret) { 1084 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n", 1085 ret); 1086 goto err_db; 1087 } 1088 1089 ret = alloc_qpc(hr_dev, hr_qp); 1090 if (ret) { 1091 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n", 1092 ret); 1093 goto err_qpc; 1094 } 1095 1096 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr); 1097 if (ret) { 1098 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret); 1099 goto err_store; 1100 } 1101 1102 if (udata) { 1103 resp.cap_flags = hr_qp->en_flags; 1104 ret = ib_copy_to_udata(udata, &resp, 1105 min(udata->outlen, sizeof(resp))); 1106 if (ret) { 1107 ibdev_err(ibdev, "copy qp resp failed!\n"); 1108 goto err_store; 1109 } 1110 } 1111 1112 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) { 1113 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp); 1114 if (ret) 1115 goto err_flow_ctrl; 1116 } 1117 1118 hr_qp->ibqp.qp_num = hr_qp->qpn; 1119 hr_qp->event = hns_roce_ib_qp_event; 1120 refcount_set(&hr_qp->refcount, 1); 1121 init_completion(&hr_qp->free); 1122 1123 return 0; 1124 1125 err_flow_ctrl: 1126 hns_roce_qp_remove(hr_dev, hr_qp); 1127 err_store: 1128 free_qpc(hr_dev, hr_qp); 1129 err_qpc: 1130 free_qp_db(hr_dev, hr_qp, udata); 1131 err_db: 1132 free_qpn(hr_dev, hr_qp); 1133 err_qpn: 1134 free_qp_buf(hr_dev, hr_qp); 1135 err_buf: 1136 free_kernel_wrid(hr_qp); 1137 return ret; 1138 } 1139 1140 void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, 1141 struct ib_udata *udata) 1142 { 1143 if (refcount_dec_and_test(&hr_qp->refcount)) 1144 complete(&hr_qp->free); 1145 wait_for_completion(&hr_qp->free); 1146 1147 free_qpc(hr_dev, hr_qp); 1148 free_qpn(hr_dev, hr_qp); 1149 free_qp_buf(hr_dev, hr_qp); 1150 free_kernel_wrid(hr_qp); 1151 free_qp_db(hr_dev, hr_qp, udata); 1152 } 1153 1154 static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type, 1155 bool is_user) 1156 { 1157 switch (type) { 1158 case IB_QPT_XRC_INI: 1159 case IB_QPT_XRC_TGT: 1160 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC)) 1161 goto out; 1162 break; 1163 case IB_QPT_UD: 1164 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 && 1165 is_user) 1166 goto out; 1167 break; 1168 case IB_QPT_RC: 1169 case IB_QPT_GSI: 1170 break; 1171 default: 1172 goto out; 1173 } 1174 1175 return 0; 1176 1177 out: 1178 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type); 1179 1180 return -EOPNOTSUPP; 1181 } 1182 1183 int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, 1184 struct ib_udata *udata) 1185 { 1186 struct ib_device *ibdev = qp->device; 1187 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev); 1188 struct hns_roce_qp *hr_qp = to_hr_qp(qp); 1189 struct ib_pd *pd = qp->pd; 1190 int ret; 1191 1192 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata); 1193 if (ret) 1194 return ret; 1195 1196 if (init_attr->qp_type == IB_QPT_XRC_TGT) 1197 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn; 1198 1199 if (init_attr->qp_type == IB_QPT_GSI) { 1200 hr_qp->port = init_attr->port_num - 1; 1201 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port]; 1202 } 1203 1204 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); 1205 if (ret) 1206 ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n", 1207 init_attr->qp_type, ret); 1208 1209 return ret; 1210 } 1211 1212 int to_hr_qp_type(int qp_type) 1213 { 1214 switch (qp_type) { 1215 case IB_QPT_RC: 1216 return SERV_TYPE_RC; 1217 case IB_QPT_UD: 1218 case IB_QPT_GSI: 1219 return SERV_TYPE_UD; 1220 case IB_QPT_XRC_INI: 1221 case IB_QPT_XRC_TGT: 1222 return SERV_TYPE_XRC; 1223 default: 1224 return -1; 1225 } 1226 } 1227 1228 static int check_mtu_validate(struct hns_roce_dev *hr_dev, 1229 struct hns_roce_qp *hr_qp, 1230 struct ib_qp_attr *attr, int attr_mask) 1231 { 1232 enum ib_mtu active_mtu; 1233 int p; 1234 1235 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1236 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu); 1237 1238 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 && 1239 attr->path_mtu > hr_dev->caps.max_mtu) || 1240 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) { 1241 ibdev_err(&hr_dev->ib_dev, 1242 "attr path_mtu(%d)invalid while modify qp", 1243 attr->path_mtu); 1244 return -EINVAL; 1245 } 1246 1247 return 0; 1248 } 1249 1250 static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1251 int attr_mask) 1252 { 1253 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1254 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1255 int p; 1256 1257 if ((attr_mask & IB_QP_PORT) && 1258 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) { 1259 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n", 1260 attr->port_num); 1261 return -EINVAL; 1262 } 1263 1264 if (attr_mask & IB_QP_PKEY_INDEX) { 1265 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port; 1266 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) { 1267 ibdev_err(&hr_dev->ib_dev, 1268 "invalid attr, pkey_index = %u.\n", 1269 attr->pkey_index); 1270 return -EINVAL; 1271 } 1272 } 1273 1274 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && 1275 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) { 1276 ibdev_err(&hr_dev->ib_dev, 1277 "invalid attr, max_rd_atomic = %u.\n", 1278 attr->max_rd_atomic); 1279 return -EINVAL; 1280 } 1281 1282 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && 1283 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) { 1284 ibdev_err(&hr_dev->ib_dev, 1285 "invalid attr, max_dest_rd_atomic = %u.\n", 1286 attr->max_dest_rd_atomic); 1287 return -EINVAL; 1288 } 1289 1290 if (attr_mask & IB_QP_PATH_MTU) 1291 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask); 1292 1293 return 0; 1294 } 1295 1296 int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1297 int attr_mask, struct ib_udata *udata) 1298 { 1299 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); 1300 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); 1301 enum ib_qp_state cur_state, new_state; 1302 int ret = -EINVAL; 1303 1304 mutex_lock(&hr_qp->mutex); 1305 1306 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state) 1307 goto out; 1308 1309 cur_state = hr_qp->state; 1310 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; 1311 1312 if (ibqp->uobject && 1313 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) { 1314 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) { 1315 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr); 1316 1317 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB) 1318 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr); 1319 } else { 1320 ibdev_warn(&hr_dev->ib_dev, 1321 "flush cqe is not supported in userspace!\n"); 1322 goto out; 1323 } 1324 } 1325 1326 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, 1327 attr_mask)) { 1328 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n"); 1329 goto out; 1330 } 1331 1332 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask); 1333 if (ret) 1334 goto out; 1335 1336 if (cur_state == new_state && cur_state == IB_QPS_RESET) 1337 goto out; 1338 1339 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state, 1340 new_state); 1341 1342 out: 1343 mutex_unlock(&hr_qp->mutex); 1344 1345 return ret; 1346 } 1347 1348 void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) 1349 __acquires(&send_cq->lock) __acquires(&recv_cq->lock) 1350 { 1351 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1352 __acquire(&send_cq->lock); 1353 __acquire(&recv_cq->lock); 1354 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1355 spin_lock_irq(&send_cq->lock); 1356 __acquire(&recv_cq->lock); 1357 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1358 spin_lock_irq(&recv_cq->lock); 1359 __acquire(&send_cq->lock); 1360 } else if (send_cq == recv_cq) { 1361 spin_lock_irq(&send_cq->lock); 1362 __acquire(&recv_cq->lock); 1363 } else if (send_cq->cqn < recv_cq->cqn) { 1364 spin_lock_irq(&send_cq->lock); 1365 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); 1366 } else { 1367 spin_lock_irq(&recv_cq->lock); 1368 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); 1369 } 1370 } 1371 1372 void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, 1373 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) 1374 __releases(&recv_cq->lock) 1375 { 1376 if (unlikely(send_cq == NULL && recv_cq == NULL)) { 1377 __release(&recv_cq->lock); 1378 __release(&send_cq->lock); 1379 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) { 1380 __release(&recv_cq->lock); 1381 spin_unlock(&send_cq->lock); 1382 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) { 1383 __release(&send_cq->lock); 1384 spin_unlock(&recv_cq->lock); 1385 } else if (send_cq == recv_cq) { 1386 __release(&recv_cq->lock); 1387 spin_unlock_irq(&send_cq->lock); 1388 } else if (send_cq->cqn < recv_cq->cqn) { 1389 spin_unlock(&recv_cq->lock); 1390 spin_unlock_irq(&send_cq->lock); 1391 } else { 1392 spin_unlock(&send_cq->lock); 1393 spin_unlock_irq(&recv_cq->lock); 1394 } 1395 } 1396 1397 static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset) 1398 { 1399 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset); 1400 } 1401 1402 void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1403 { 1404 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); 1405 } 1406 1407 void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n) 1408 { 1409 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); 1410 } 1411 1412 void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n) 1413 { 1414 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); 1415 } 1416 1417 bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq, 1418 struct ib_cq *ib_cq) 1419 { 1420 struct hns_roce_cq *hr_cq; 1421 u32 cur; 1422 1423 cur = hr_wq->head - hr_wq->tail; 1424 if (likely(cur + nreq < hr_wq->wqe_cnt)) 1425 return false; 1426 1427 hr_cq = to_hr_cq(ib_cq); 1428 spin_lock(&hr_cq->lock); 1429 cur = hr_wq->head - hr_wq->tail; 1430 spin_unlock(&hr_cq->lock); 1431 1432 return cur + nreq >= hr_wq->wqe_cnt; 1433 } 1434 1435 int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) 1436 { 1437 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; 1438 unsigned int reserved_from_bot; 1439 unsigned int i; 1440 1441 qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps, 1442 sizeof(u32), GFP_KERNEL); 1443 if (!qp_table->idx_table.spare_idx) 1444 return -ENOMEM; 1445 1446 mutex_init(&qp_table->scc_mutex); 1447 mutex_init(&qp_table->bank_mutex); 1448 xa_init(&hr_dev->qp_table_xa); 1449 1450 reserved_from_bot = hr_dev->caps.reserved_qps; 1451 1452 for (i = 0; i < reserved_from_bot; i++) { 1453 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++; 1454 hr_dev->qp_table.bank[get_qp_bankid(i)].min++; 1455 } 1456 1457 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) { 1458 ida_init(&hr_dev->qp_table.bank[i].ida); 1459 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps / 1460 HNS_ROCE_QP_BANK_NUM - 1; 1461 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min; 1462 } 1463 1464 return 0; 1465 } 1466 1467 void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev) 1468 { 1469 int i; 1470 1471 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) 1472 ida_destroy(&hr_dev->qp_table.bank[i].ida); 1473 kfree(hr_dev->qp_table.idx_table.spare_idx); 1474 } 1475