1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2015 - 2023 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "irdma_main.h" 36 37 LIST_HEAD(irdma_handlers); 38 DEFINE_SPINLOCK(irdma_handler_lock); 39 40 static const struct ae_desc ae_desc_list[] = { 41 {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"}, 42 {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"}, 43 {IRDMA_AE_AMP_BAD_QP, 44 "Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"}, 45 {IRDMA_AE_AMP_BAD_PD, 46 "Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"}, 47 {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"}, 48 {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"}, 49 {IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"}, 50 {IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"}, 51 {IRDMA_AE_AMP_TO_WRAP, 52 "Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"}, 53 {IRDMA_AE_AMP_FASTREG_VALID_STAG, 54 "Fastreg error: Registration to a valid MR"}, 55 {IRDMA_AE_AMP_FASTREG_MW_STAG, 56 "Fastreg error: Registration to a valid Memory Window (MW)"}, 57 {IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"}, 58 {IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"}, 59 {IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"}, 60 {IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS, 61 "Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"}, 62 {IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS, 63 "Attempt to invalidate MR with a bound Memory Window (MW)"}, 64 {IRDMA_AE_AMP_MWBIND_VALID_STAG, 65 "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"}, 66 {IRDMA_AE_AMP_MWBIND_OF_MR_STAG, 67 "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"}, 68 {IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG, 69 "Attempt to bind an Memory Window (MW) to a zero based MR"}, 70 {IRDMA_AE_AMP_MWBIND_TO_MW_STAG, 71 "Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"}, 72 {IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"}, 73 {IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"}, 74 {IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT, 75 "Memory Window (MW) bind error: Invalid parent MR"}, 76 {IRDMA_AE_AMP_MWBIND_BIND_DISABLED, 77 "Memory Window (MW) bind error: Disabled bind support"}, 78 {IRDMA_AE_PRIV_OPERATION_DENIED, 79 "Denying a privileged operation on a non-privileged QP"}, 80 {IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"}, 81 {IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW, 82 "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"}, 83 {IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG, 84 "Fastreg error: Invalid host page size config"}, 85 {IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"}, 86 {IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH, 87 "Fastreg error: Invalid request to change physical MR to virtual or vice versa"}, 88 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG, 89 "Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"}, 90 {IRDMA_AE_UDA_XMIT_BAD_PD, 91 "Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"}, 92 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT, 93 "Userspace Direct Access (UDA) QP xmit error: Too short packet length"}, 94 {IRDMA_AE_UDA_L4LEN_INVALID, 95 "Userspace Direct Access (UDA) error: Invalid packet length field"}, 96 {IRDMA_AE_BAD_CLOSE, 97 "iWARP error: Data is received when QP state is closing"}, 98 {IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE, 99 "iWARP error: FIN is received when xmit data is pending"}, 100 {IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"}, 101 {IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO, 102 "QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"}, 103 {IRDMA_AE_STAG_ZERO_INVALID, 104 "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"}, 105 {IRDMA_AE_IB_RREQ_AND_Q1_FULL, 106 "QP error: Received RDMA Read request when the inbound RDMA Read queue is full"}, 107 {IRDMA_AE_IB_INVALID_REQUEST, 108 "QP error: Invalid operation detected by the remote peer"}, 109 {IRDMA_AE_WQE_UNEXPECTED_OPCODE, 110 "QP error: Invalid opcode in SQ WQE"}, 111 {IRDMA_AE_WQE_INVALID_PARAMETER, 112 "QP error: Invalid parameter in a WQE"}, 113 {IRDMA_AE_WQE_INVALID_FRAG_DATA, 114 "QP error: Invalid fragment in a WQE"}, 115 {IRDMA_AE_IB_REMOTE_ACCESS_ERROR, 116 "RoCEv2 error: Remote access error"}, 117 {IRDMA_AE_IB_REMOTE_OP_ERROR, 118 "RoCEv2 error: Remote operation error"}, 119 {IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"}, 120 {IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN, 121 "iWARP error: Invalid message sequence number"}, 122 {IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER, 123 "iWARP error: Inbound message is too long for the available buffer"}, 124 {IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"}, 125 {IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"}, 126 {IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE, 127 "iWARP error: Inbound Send message when no receive buffer is available"}, 128 {IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"}, 129 {IRDMA_AE_DDP_NO_L_BIT, 130 "iWARP error: Last bit not set in an inbound packet which completes RDMA Read"}, 131 {IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"}, 132 {IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"}, 133 {IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"}, 134 {IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP, 135 "Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"}, 136 {IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"}, 137 {IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"}, 138 {IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"}, 139 {IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"}, 140 {IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"}, 141 {IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"}, 142 {IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"}, 143 {IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"}, 144 {IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"}, 145 {IRDMA_AE_LLP_CLOSE_COMPLETE, 146 "iWARP event: Graceful close complete"}, 147 {IRDMA_AE_LLP_CONNECTION_RESET, 148 "iWARP event: Received a TCP packet with a RST bit set"}, 149 {IRDMA_AE_LLP_FIN_RECEIVED, 150 "iWARP event: Received a TCP packet with a FIN bit set"}, 151 {IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH, 152 "iWARP error: Unable to close a gap in the TCP sequence"}, 153 {IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"}, 154 {IRDMA_AE_LLP_SEGMENT_TOO_SMALL, 155 "iWARP error: Received a packet with insufficient space for protocol headers"}, 156 {IRDMA_AE_LLP_SYN_RECEIVED, 157 "iWARP event: Received a TCP packet with a SYN bit set"}, 158 {IRDMA_AE_LLP_TERMINATE_RECEIVED, 159 "iWARP error: Received a terminate message"}, 160 {IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"}, 161 {IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES, 162 "Connection error: The max number of keepalive retries has been reached"}, 163 {IRDMA_AE_LLP_DOUBT_REACHABILITY, 164 "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"}, 165 {IRDMA_AE_LLP_CONNECTION_ESTABLISHED, 166 "iWARP event: Connection established"}, 167 {IRDMA_AE_RESOURCE_EXHAUSTION, 168 "QP error: Resource exhaustion"}, 169 {IRDMA_AE_RESET_SENT, 170 "Reset sent (as requested via Modify QP)"}, 171 {IRDMA_AE_TERMINATE_SENT, 172 "Terminate sent (as requested via Modify QP)"}, 173 {IRDMA_AE_RESET_NOT_SENT, 174 "Reset not sent (but requested via Modify QP)"}, 175 {IRDMA_AE_LCE_QP_CATASTROPHIC, 176 "QP error: HW transaction resulted in catastrophic error"}, 177 {IRDMA_AE_LCE_FUNCTION_CATASTROPHIC, 178 "PCIe function error: HW transaction resulted in catastrophic error"}, 179 {IRDMA_AE_LCE_CQ_CATASTROPHIC, 180 "CQ error: HW transaction resulted in catastrophic error"}, 181 {IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"}, 182 }; 183 184 /** 185 * irdma_get_ae_desc - returns AE description 186 * @ae_id: the AE number 187 */ 188 const char * 189 irdma_get_ae_desc(u16 ae_id) 190 { 191 const char *desc = ""; 192 int i; 193 194 for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) { 195 if (ae_desc_list[i].id == ae_id) { 196 desc = ae_desc_list[i].desc; 197 break; 198 } 199 } 200 return desc; 201 } 202 203 /** 204 * irdma_arp_table -manage arp table 205 * @rf: RDMA PCI function 206 * @ip_addr: ip address for device 207 * @mac_addr: mac address ptr 208 * @action: modify, delete or add 209 */ 210 int 211 irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, const u8 *mac_addr, 212 u32 action) 213 { 214 unsigned long flags; 215 int arp_index; 216 u32 ip[4] = {}; 217 218 memcpy(ip, ip_addr, sizeof(ip)); 219 220 spin_lock_irqsave(&rf->arp_lock, flags); 221 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) { 222 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip))) 223 break; 224 } 225 226 switch (action) { 227 case IRDMA_ARP_ADD: 228 if (arp_index != rf->arp_table_size) { 229 arp_index = -1; 230 break; 231 } 232 233 arp_index = 0; 234 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size, 235 (u32 *)&arp_index, &rf->next_arp_index)) { 236 arp_index = -1; 237 break; 238 } 239 240 memcpy(rf->arp_table[arp_index].ip_addr, ip, 241 sizeof(rf->arp_table[arp_index].ip_addr)); 242 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr); 243 break; 244 case IRDMA_ARP_RESOLVE: 245 if (arp_index == rf->arp_table_size) 246 arp_index = -1; 247 break; 248 case IRDMA_ARP_DELETE: 249 if (arp_index == rf->arp_table_size) { 250 arp_index = -1; 251 break; 252 } 253 254 memset(rf->arp_table[arp_index].ip_addr, 0, 255 sizeof(rf->arp_table[arp_index].ip_addr)); 256 eth_zero_addr(rf->arp_table[arp_index].mac_addr); 257 irdma_free_rsrc(rf, rf->allocated_arps, arp_index); 258 break; 259 default: 260 arp_index = -1; 261 break; 262 } 263 264 spin_unlock_irqrestore(&rf->arp_lock, flags); 265 return arp_index; 266 } 267 268 /** 269 * irdma_add_arp - add a new arp entry if needed 270 * @rf: RDMA function 271 * @ip: IP address 272 * @mac: MAC address 273 */ 274 int 275 irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac) 276 { 277 int arpidx; 278 279 arpidx = irdma_arp_table(rf, &ip[0], NULL, IRDMA_ARP_RESOLVE); 280 if (arpidx >= 0) { 281 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac)) 282 return arpidx; 283 284 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip, 285 IRDMA_ARP_DELETE); 286 } 287 288 irdma_manage_arp_cache(rf, mac, ip, IRDMA_ARP_ADD); 289 290 return irdma_arp_table(rf, ip, NULL, IRDMA_ARP_RESOLVE); 291 } 292 293 /** 294 * irdma_netdevice_event - system notifier for netdev events 295 * @notifier: not used 296 * @event: event for notifier 297 * @ptr: netdev 298 */ 299 int 300 irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, 301 void *ptr) 302 { 303 struct irdma_device *iwdev; 304 struct ifnet *netdev = netdev_notifier_info_to_ifp(ptr); 305 306 iwdev = container_of(notifier, struct irdma_device, nb_netdevice_event); 307 if (iwdev->netdev != netdev) 308 return NOTIFY_DONE; 309 310 iwdev->iw_status = 1; 311 switch (event) { 312 case NETDEV_DOWN: 313 iwdev->iw_status = 0; 314 /* fallthrough */ 315 case NETDEV_UP: 316 irdma_port_ibevent(iwdev); 317 break; 318 default: 319 break; 320 } 321 322 return NOTIFY_DONE; 323 } 324 325 void 326 irdma_unregister_notifiers(struct irdma_device *iwdev) 327 { 328 unregister_netdevice_notifier(&iwdev->nb_netdevice_event); 329 } 330 331 int 332 irdma_register_notifiers(struct irdma_device *iwdev) 333 { 334 int ret; 335 336 iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event; 337 ret = register_netdevice_notifier(&iwdev->nb_netdevice_event); 338 if (ret) { 339 irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n"); 340 return ret; 341 } 342 return ret; 343 } 344 /** 345 * irdma_alloc_and_get_cqp_request - get cqp struct 346 * @cqp: device cqp ptr 347 * @wait: cqp to be used in wait mode 348 */ 349 struct irdma_cqp_request * 350 irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, 351 bool wait) 352 { 353 struct irdma_cqp_request *cqp_request = NULL; 354 unsigned long flags; 355 356 spin_lock_irqsave(&cqp->req_lock, flags); 357 if (!list_empty(&cqp->cqp_avail_reqs)) { 358 cqp_request = list_entry(cqp->cqp_avail_reqs.next, 359 struct irdma_cqp_request, list); 360 list_del_init(&cqp_request->list); 361 } 362 spin_unlock_irqrestore(&cqp->req_lock, flags); 363 if (!cqp_request) { 364 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC); 365 if (cqp_request) { 366 cqp_request->dynamic = true; 367 if (wait) 368 init_waitqueue_head(&cqp_request->waitq); 369 } 370 } 371 if (!cqp_request) { 372 irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory"); 373 return NULL; 374 } 375 376 cqp_request->waiting = wait; 377 atomic_set(&cqp_request->refcnt, 1); 378 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info)); 379 380 return cqp_request; 381 } 382 383 /** 384 * irdma_get_cqp_request - increase refcount for cqp_request 385 * @cqp_request: pointer to cqp_request instance 386 */ 387 static inline void 388 irdma_get_cqp_request(struct irdma_cqp_request *cqp_request) 389 { 390 atomic_inc(&cqp_request->refcnt); 391 } 392 393 /** 394 * irdma_free_cqp_request - free cqp request 395 * @cqp: cqp ptr 396 * @cqp_request: to be put back in cqp list 397 */ 398 void 399 irdma_free_cqp_request(struct irdma_cqp *cqp, 400 struct irdma_cqp_request *cqp_request) 401 { 402 unsigned long flags; 403 404 if (cqp_request->dynamic) { 405 kfree(cqp_request); 406 } else { 407 WRITE_ONCE(cqp_request->request_done, false); 408 cqp_request->callback_fcn = NULL; 409 cqp_request->waiting = false; 410 411 spin_lock_irqsave(&cqp->req_lock, flags); 412 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs); 413 spin_unlock_irqrestore(&cqp->req_lock, flags); 414 } 415 wake_up(&cqp->remove_wq); 416 } 417 418 /** 419 * irdma_put_cqp_request - dec ref count and free if 0 420 * @cqp: cqp ptr 421 * @cqp_request: to be put back in cqp list 422 */ 423 void 424 irdma_put_cqp_request(struct irdma_cqp *cqp, 425 struct irdma_cqp_request *cqp_request) 426 { 427 if (atomic_dec_and_test(&cqp_request->refcnt)) 428 irdma_free_cqp_request(cqp, cqp_request); 429 } 430 431 /** 432 * irdma_free_pending_cqp_request -free pending cqp request objs 433 * @cqp: cqp ptr 434 * @cqp_request: to be put back in cqp list 435 */ 436 static void 437 irdma_free_pending_cqp_request(struct irdma_cqp *cqp, 438 struct irdma_cqp_request *cqp_request) 439 { 440 if (cqp_request->waiting) { 441 cqp_request->compl_info.error = true; 442 WRITE_ONCE(cqp_request->request_done, true); 443 wake_up(&cqp_request->waitq); 444 } 445 wait_event_timeout(cqp->remove_wq, 446 atomic_read(&cqp_request->refcnt) == 1, 1000); 447 irdma_put_cqp_request(cqp, cqp_request); 448 } 449 450 /** 451 * irdma_cleanup_pending_cqp_op - clean-up cqp with no 452 * completions 453 * @rf: RDMA PCI function 454 */ 455 void 456 irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf) 457 { 458 struct irdma_sc_dev *dev = &rf->sc_dev; 459 struct irdma_cqp *cqp = &rf->cqp; 460 struct irdma_cqp_request *cqp_request = NULL; 461 struct cqp_cmds_info *pcmdinfo = NULL; 462 u32 i, pending_work, wqe_idx; 463 464 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring); 465 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring); 466 for (i = 0; i < pending_work; i++) { 467 cqp_request = (struct irdma_cqp_request *)(uintptr_t) 468 cqp->scratch_array[wqe_idx]; 469 if (cqp_request) 470 irdma_free_pending_cqp_request(cqp, cqp_request); 471 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring); 472 } 473 474 while (!list_empty(&dev->cqp_cmd_head)) { 475 pcmdinfo = irdma_remove_cqp_head(dev); 476 cqp_request = 477 container_of(pcmdinfo, struct irdma_cqp_request, info); 478 if (cqp_request) 479 irdma_free_pending_cqp_request(cqp, cqp_request); 480 } 481 } 482 483 /** 484 * irdma_wait_event - wait for completion 485 * @rf: RDMA PCI function 486 * @cqp_request: cqp request to wait 487 */ 488 static int 489 irdma_wait_event(struct irdma_pci_f *rf, 490 struct irdma_cqp_request *cqp_request) 491 { 492 struct irdma_cqp_timeout cqp_timeout = {0}; 493 bool cqp_error = false; 494 int err_code = 0; 495 496 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); 497 do { 498 int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms; 499 500 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 501 if (wait_event_timeout(cqp_request->waitq, 502 READ_ONCE(cqp_request->request_done), 503 msecs_to_jiffies(wait_time_ms))) 504 break; 505 506 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); 507 508 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) 509 continue; 510 511 if (!rf->reset) { 512 rf->reset = true; 513 rf->gen_ops.request_reset(rf); 514 } 515 return -ETIMEDOUT; 516 } while (1); 517 518 cqp_error = cqp_request->compl_info.error; 519 if (cqp_error) { 520 err_code = -EIO; 521 if (cqp_request->compl_info.maj_err_code == 0xFFFF) { 522 if (cqp_request->compl_info.min_err_code == 0x8002) { 523 err_code = -EBUSY; 524 } else if (cqp_request->compl_info.min_err_code == 0x8029) { 525 if (!rf->reset) { 526 rf->reset = true; 527 rf->gen_ops.request_reset(rf); 528 } 529 } 530 } 531 } 532 533 return err_code; 534 } 535 536 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = { 537 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd", 538 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd", 539 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd", 540 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd", 541 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd", 542 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd", 543 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd", 544 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd", 545 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd", 546 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd", 547 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd", 548 [IRDMA_OP_QP_CREATE] = "Create QP Cmd", 549 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd", 550 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd", 551 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd", 552 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd", 553 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd", 554 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd", 555 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd", 556 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd", 557 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd", 558 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", 559 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", 560 [IRDMA_OP_RESUME] = "Resume QP Cmd", 561 [IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] = 562 "Manage Virtual Channel Requester Function PBLE Backing Pages Cmd", 563 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", 564 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", 565 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", 566 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd", 567 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd", 568 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd", 569 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd", 570 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd", 571 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd", 572 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd", 573 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd", 574 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", 575 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", 576 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", 577 [IRDMA_OP_WS_FAILOVER_START] = "Failover Start Cmd", 578 [IRDMA_OP_WS_FAILOVER_COMPLETE] = "Failover Complete Cmd", 579 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", 580 [IRDMA_OP_GEN_AE] = "Generate AE Cmd", 581 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", 582 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd", 583 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd", 584 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd", 585 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd", 586 }; 587 588 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = { 589 {0xffff, 0x8002, "Invalid State"}, 590 {0xffff, 0x8006, "Flush No Wqe Pending"}, 591 {0xffff, 0x8007, "Modify QP Bad Close"}, 592 {0xffff, 0x8009, "LLP Closed"}, 593 {0xffff, 0x800a, "Reset Not Sent"}, 594 {0xffff, 0x200, "Failover Pending"} 595 }; 596 597 /** 598 * irdma_cqp_crit_err - check if CQP error is critical 599 * @dev: pointer to dev structure 600 * @cqp_cmd: code for last CQP operation 601 * @maj_err_code: major error code 602 * @min_err_code: minot error code 603 */ 604 bool 605 irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd, 606 u16 maj_err_code, u16 min_err_code) 607 { 608 int i; 609 610 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) { 611 if (maj_err_code == irdma_noncrit_err_list[i].maj && 612 min_err_code == irdma_noncrit_err_list[i].min) { 613 irdma_debug(dev, IRDMA_DEBUG_CQP, 614 "[%s Error][%s] maj=0x%x min=0x%x\n", 615 irdma_noncrit_err_list[i].desc, 616 irdma_cqp_cmd_names[cqp_cmd], maj_err_code, 617 min_err_code); 618 return false; 619 } 620 } 621 return true; 622 } 623 624 /** 625 * irdma_handle_cqp_op - process cqp command 626 * @rf: RDMA PCI function 627 * @cqp_request: cqp request to process 628 */ 629 int 630 irdma_handle_cqp_op(struct irdma_pci_f *rf, 631 struct irdma_cqp_request *cqp_request) 632 { 633 struct irdma_sc_dev *dev = &rf->sc_dev; 634 struct cqp_cmds_info *info = &cqp_request->info; 635 int status; 636 bool put_cqp_request = true; 637 638 if (rf->reset) 639 return 0; 640 641 irdma_get_cqp_request(cqp_request); 642 status = irdma_process_cqp_cmd(dev, info); 643 if (status) 644 goto err; 645 646 if (cqp_request->waiting) { 647 put_cqp_request = false; 648 status = irdma_wait_event(rf, cqp_request); 649 if (status) 650 goto err; 651 } 652 653 return 0; 654 655 err: 656 if (irdma_cqp_crit_err(dev, info->cqp_cmd, 657 cqp_request->compl_info.maj_err_code, 658 cqp_request->compl_info.min_err_code)) 659 irdma_dev_err(&rf->iwdev->ibdev, 660 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", 661 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, 662 cqp_request->waiting, cqp_request->compl_info.error, 663 cqp_request->compl_info.maj_err_code, 664 cqp_request->compl_info.min_err_code); 665 666 if (put_cqp_request) 667 irdma_put_cqp_request(&rf->cqp, cqp_request); 668 669 return status; 670 } 671 672 void 673 irdma_qp_add_ref(struct ib_qp *ibqp) 674 { 675 struct irdma_qp *iwqp = to_iwqp(ibqp); 676 677 atomic_inc(&iwqp->refcnt); 678 } 679 680 void 681 irdma_qp_rem_ref(struct ib_qp *ibqp) 682 { 683 struct irdma_qp *iwqp = to_iwqp(ibqp); 684 struct irdma_device *iwdev = iwqp->iwdev; 685 unsigned long flags; 686 687 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); 688 if (!atomic_dec_and_test(&iwqp->refcnt)) { 689 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 690 return; 691 } 692 693 iwdev->rf->qp_table[iwqp->ibqp.qp_num] = NULL; 694 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); 695 complete(&iwqp->free_qp); 696 } 697 698 void 699 irdma_cq_add_ref(struct ib_cq *ibcq) 700 { 701 struct irdma_cq *iwcq = to_iwcq(ibcq); 702 703 atomic_inc(&iwcq->refcnt); 704 } 705 706 void 707 irdma_cq_rem_ref(struct ib_cq *ibcq) 708 { 709 struct irdma_cq *iwcq = to_iwcq(ibcq); 710 struct irdma_pci_f *rf = container_of(iwcq->sc_cq.dev, struct irdma_pci_f, sc_dev); 711 unsigned long flags; 712 713 spin_lock_irqsave(&rf->cqtable_lock, flags); 714 if (!atomic_dec_and_test(&iwcq->refcnt)) { 715 spin_unlock_irqrestore(&rf->cqtable_lock, flags); 716 return; 717 } 718 719 rf->cq_table[iwcq->cq_num] = NULL; 720 spin_unlock_irqrestore(&rf->cqtable_lock, flags); 721 complete(&iwcq->free_cq); 722 } 723 724 struct ib_device * 725 to_ibdev(struct irdma_sc_dev *dev) 726 { 727 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; 728 } 729 730 /** 731 * irdma_get_qp - get qp address 732 * @device: iwarp device 733 * @qpn: qp number 734 */ 735 struct ib_qp * 736 irdma_get_qp(struct ib_device *device, int qpn) 737 { 738 struct irdma_device *iwdev = to_iwdev(device); 739 740 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp) 741 return NULL; 742 743 return &iwdev->rf->qp_table[qpn]->ibqp; 744 } 745 746 /** 747 * irdma_remove_cqp_head - return head entry and remove 748 * @dev: device 749 */ 750 void * 751 irdma_remove_cqp_head(struct irdma_sc_dev *dev) 752 { 753 struct list_head *entry; 754 struct list_head *list = &dev->cqp_cmd_head; 755 756 if (list_empty(list)) 757 return NULL; 758 759 entry = list->next; 760 list_del(entry); 761 762 return entry; 763 } 764 765 /** 766 * irdma_cqp_sds_cmd - create cqp command for sd 767 * @dev: hardware control device structure 768 * @sdinfo: information for sd cqp 769 * 770 */ 771 int 772 irdma_cqp_sds_cmd(struct irdma_sc_dev *dev, 773 struct irdma_update_sds_info *sdinfo) 774 { 775 struct irdma_cqp_request *cqp_request; 776 struct cqp_cmds_info *cqp_info; 777 struct irdma_pci_f *rf = dev_to_rf(dev); 778 int status; 779 780 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 781 if (!cqp_request) 782 return -ENOMEM; 783 784 cqp_info = &cqp_request->info; 785 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo, 786 sizeof(cqp_info->in.u.update_pe_sds.info)); 787 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS; 788 cqp_info->post_sq = 1; 789 cqp_info->in.u.update_pe_sds.dev = dev; 790 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request; 791 792 status = irdma_handle_cqp_op(rf, cqp_request); 793 irdma_put_cqp_request(&rf->cqp, cqp_request); 794 795 return status; 796 } 797 798 /** 799 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume 800 * @qp: hardware control qp 801 * @op: suspend or resume 802 */ 803 int 804 irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op) 805 { 806 struct irdma_sc_dev *dev = qp->dev; 807 struct irdma_cqp_request *cqp_request; 808 struct irdma_sc_cqp *cqp = dev->cqp; 809 struct cqp_cmds_info *cqp_info; 810 struct irdma_pci_f *rf = dev_to_rf(dev); 811 int status; 812 813 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 814 if (!cqp_request) 815 return -ENOMEM; 816 817 cqp_info = &cqp_request->info; 818 cqp_info->cqp_cmd = op; 819 cqp_info->in.u.suspend_resume.cqp = cqp; 820 cqp_info->in.u.suspend_resume.qp = qp; 821 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request; 822 823 status = irdma_handle_cqp_op(rf, cqp_request); 824 irdma_put_cqp_request(&rf->cqp, cqp_request); 825 826 return status; 827 } 828 829 /** 830 * irdma_term_modify_qp - modify qp for term message 831 * @qp: hardware control qp 832 * @next_state: qp's next state 833 * @term: terminate code 834 * @term_len: length 835 */ 836 void 837 irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term, 838 u8 term_len) 839 { 840 struct irdma_qp *iwqp; 841 842 iwqp = qp->qp_uk.back_qp; 843 irdma_next_iw_state(iwqp, next_state, 0, term, term_len); 844 }; 845 846 /** 847 * irdma_terminate_done - after terminate is completed 848 * @qp: hardware control qp 849 * @timeout_occurred: indicates if terminate timer expired 850 */ 851 void 852 irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred) 853 { 854 struct irdma_qp *iwqp; 855 u8 hte = 0; 856 bool first_time; 857 unsigned long flags; 858 859 iwqp = qp->qp_uk.back_qp; 860 spin_lock_irqsave(&iwqp->lock, flags); 861 if (iwqp->hte_added) { 862 iwqp->hte_added = 0; 863 hte = 1; 864 } 865 first_time = !(qp->term_flags & IRDMA_TERM_DONE); 866 qp->term_flags |= IRDMA_TERM_DONE; 867 spin_unlock_irqrestore(&iwqp->lock, flags); 868 if (first_time) { 869 if (!timeout_occurred) 870 irdma_terminate_del_timer(qp); 871 872 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0); 873 irdma_cm_disconn(iwqp); 874 } 875 } 876 877 static void 878 irdma_terminate_timeout(struct timer_list *t) 879 { 880 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer); 881 struct irdma_sc_qp *qp = &iwqp->sc_qp; 882 883 irdma_terminate_done(qp, 1); 884 irdma_qp_rem_ref(&iwqp->ibqp); 885 } 886 887 /** 888 * irdma_terminate_start_timer - start terminate timeout 889 * @qp: hardware control qp 890 */ 891 void 892 irdma_terminate_start_timer(struct irdma_sc_qp *qp) 893 { 894 struct irdma_qp *iwqp; 895 896 iwqp = qp->qp_uk.back_qp; 897 irdma_qp_add_ref(&iwqp->ibqp); 898 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0); 899 iwqp->terminate_timer.expires = jiffies + HZ; 900 901 add_timer(&iwqp->terminate_timer); 902 } 903 904 /** 905 * irdma_terminate_del_timer - delete terminate timeout 906 * @qp: hardware control qp 907 */ 908 void 909 irdma_terminate_del_timer(struct irdma_sc_qp *qp) 910 { 911 struct irdma_qp *iwqp; 912 int ret; 913 914 iwqp = qp->qp_uk.back_qp; 915 ret = irdma_del_timer_compat(&iwqp->terminate_timer); 916 if (ret) 917 irdma_qp_rem_ref(&iwqp->ibqp); 918 } 919 920 /** 921 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm 922 * @dev: function device struct 923 * @val_mem: buffer for fpm 924 * @hmc_fn_id: function id for fpm 925 */ 926 int 927 irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev, 928 struct irdma_dma_mem *val_mem, u16 hmc_fn_id) 929 { 930 struct irdma_cqp_request *cqp_request; 931 struct cqp_cmds_info *cqp_info; 932 struct irdma_pci_f *rf = dev_to_rf(dev); 933 int status; 934 935 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 936 if (!cqp_request) 937 return -ENOMEM; 938 939 cqp_info = &cqp_request->info; 940 cqp_request->param = NULL; 941 cqp_info->in.u.query_fpm_val.cqp = dev->cqp; 942 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa; 943 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va; 944 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id; 945 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL; 946 cqp_info->post_sq = 1; 947 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request; 948 949 status = irdma_handle_cqp_op(rf, cqp_request); 950 irdma_put_cqp_request(&rf->cqp, cqp_request); 951 952 return status; 953 } 954 955 /** 956 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw 957 * @dev: hardware control device structure 958 * @val_mem: buffer with fpm values 959 * @hmc_fn_id: function id for fpm 960 */ 961 int 962 irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev, 963 struct irdma_dma_mem *val_mem, u16 hmc_fn_id) 964 { 965 struct irdma_cqp_request *cqp_request; 966 struct cqp_cmds_info *cqp_info; 967 struct irdma_pci_f *rf = dev_to_rf(dev); 968 int status; 969 970 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 971 if (!cqp_request) 972 return -ENOMEM; 973 974 cqp_info = &cqp_request->info; 975 cqp_request->param = NULL; 976 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp; 977 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa; 978 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va; 979 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id; 980 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL; 981 cqp_info->post_sq = 1; 982 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request; 983 984 status = irdma_handle_cqp_op(rf, cqp_request); 985 irdma_put_cqp_request(&rf->cqp, cqp_request); 986 987 return status; 988 } 989 990 /** 991 * irdma_cqp_cq_create_cmd - create a cq for the cqp 992 * @dev: device pointer 993 * @cq: pointer to created cq 994 */ 995 int 996 irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 997 { 998 struct irdma_pci_f *rf = dev_to_rf(dev); 999 struct irdma_cqp *iwcqp = &rf->cqp; 1000 struct irdma_cqp_request *cqp_request; 1001 struct cqp_cmds_info *cqp_info; 1002 int status; 1003 1004 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1005 if (!cqp_request) 1006 return -ENOMEM; 1007 1008 cqp_info = &cqp_request->info; 1009 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 1010 cqp_info->post_sq = 1; 1011 cqp_info->in.u.cq_create.cq = cq; 1012 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 1013 1014 status = irdma_handle_cqp_op(rf, cqp_request); 1015 irdma_put_cqp_request(iwcqp, cqp_request); 1016 1017 return status; 1018 } 1019 1020 /** 1021 * irdma_cqp_qp_create_cmd - create a qp for the cqp 1022 * @dev: device pointer 1023 * @qp: pointer to created qp 1024 */ 1025 int 1026 irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1027 { 1028 struct irdma_pci_f *rf = dev_to_rf(dev); 1029 struct irdma_cqp *iwcqp = &rf->cqp; 1030 struct irdma_cqp_request *cqp_request; 1031 struct cqp_cmds_info *cqp_info; 1032 struct irdma_create_qp_info *qp_info; 1033 int status; 1034 1035 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1036 if (!cqp_request) 1037 return -ENOMEM; 1038 1039 cqp_info = &cqp_request->info; 1040 qp_info = &cqp_request->info.in.u.qp_create.info; 1041 memset(qp_info, 0, sizeof(*qp_info)); 1042 qp_info->cq_num_valid = true; 1043 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS; 1044 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 1045 cqp_info->post_sq = 1; 1046 cqp_info->in.u.qp_create.qp = qp; 1047 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 1048 1049 status = irdma_handle_cqp_op(rf, cqp_request); 1050 irdma_put_cqp_request(iwcqp, cqp_request); 1051 1052 return status; 1053 } 1054 1055 /** 1056 * irdma_dealloc_push_page - free a push page for qp 1057 * @rf: RDMA PCI function 1058 * @qp: hardware control qp 1059 */ 1060 void 1061 irdma_dealloc_push_page(struct irdma_pci_f *rf, 1062 struct irdma_sc_qp *qp) 1063 { 1064 struct irdma_cqp_request *cqp_request; 1065 struct cqp_cmds_info *cqp_info; 1066 int status; 1067 1068 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) 1069 return; 1070 1071 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false); 1072 if (!cqp_request) 1073 return; 1074 1075 cqp_info = &cqp_request->info; 1076 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 1077 cqp_info->post_sq = 1; 1078 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx; 1079 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle; 1080 cqp_info->in.u.manage_push_page.info.free_page = 1; 1081 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 1082 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp; 1083 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 1084 status = irdma_handle_cqp_op(rf, cqp_request); 1085 if (!status) 1086 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 1087 irdma_put_cqp_request(&rf->cqp, cqp_request); 1088 } 1089 1090 /** 1091 * irdma_cq_wq_destroy - send cq destroy cqp 1092 * @rf: RDMA PCI function 1093 * @cq: hardware control cq 1094 */ 1095 void 1096 irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq) 1097 { 1098 struct irdma_cqp_request *cqp_request; 1099 struct cqp_cmds_info *cqp_info; 1100 1101 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1102 if (!cqp_request) 1103 return; 1104 1105 cqp_info = &cqp_request->info; 1106 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY; 1107 cqp_info->post_sq = 1; 1108 cqp_info->in.u.cq_destroy.cq = cq; 1109 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request; 1110 1111 irdma_handle_cqp_op(rf, cqp_request); 1112 irdma_put_cqp_request(&rf->cqp, cqp_request); 1113 } 1114 1115 /** 1116 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait 1117 * @cqp_request: modify QP completion 1118 */ 1119 static void 1120 irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request) 1121 { 1122 struct cqp_cmds_info *cqp_info; 1123 struct irdma_qp *iwqp; 1124 1125 cqp_info = &cqp_request->info; 1126 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp; 1127 atomic_dec(&iwqp->hw_mod_qp_pend); 1128 wake_up(&iwqp->mod_qp_waitq); 1129 } 1130 1131 /** 1132 * irdma_hw_modify_qp - setup cqp for modify qp 1133 * @iwdev: RDMA device 1134 * @iwqp: qp ptr (user or kernel) 1135 * @info: info for modify qp 1136 * @wait: flag to wait or not for modify qp completion 1137 */ 1138 int 1139 irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp, 1140 struct irdma_modify_qp_info *info, bool wait) 1141 { 1142 int status; 1143 struct irdma_pci_f *rf = iwdev->rf; 1144 struct irdma_cqp_request *cqp_request; 1145 struct cqp_cmds_info *cqp_info; 1146 struct irdma_modify_qp_info *m_info; 1147 1148 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1149 if (!cqp_request) 1150 return -ENOMEM; 1151 1152 if (!wait) { 1153 cqp_request->callback_fcn = irdma_hw_modify_qp_callback; 1154 atomic_inc(&iwqp->hw_mod_qp_pend); 1155 } 1156 cqp_info = &cqp_request->info; 1157 m_info = &cqp_info->in.u.qp_modify.info; 1158 memcpy(m_info, info, sizeof(*m_info)); 1159 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1160 cqp_info->post_sq = 1; 1161 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1162 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1163 status = irdma_handle_cqp_op(rf, cqp_request); 1164 irdma_put_cqp_request(&rf->cqp, cqp_request); 1165 if (status) { 1166 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 1167 return status; 1168 1169 switch (m_info->next_iwarp_state) { 1170 struct irdma_gen_ae_info ae_info; 1171 1172 case IRDMA_QP_STATE_RTS: 1173 case IRDMA_QP_STATE_IDLE: 1174 case IRDMA_QP_STATE_TERMINATE: 1175 case IRDMA_QP_STATE_CLOSING: 1176 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE) 1177 irdma_send_reset(iwqp->cm_node); 1178 else 1179 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE; 1180 if (!wait) { 1181 ae_info.ae_code = IRDMA_AE_BAD_CLOSE; 1182 ae_info.ae_src = 0; 1183 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false); 1184 } else { 1185 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, 1186 wait); 1187 if (!cqp_request) 1188 return -ENOMEM; 1189 1190 cqp_info = &cqp_request->info; 1191 m_info = &cqp_info->in.u.qp_modify.info; 1192 memcpy(m_info, info, sizeof(*m_info)); 1193 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY; 1194 cqp_info->post_sq = 1; 1195 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp; 1196 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request; 1197 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR; 1198 m_info->reset_tcp_conn = true; 1199 irdma_handle_cqp_op(rf, cqp_request); 1200 irdma_put_cqp_request(&rf->cqp, cqp_request); 1201 } 1202 break; 1203 case IRDMA_QP_STATE_ERROR: 1204 default: 1205 break; 1206 } 1207 } 1208 1209 return status; 1210 } 1211 1212 /** 1213 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq 1214 * @dev: device pointer 1215 * @cq: pointer to cq 1216 */ 1217 void 1218 irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq) 1219 { 1220 struct irdma_pci_f *rf = dev_to_rf(dev); 1221 1222 irdma_cq_wq_destroy(rf, cq); 1223 } 1224 1225 /** 1226 * irdma_cqp_qp_destroy_cmd - destroy the cqp 1227 * @dev: device pointer 1228 * @qp: pointer to qp 1229 */ 1230 int 1231 irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1232 { 1233 struct irdma_pci_f *rf = dev_to_rf(dev); 1234 struct irdma_cqp *iwcqp = &rf->cqp; 1235 struct irdma_cqp_request *cqp_request; 1236 struct cqp_cmds_info *cqp_info; 1237 int status; 1238 1239 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 1240 if (!cqp_request) 1241 return -ENOMEM; 1242 1243 cqp_info = &cqp_request->info; 1244 memset(cqp_info, 0, sizeof(*cqp_info)); 1245 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY; 1246 cqp_info->post_sq = 1; 1247 cqp_info->in.u.qp_destroy.qp = qp; 1248 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request; 1249 cqp_info->in.u.qp_destroy.remove_hash_idx = true; 1250 1251 status = irdma_handle_cqp_op(rf, cqp_request); 1252 irdma_put_cqp_request(&rf->cqp, cqp_request); 1253 1254 return status; 1255 } 1256 1257 /** 1258 * irdma_ieq_mpa_crc_ae - generate AE for crc error 1259 * @dev: hardware control device structure 1260 * @qp: hardware control qp 1261 */ 1262 void 1263 irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp) 1264 { 1265 struct irdma_gen_ae_info info = {0}; 1266 struct irdma_pci_f *rf = dev_to_rf(dev); 1267 1268 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n"); 1269 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; 1270 info.ae_src = IRDMA_AE_SOURCE_RQ; 1271 irdma_gen_ae(rf, qp, &info, false); 1272 } 1273 1274 /** 1275 * irdma_ieq_get_qp - get qp based on quad in puda buffer 1276 * @dev: hardware control device structure 1277 * @buf: receive puda buffer on exception q 1278 */ 1279 struct irdma_sc_qp * 1280 irdma_ieq_get_qp(struct irdma_sc_dev *dev, 1281 struct irdma_puda_buf *buf) 1282 { 1283 struct irdma_qp *iwqp; 1284 struct irdma_cm_node *cm_node; 1285 struct irdma_device *iwdev = buf->vsi->back_vsi; 1286 u32 loc_addr[4] = {0}; 1287 u32 rem_addr[4] = {0}; 1288 u16 loc_port, rem_port; 1289 struct ip6_hdr *ip6h; 1290 struct ip *iph = (struct ip *)buf->iph; 1291 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1292 1293 if (iph->ip_v == 4) { 1294 loc_addr[0] = ntohl(iph->ip_dst.s_addr); 1295 rem_addr[0] = ntohl(iph->ip_src.s_addr); 1296 } else { 1297 ip6h = (struct ip6_hdr *)buf->iph; 1298 irdma_copy_ip_ntohl(loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32); 1299 irdma_copy_ip_ntohl(rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32); 1300 } 1301 loc_port = ntohs(tcph->th_dport); 1302 rem_port = ntohs(tcph->th_sport); 1303 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port, 1304 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF); 1305 if (!cm_node) 1306 return NULL; 1307 1308 iwqp = cm_node->iwqp; 1309 irdma_rem_ref_cm_node(cm_node); 1310 1311 return &iwqp->sc_qp; 1312 } 1313 1314 /** 1315 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs 1316 * @qp: qp ptr 1317 */ 1318 void 1319 irdma_send_ieq_ack(struct irdma_sc_qp *qp) 1320 { 1321 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node; 1322 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf; 1323 struct tcphdr *tcph = (struct tcphdr *)buf->tcph; 1324 1325 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum; 1326 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack); 1327 1328 irdma_send_ack(cm_node); 1329 } 1330 1331 /** 1332 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer 1333 * @qp: qp pointer 1334 * @ah_info: AH info pointer 1335 */ 1336 void 1337 irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp, 1338 struct irdma_ah_info *ah_info) 1339 { 1340 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf; 1341 struct ip *iph; 1342 struct ip6_hdr *ip6h; 1343 1344 memset(ah_info, 0, sizeof(*ah_info)); 1345 ah_info->do_lpbk = true; 1346 ah_info->vlan_tag = buf->vlan_id; 1347 ah_info->insert_vlan_tag = buf->vlan_valid; 1348 ah_info->ipv4_valid = buf->ipv4; 1349 ah_info->vsi = qp->vsi; 1350 1351 if (buf->smac_valid) 1352 ether_addr_copy(ah_info->mac_addr, buf->smac); 1353 1354 if (buf->ipv4) { 1355 ah_info->ipv4_valid = true; 1356 iph = (struct ip *)buf->iph; 1357 ah_info->hop_ttl = iph->ip_ttl; 1358 ah_info->tc_tos = iph->ip_tos; 1359 ah_info->dest_ip_addr[0] = ntohl(iph->ip_dst.s_addr); 1360 ah_info->src_ip_addr[0] = ntohl(iph->ip_src.s_addr); 1361 } else { 1362 ip6h = (struct ip6_hdr *)buf->iph; 1363 ah_info->hop_ttl = ip6h->ip6_hops; 1364 ah_info->tc_tos = ip6h->ip6_vfc; 1365 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 1366 ip6h->ip6_dst.__u6_addr.__u6_addr32); 1367 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 1368 ip6h->ip6_src.__u6_addr.__u6_addr32); 1369 } 1370 1371 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev), 1372 ah_info->dest_ip_addr, 1373 NULL, IRDMA_ARP_RESOLVE); 1374 } 1375 1376 /** 1377 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer 1378 * @buf: puda to update 1379 * @len: length of buffer 1380 * @seqnum: seq number for tcp 1381 */ 1382 static void 1383 irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf, 1384 u16 len, u32 seqnum) 1385 { 1386 struct tcphdr *tcph; 1387 struct ip *iph; 1388 u16 iphlen; 1389 u16 pktsize; 1390 u8 *addr = buf->mem.va; 1391 1392 iphlen = (buf->ipv4) ? 20 : 40; 1393 iph = (struct ip *)(addr + buf->maclen); 1394 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen); 1395 pktsize = len + buf->tcphlen + iphlen; 1396 iph->ip_len = htons(pktsize); 1397 tcph->th_seq = htonl(seqnum); 1398 } 1399 1400 /** 1401 * irdma_ieq_update_tcpip_info - update tcpip in the buffer 1402 * @buf: puda to update 1403 * @len: length of buffer 1404 * @seqnum: seq number for tcp 1405 */ 1406 void 1407 irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len, 1408 u32 seqnum) 1409 { 1410 struct tcphdr *tcph; 1411 u8 *addr; 1412 1413 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1414 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum); 1415 1416 addr = buf->mem.va; 1417 tcph = (struct tcphdr *)addr; 1418 tcph->th_seq = htonl(seqnum); 1419 } 1420 1421 /** 1422 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda 1423 * buffer 1424 * @info: to get information 1425 * @buf: puda buffer 1426 */ 1427 static int 1428 irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1429 struct irdma_puda_buf *buf) 1430 { 1431 struct ip *iph; 1432 struct ip6_hdr *ip6h; 1433 struct tcphdr *tcph; 1434 u16 iphlen; 1435 u16 pkt_len; 1436 u8 *mem = buf->mem.va; 1437 struct ether_header *ethh = buf->mem.va; 1438 1439 if (ethh->ether_type == htons(0x8100)) { 1440 info->vlan_valid = true; 1441 buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) & 1442 EVL_VLID_MASK; 1443 } 1444 1445 buf->maclen = (info->vlan_valid) ? 18 : 14; 1446 iphlen = (info->l3proto) ? 40 : 20; 1447 buf->ipv4 = (info->l3proto) ? false : true; 1448 buf->iph = mem + buf->maclen; 1449 iph = (struct ip *)buf->iph; 1450 buf->tcph = buf->iph + iphlen; 1451 tcph = (struct tcphdr *)buf->tcph; 1452 1453 if (buf->ipv4) { 1454 pkt_len = ntohs(iph->ip_len); 1455 } else { 1456 ip6h = (struct ip6_hdr *)buf->iph; 1457 pkt_len = ntohs(ip6h->ip6_plen) + iphlen; 1458 } 1459 1460 buf->totallen = pkt_len + buf->maclen; 1461 1462 if (info->payload_len < buf->totallen) { 1463 irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR, 1464 "payload_len = 0x%x totallen expected0x%x\n", 1465 info->payload_len, buf->totallen); 1466 return -EINVAL; 1467 } 1468 1469 buf->tcphlen = tcph->th_off << 2; 1470 buf->datalen = pkt_len - iphlen - buf->tcphlen; 1471 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1472 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen; 1473 buf->seqnum = ntohl(tcph->th_seq); 1474 1475 return 0; 1476 } 1477 1478 /** 1479 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer 1480 * @info: to get information 1481 * @buf: puda buffer 1482 */ 1483 int 1484 irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info, 1485 struct irdma_puda_buf *buf) 1486 { 1487 struct tcphdr *tcph; 1488 u32 pkt_len; 1489 u8 *mem; 1490 1491 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) 1492 return irdma_gen1_puda_get_tcpip_info(info, buf); 1493 1494 mem = buf->mem.va; 1495 buf->vlan_valid = info->vlan_valid; 1496 if (info->vlan_valid) 1497 buf->vlan_id = info->vlan; 1498 1499 buf->ipv4 = info->ipv4; 1500 if (buf->ipv4) 1501 buf->iph = mem + IRDMA_IPV4_PAD; 1502 else 1503 buf->iph = mem; 1504 1505 buf->tcph = mem + IRDMA_TCP_OFFSET; 1506 tcph = (struct tcphdr *)buf->tcph; 1507 pkt_len = info->payload_len; 1508 buf->totallen = pkt_len; 1509 buf->tcphlen = tcph->th_off << 2; 1510 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen; 1511 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL; 1512 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen; 1513 buf->seqnum = ntohl(tcph->th_seq); 1514 1515 if (info->smac_valid) { 1516 ether_addr_copy(buf->smac, info->smac); 1517 buf->smac_valid = true; 1518 } 1519 1520 return 0; 1521 } 1522 1523 /** 1524 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats 1525 * @t: timer_list pointer 1526 */ 1527 static void 1528 irdma_hw_stats_timeout(struct timer_list *t) 1529 { 1530 struct irdma_vsi_pestat *pf_devstat = 1531 from_timer(pf_devstat, t, stats_timer); 1532 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi; 1533 1534 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1535 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false); 1536 1537 mod_timer(&pf_devstat->stats_timer, 1538 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1539 } 1540 1541 /** 1542 * irdma_hw_stats_start_timer - Start periodic stats timer 1543 * @vsi: vsi structure pointer 1544 */ 1545 void 1546 irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi) 1547 { 1548 struct irdma_vsi_pestat *devstat = vsi->pestat; 1549 1550 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0); 1551 mod_timer(&devstat->stats_timer, 1552 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY)); 1553 } 1554 1555 /** 1556 * irdma_hw_stats_stop_timer - Delete periodic stats timer 1557 * @vsi: pointer to vsi structure 1558 */ 1559 void 1560 irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi) 1561 { 1562 struct irdma_vsi_pestat *devstat = vsi->pestat; 1563 1564 del_timer_sync(&devstat->stats_timer); 1565 } 1566 1567 /** 1568 * irdma_process_stats - Checking for wrap and update stats 1569 * @pestat: stats structure pointer 1570 */ 1571 static inline void 1572 irdma_process_stats(struct irdma_vsi_pestat *pestat) 1573 { 1574 sc_vsi_update_stats(pestat->vsi); 1575 } 1576 1577 /** 1578 * irdma_process_cqp_stats - Checking for wrap and update stats 1579 * @cqp_request: cqp_request structure pointer 1580 */ 1581 static void 1582 irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request) 1583 { 1584 struct irdma_vsi_pestat *pestat = cqp_request->param; 1585 1586 irdma_process_stats(pestat); 1587 } 1588 1589 /** 1590 * irdma_cqp_gather_stats_cmd - Gather stats 1591 * @dev: pointer to device structure 1592 * @pestat: pointer to stats info 1593 * @wait: flag to wait or not wait for stats 1594 */ 1595 int 1596 irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev, 1597 struct irdma_vsi_pestat *pestat, bool wait) 1598 { 1599 1600 struct irdma_pci_f *rf = dev_to_rf(dev); 1601 struct irdma_cqp *iwcqp = &rf->cqp; 1602 struct irdma_cqp_request *cqp_request; 1603 struct cqp_cmds_info *cqp_info; 1604 int status; 1605 1606 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1607 if (!cqp_request) 1608 return -ENOMEM; 1609 1610 cqp_info = &cqp_request->info; 1611 memset(cqp_info, 0, sizeof(*cqp_info)); 1612 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER; 1613 cqp_info->post_sq = 1; 1614 cqp_info->in.u.stats_gather.info = pestat->gather_info; 1615 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request; 1616 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp; 1617 cqp_request->param = pestat; 1618 if (!wait) 1619 cqp_request->callback_fcn = irdma_process_cqp_stats; 1620 status = irdma_handle_cqp_op(rf, cqp_request); 1621 if (wait) 1622 irdma_process_stats(pestat); 1623 irdma_put_cqp_request(&rf->cqp, cqp_request); 1624 1625 return status; 1626 } 1627 1628 /** 1629 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance 1630 * @vsi: pointer to vsi structure 1631 * @cmd: command to allocate or free 1632 * @stats_info: pointer to allocate stats info 1633 */ 1634 int 1635 irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, 1636 struct irdma_stats_inst_info *stats_info) 1637 { 1638 struct irdma_pci_f *rf = dev_to_rf(vsi->dev); 1639 struct irdma_cqp *iwcqp = &rf->cqp; 1640 struct irdma_cqp_request *cqp_request; 1641 struct cqp_cmds_info *cqp_info; 1642 int status; 1643 bool wait = false; 1644 1645 if (cmd == IRDMA_OP_STATS_ALLOCATE) 1646 wait = true; 1647 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait); 1648 if (!cqp_request) 1649 return -ENOMEM; 1650 1651 cqp_info = &cqp_request->info; 1652 memset(cqp_info, 0, sizeof(*cqp_info)); 1653 cqp_info->cqp_cmd = cmd; 1654 cqp_info->post_sq = 1; 1655 cqp_info->in.u.stats_manage.info = *stats_info; 1656 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request; 1657 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp; 1658 status = irdma_handle_cqp_op(rf, cqp_request); 1659 if (wait) 1660 stats_info->stats_idx = cqp_request->compl_info.op_ret_val; 1661 irdma_put_cqp_request(iwcqp, cqp_request); 1662 1663 return status; 1664 } 1665 1666 /** 1667 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0 1668 * @dev: pointer to device info 1669 * @sc_ceq: pointer to ceq structure 1670 * @op: Create or Destroy 1671 */ 1672 int 1673 irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, 1674 u8 op) 1675 { 1676 struct irdma_cqp_request *cqp_request; 1677 struct cqp_cmds_info *cqp_info; 1678 struct irdma_pci_f *rf = dev_to_rf(dev); 1679 int status; 1680 1681 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1682 if (!cqp_request) 1683 return -ENOMEM; 1684 1685 cqp_info = &cqp_request->info; 1686 cqp_info->post_sq = 1; 1687 cqp_info->cqp_cmd = op; 1688 cqp_info->in.u.ceq_create.ceq = sc_ceq; 1689 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request; 1690 1691 status = irdma_handle_cqp_op(rf, cqp_request); 1692 irdma_put_cqp_request(&rf->cqp, cqp_request); 1693 1694 return status; 1695 } 1696 1697 /** 1698 * irdma_cqp_aeq_cmd - Create/Destroy AEQ 1699 * @dev: pointer to device info 1700 * @sc_aeq: pointer to aeq structure 1701 * @op: Create or Destroy 1702 */ 1703 int 1704 irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, 1705 u8 op) 1706 { 1707 struct irdma_cqp_request *cqp_request; 1708 struct cqp_cmds_info *cqp_info; 1709 struct irdma_pci_f *rf = dev_to_rf(dev); 1710 int status; 1711 1712 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1713 if (!cqp_request) 1714 return -ENOMEM; 1715 1716 cqp_info = &cqp_request->info; 1717 cqp_info->post_sq = 1; 1718 cqp_info->cqp_cmd = op; 1719 cqp_info->in.u.aeq_create.aeq = sc_aeq; 1720 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request; 1721 1722 status = irdma_handle_cqp_op(rf, cqp_request); 1723 irdma_put_cqp_request(&rf->cqp, cqp_request); 1724 1725 return status; 1726 } 1727 1728 /** 1729 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node 1730 * @dev: pointer to device structure 1731 * @cmd: Add, modify or delete 1732 * @node_info: pointer to ws node info 1733 */ 1734 int 1735 irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, 1736 struct irdma_ws_node_info *node_info) 1737 { 1738 struct irdma_pci_f *rf = dev_to_rf(dev); 1739 struct irdma_cqp *iwcqp = &rf->cqp; 1740 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; 1741 struct irdma_cqp_request *cqp_request; 1742 struct cqp_cmds_info *cqp_info; 1743 int status; 1744 bool poll; 1745 1746 if (!rf->sc_dev.ceq_valid) 1747 poll = true; 1748 else 1749 poll = false; 1750 1751 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll); 1752 if (!cqp_request) 1753 return -ENOMEM; 1754 1755 cqp_info = &cqp_request->info; 1756 memset(cqp_info, 0, sizeof(*cqp_info)); 1757 cqp_info->cqp_cmd = cmd; 1758 cqp_info->post_sq = 1; 1759 cqp_info->in.u.ws_node.info = *node_info; 1760 cqp_info->in.u.ws_node.cqp = cqp; 1761 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request; 1762 status = irdma_handle_cqp_op(rf, cqp_request); 1763 if (status) 1764 goto exit; 1765 1766 if (poll) { 1767 struct irdma_ccq_cqe_info compl_info; 1768 1769 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, 1770 &compl_info); 1771 node_info->qs_handle = compl_info.op_ret_val; 1772 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, 1773 "opcode=%d, compl_info.retval=%d\n", 1774 compl_info.op_code, compl_info.op_ret_val); 1775 } else { 1776 node_info->qs_handle = cqp_request->compl_info.op_ret_val; 1777 } 1778 1779 exit: 1780 irdma_put_cqp_request(&rf->cqp, cqp_request); 1781 1782 return status; 1783 } 1784 1785 /** 1786 * irdma_ah_cqp_op - perform an AH cqp operation 1787 * @rf: RDMA PCI function 1788 * @sc_ah: address handle 1789 * @cmd: AH operation 1790 * @wait: wait if true 1791 * @callback_fcn: Callback function on CQP op completion 1792 * @cb_param: parameter for callback function 1793 * 1794 * returns errno 1795 */ 1796 int 1797 irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd, 1798 bool wait, 1799 void (*callback_fcn) (struct irdma_cqp_request *), 1800 void *cb_param) 1801 { 1802 struct irdma_cqp_request *cqp_request; 1803 struct cqp_cmds_info *cqp_info; 1804 int status; 1805 1806 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY) 1807 return -EINVAL; 1808 1809 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait); 1810 if (!cqp_request) 1811 return -ENOMEM; 1812 1813 cqp_info = &cqp_request->info; 1814 cqp_info->cqp_cmd = cmd; 1815 cqp_info->post_sq = 1; 1816 if (cmd == IRDMA_OP_AH_CREATE) { 1817 cqp_info->in.u.ah_create.info = sc_ah->ah_info; 1818 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request; 1819 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp; 1820 } else if (cmd == IRDMA_OP_AH_DESTROY) { 1821 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info; 1822 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request; 1823 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp; 1824 } 1825 1826 if (!wait) { 1827 cqp_request->callback_fcn = callback_fcn; 1828 cqp_request->param = cb_param; 1829 } 1830 status = irdma_handle_cqp_op(rf, cqp_request); 1831 irdma_put_cqp_request(&rf->cqp, cqp_request); 1832 1833 if (status) 1834 return -ENOMEM; 1835 1836 if (wait) 1837 sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY); 1838 1839 return 0; 1840 } 1841 1842 /** 1843 * irdma_ieq_ah_cb - callback after creation of AH for IEQ 1844 * @cqp_request: pointer to cqp_request of create AH 1845 */ 1846 static void 1847 irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request) 1848 { 1849 struct irdma_sc_qp *qp = cqp_request->param; 1850 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah; 1851 unsigned long flags; 1852 1853 spin_lock_irqsave(&qp->pfpdu.lock, flags); 1854 if (!cqp_request->compl_info.op_ret_val) { 1855 sc_ah->ah_info.ah_valid = true; 1856 irdma_ieq_process_fpdus(qp, qp->vsi->ieq); 1857 } else { 1858 sc_ah->ah_info.ah_valid = false; 1859 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp); 1860 } 1861 spin_unlock_irqrestore(&qp->pfpdu.lock, flags); 1862 } 1863 1864 /** 1865 * irdma_ilq_ah_cb - callback after creation of AH for ILQ 1866 * @cqp_request: pointer to cqp_request of create AH 1867 */ 1868 static void 1869 irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request) 1870 { 1871 struct irdma_cm_node *cm_node = cqp_request->param; 1872 struct irdma_sc_ah *sc_ah = cm_node->ah; 1873 1874 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val; 1875 irdma_add_conn_est_qh(cm_node); 1876 } 1877 1878 /** 1879 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's 1880 * @dev: device pointer 1881 * @ah_info: Address handle info 1882 * @wait: When true will wait for operation to complete 1883 * @type: ILQ/IEQ 1884 * @cb_param: Callback param when not waiting 1885 * @ah_ret: Returned pointer to address handle if created 1886 * 1887 */ 1888 int 1889 irdma_puda_create_ah(struct irdma_sc_dev *dev, 1890 struct irdma_ah_info *ah_info, bool wait, 1891 enum puda_rsrc_type type, void *cb_param, 1892 struct irdma_sc_ah **ah_ret) 1893 { 1894 struct irdma_sc_ah *ah; 1895 struct irdma_pci_f *rf = dev_to_rf(dev); 1896 int err; 1897 1898 ah = kzalloc(sizeof(*ah), GFP_ATOMIC); 1899 *ah_ret = ah; 1900 if (!ah) 1901 return -ENOMEM; 1902 1903 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, 1904 &ah_info->ah_idx, &rf->next_ah); 1905 if (err) 1906 goto err_free; 1907 1908 ah->dev = dev; 1909 ah->ah_info = *ah_info; 1910 1911 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ) 1912 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 1913 irdma_ilq_ah_cb, cb_param); 1914 else 1915 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait, 1916 irdma_ieq_ah_cb, cb_param); 1917 1918 if (err) 1919 goto error; 1920 return 0; 1921 1922 error: 1923 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 1924 err_free: 1925 kfree(ah); 1926 *ah_ret = NULL; 1927 return -ENOMEM; 1928 } 1929 1930 /** 1931 * irdma_puda_free_ah - free a puda address handle 1932 * @dev: device pointer 1933 * @ah: The address handle to free 1934 */ 1935 void 1936 irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah) 1937 { 1938 struct irdma_pci_f *rf = dev_to_rf(dev); 1939 1940 if (!ah) 1941 return; 1942 1943 if (ah->ah_info.ah_valid) { 1944 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL); 1945 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx); 1946 } 1947 1948 kfree(ah); 1949 } 1950 1951 /** 1952 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP 1953 * @cqp_request: pointer to cqp_request of create AH 1954 */ 1955 void 1956 irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request) 1957 { 1958 struct irdma_sc_ah *sc_ah = cqp_request->param; 1959 1960 if (!cqp_request->compl_info.op_ret_val) 1961 sc_ah->ah_info.ah_valid = true; 1962 else 1963 sc_ah->ah_info.ah_valid = false; 1964 } 1965 1966 /** 1967 * irdma_prm_add_pble_mem - add moemory to pble resources 1968 * @pprm: pble resource manager 1969 * @pchunk: chunk of memory to add 1970 */ 1971 int 1972 irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, 1973 struct irdma_chunk *pchunk) 1974 { 1975 u64 sizeofbitmap; 1976 1977 if (pchunk->size & 0xfff) 1978 return -EINVAL; 1979 1980 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; 1981 1982 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); 1983 if (!pchunk->bitmapbuf) 1984 return -ENOMEM; 1985 1986 pchunk->sizeofbitmap = sizeofbitmap; 1987 /* each pble is 8 bytes hence shift by 3 */ 1988 pprm->total_pble_alloc += pchunk->size >> 3; 1989 pprm->free_pble_cnt += pchunk->size >> 3; 1990 1991 return 0; 1992 } 1993 1994 /** 1995 * irdma_prm_get_pbles - get pble's from prm 1996 * @pprm: pble resource manager 1997 * @chunkinfo: nformation about chunk where pble's were acquired 1998 * @mem_size: size of pble memory needed 1999 * @vaddr: returns virtual address of pble memory 2000 * @fpm_addr: returns fpm address of pble memory 2001 */ 2002 int 2003 irdma_prm_get_pbles(struct irdma_pble_prm *pprm, 2004 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size, 2005 u64 **vaddr, u64 *fpm_addr) 2006 { 2007 u64 bits_needed; 2008 u64 bit_idx = PBLE_INVALID_IDX; 2009 struct irdma_chunk *pchunk = NULL; 2010 struct list_head *chunk_entry = (&pprm->clist)->next; 2011 u32 offset; 2012 unsigned long flags; 2013 *vaddr = NULL; 2014 *fpm_addr = 0; 2015 2016 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift)); 2017 2018 spin_lock_irqsave(&pprm->prm_lock, flags); 2019 while (chunk_entry != &pprm->clist) { 2020 pchunk = (struct irdma_chunk *)chunk_entry; 2021 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf, 2022 pchunk->sizeofbitmap, 0, 2023 bits_needed, 0); 2024 if (bit_idx < pchunk->sizeofbitmap) 2025 break; 2026 2027 /* list.next used macro */ 2028 chunk_entry = (&pchunk->list)->next; 2029 } 2030 2031 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) { 2032 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2033 return -ENOMEM; 2034 } 2035 2036 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed); 2037 offset = bit_idx << pprm->pble_shift; 2038 *vaddr = (u64 *)((u8 *)pchunk->vaddr + offset); 2039 *fpm_addr = pchunk->fpm_addr + offset; 2040 2041 chunkinfo->pchunk = pchunk; 2042 chunkinfo->bit_idx = bit_idx; 2043 chunkinfo->bits_used = bits_needed; 2044 /* 3 is sizeof pble divide */ 2045 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3); 2046 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2047 2048 return 0; 2049 } 2050 2051 /** 2052 * irdma_prm_return_pbles - return pbles back to prm 2053 * @pprm: pble resource manager 2054 * @chunkinfo: chunk where pble's were acquired and to be freed 2055 */ 2056 void 2057 irdma_prm_return_pbles(struct irdma_pble_prm *pprm, 2058 struct irdma_pble_chunkinfo *chunkinfo) 2059 { 2060 unsigned long flags; 2061 2062 spin_lock_irqsave(&pprm->prm_lock, flags); 2063 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3); 2064 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx, 2065 chunkinfo->bits_used); 2066 spin_unlock_irqrestore(&pprm->prm_lock, flags); 2067 } 2068 2069 int 2070 irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t * pg_dma, 2071 u32 pg_cnt) 2072 { 2073 struct page *vm_page; 2074 int i; 2075 u8 *addr; 2076 2077 addr = (u8 *)(uintptr_t)va; 2078 for (i = 0; i < pg_cnt; i++) { 2079 vm_page = vmalloc_to_page(addr); 2080 if (!vm_page) 2081 goto err; 2082 2083 pg_dma[i] = dma_map_page(hw_to_dev(hw), vm_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); 2084 if (dma_mapping_error(hw_to_dev(hw), pg_dma[i])) 2085 goto err; 2086 2087 addr += PAGE_SIZE; 2088 } 2089 2090 return 0; 2091 2092 err: 2093 irdma_unmap_vm_page_list(hw, pg_dma, i); 2094 return -ENOMEM; 2095 } 2096 2097 void 2098 irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t * pg_dma, u32 pg_cnt) 2099 { 2100 int i; 2101 2102 for (i = 0; i < pg_cnt; i++) 2103 dma_unmap_page(hw_to_dev(hw), pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL); 2104 } 2105 2106 /** 2107 * irdma_pble_free_paged_mem - free virtual paged memory 2108 * @chunk: chunk to free with paged memory 2109 */ 2110 void 2111 irdma_pble_free_paged_mem(struct irdma_chunk *chunk) 2112 { 2113 if (!chunk->pg_cnt) 2114 goto done; 2115 2116 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs, 2117 chunk->pg_cnt); 2118 2119 done: 2120 kfree(chunk->dmainfo.dmaaddrs); 2121 chunk->dmainfo.dmaaddrs = NULL; 2122 vfree(chunk->vaddr); 2123 chunk->vaddr = NULL; 2124 chunk->type = 0; 2125 } 2126 2127 /** 2128 * irdma_pble_get_paged_mem -allocate paged memory for pbles 2129 * @chunk: chunk to add for paged memory 2130 * @pg_cnt: number of pages needed 2131 */ 2132 int 2133 irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt) 2134 { 2135 u32 size; 2136 void *va; 2137 2138 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL); 2139 if (!chunk->dmainfo.dmaaddrs) 2140 return -ENOMEM; 2141 2142 size = PAGE_SIZE * pg_cnt; 2143 va = vmalloc(size); 2144 if (!va) 2145 goto err; 2146 2147 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs, 2148 pg_cnt)) { 2149 vfree(va); 2150 goto err; 2151 } 2152 chunk->vaddr = va; 2153 chunk->size = size; 2154 chunk->pg_cnt = pg_cnt; 2155 chunk->type = PBLE_SD_PAGED; 2156 2157 return 0; 2158 err: 2159 kfree(chunk->dmainfo.dmaaddrs); 2160 chunk->dmainfo.dmaaddrs = NULL; 2161 2162 return -ENOMEM; 2163 } 2164 2165 /** 2166 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID 2167 * @dev: device pointer 2168 */ 2169 u16 2170 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev) 2171 { 2172 struct irdma_pci_f *rf = dev_to_rf(dev); 2173 u32 next = 1; 2174 u32 node_id; 2175 2176 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id, 2177 &node_id, &next)) 2178 return IRDMA_WS_NODE_INVALID; 2179 2180 return (u16)node_id; 2181 } 2182 2183 /** 2184 * irdma_free_ws_node_id - Free a tx scheduler node ID 2185 * @dev: device pointer 2186 * @node_id: Work scheduler node ID 2187 */ 2188 void 2189 irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id) 2190 { 2191 struct irdma_pci_f *rf = dev_to_rf(dev); 2192 2193 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id); 2194 } 2195 2196 /** 2197 * irdma_modify_qp_to_err - Modify a QP to error 2198 * @sc_qp: qp structure 2199 */ 2200 void 2201 irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp) 2202 { 2203 struct irdma_qp *qp = sc_qp->qp_uk.back_qp; 2204 struct ib_qp_attr attr; 2205 2206 if (qp->iwdev->rf->reset) 2207 return; 2208 attr.qp_state = IB_QPS_ERR; 2209 2210 if (rdma_protocol_roce(qp->ibqp.device, 1)) 2211 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2212 else 2213 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL); 2214 } 2215 2216 void 2217 irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) 2218 { 2219 struct ib_event ibevent; 2220 2221 if (!iwqp->ibqp.event_handler) 2222 return; 2223 2224 switch (event) { 2225 case IRDMA_QP_EVENT_CATASTROPHIC: 2226 ibevent.event = IB_EVENT_QP_FATAL; 2227 break; 2228 case IRDMA_QP_EVENT_ACCESS_ERR: 2229 ibevent.event = IB_EVENT_QP_ACCESS_ERR; 2230 break; 2231 case IRDMA_QP_EVENT_REQ_ERR: 2232 ibevent.event = IB_EVENT_QP_REQ_ERR; 2233 break; 2234 } 2235 ibevent.device = iwqp->ibqp.device; 2236 ibevent.element.qp = &iwqp->ibqp; 2237 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context); 2238 } 2239 2240 static void 2241 clear_qp_ctx_addr(__le64 * ctx) 2242 { 2243 u64 tmp; 2244 2245 get_64bit_val(ctx, 272, &tmp); 2246 tmp &= GENMASK_ULL(63, 58); 2247 set_64bit_val(ctx, 272, tmp); 2248 2249 get_64bit_val(ctx, 296, &tmp); 2250 tmp &= GENMASK_ULL(7, 0); 2251 set_64bit_val(ctx, 296, tmp); 2252 2253 get_64bit_val(ctx, 312, &tmp); 2254 tmp &= GENMASK_ULL(7, 0); 2255 set_64bit_val(ctx, 312, tmp); 2256 2257 set_64bit_val(ctx, 368, 0); 2258 } 2259 2260 /** 2261 * irdma_upload_qp_context - upload raw QP context 2262 * @iwqp: QP pointer 2263 * @freeze: freeze QP 2264 * @raw: raw context flag 2265 */ 2266 int 2267 irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw) 2268 { 2269 struct irdma_dma_mem dma_mem; 2270 struct irdma_sc_dev *dev; 2271 struct irdma_sc_qp *qp; 2272 struct irdma_cqp *iwcqp; 2273 struct irdma_cqp_request *cqp_request; 2274 struct cqp_cmds_info *cqp_info; 2275 struct irdma_upload_context_info *info; 2276 struct irdma_pci_f *rf; 2277 int ret; 2278 u32 *ctx; 2279 2280 rf = iwqp->iwdev->rf; 2281 if (!rf) 2282 return -EINVAL; 2283 2284 qp = &iwqp->sc_qp; 2285 dev = &rf->sc_dev; 2286 iwcqp = &rf->cqp; 2287 2288 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true); 2289 if (!cqp_request) 2290 return -EINVAL; 2291 2292 cqp_info = &cqp_request->info; 2293 info = &cqp_info->in.u.qp_upload_context.info; 2294 memset(info, 0, sizeof(struct irdma_upload_context_info)); 2295 cqp_info->cqp_cmd = IRDMA_OP_QP_UPLOAD_CONTEXT; 2296 cqp_info->post_sq = 1; 2297 cqp_info->in.u.qp_upload_context.dev = dev; 2298 cqp_info->in.u.qp_upload_context.scratch = (uintptr_t)cqp_request; 2299 2300 dma_mem.size = PAGE_SIZE; 2301 dma_mem.va = irdma_allocate_dma_mem(dev->hw, &dma_mem, dma_mem.size, PAGE_SIZE); 2302 if (!dma_mem.va) { 2303 irdma_put_cqp_request(&rf->cqp, cqp_request); 2304 return -ENOMEM; 2305 } 2306 2307 ctx = dma_mem.va; 2308 info->buf_pa = dma_mem.pa; 2309 info->raw_format = raw; 2310 info->freeze_qp = freeze; 2311 info->qp_type = qp->qp_uk.qp_type; /* 1 is iWARP and 2 UDA */ 2312 info->qp_id = qp->qp_uk.qp_id; 2313 ret = irdma_handle_cqp_op(rf, cqp_request); 2314 if (ret) 2315 goto error; 2316 irdma_debug(dev, IRDMA_DEBUG_QP, "PRINT CONTXT QP [%d]\n", info->qp_id); 2317 { 2318 u32 i, j; 2319 2320 clear_qp_ctx_addr(dma_mem.va); 2321 for (i = 0, j = 0; i < 32; i++, j += 4) 2322 irdma_debug(dev, IRDMA_DEBUG_QP, 2323 "%d:\t [%08X %08x %08X %08X]\n", (j * 4), 2324 ctx[j], ctx[j + 1], ctx[j + 2], ctx[j + 3]); 2325 } 2326 error: 2327 irdma_put_cqp_request(iwcqp, cqp_request); 2328 irdma_free_dma_mem(dev->hw, &dma_mem); 2329 2330 return ret; 2331 } 2332 2333 bool 2334 irdma_cq_empty(struct irdma_cq *iwcq) 2335 { 2336 struct irdma_cq_uk *ukcq; 2337 u64 qword3; 2338 __le64 *cqe; 2339 u8 polarity; 2340 2341 ukcq = &iwcq->sc_cq.cq_uk; 2342 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq); 2343 get_64bit_val(cqe, 24, &qword3); 2344 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); 2345 2346 return polarity != ukcq->polarity; 2347 } 2348 2349 void 2350 irdma_remove_cmpls_list(struct irdma_cq *iwcq) 2351 { 2352 struct irdma_cmpl_gen *cmpl_node; 2353 struct list_head *tmp_node, *list_node; 2354 2355 list_for_each_safe(list_node, tmp_node, &iwcq->cmpl_generated) { 2356 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list); 2357 list_del(&cmpl_node->list); 2358 kfree(cmpl_node); 2359 } 2360 } 2361 2362 int 2363 irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info) 2364 { 2365 struct irdma_cmpl_gen *cmpl; 2366 2367 if (list_empty(&iwcq->cmpl_generated)) 2368 return -ENOENT; 2369 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list); 2370 list_del(&cmpl->list); 2371 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info)); 2372 kfree(cmpl); 2373 2374 irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS, 2375 "%s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%lx\n", 2376 __func__, cq_poll_info->qp_id, cq_poll_info->op_type, 2377 cq_poll_info->wr_id); 2378 2379 return 0; 2380 } 2381 2382 /** 2383 * irdma_set_cpi_common_values - fill in values for polling info struct 2384 * @cpi: resulting structure of cq_poll_info type 2385 * @qp: QPair 2386 * @qp_num: id of the QP 2387 */ 2388 static void 2389 irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi, 2390 struct irdma_qp_uk *qp, u32 qp_num) 2391 { 2392 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED; 2393 cpi->error = 1; 2394 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR; 2395 cpi->minor_err = FLUSH_GENERAL_ERR; 2396 cpi->qp_handle = (irdma_qp_handle) (uintptr_t)qp; 2397 cpi->qp_id = qp_num; 2398 } 2399 2400 static inline void 2401 irdma_comp_handler(struct irdma_cq *cq) 2402 { 2403 if (!cq->ibcq.comp_handler) 2404 return; 2405 2406 if (atomic_cmpxchg(&cq->armed, 1, 0)) 2407 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 2408 } 2409 2410 /** 2411 * irdma_generate_flush_completions - generate completion from WRs 2412 * @iwqp: pointer to QP 2413 */ 2414 void 2415 irdma_generate_flush_completions(struct irdma_qp *iwqp) 2416 { 2417 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk; 2418 struct irdma_ring *sq_ring = &qp->sq_ring; 2419 struct irdma_ring *rq_ring = &qp->rq_ring; 2420 struct irdma_cmpl_gen *cmpl; 2421 __le64 *sw_wqe; 2422 u64 wqe_qword; 2423 u32 wqe_idx; 2424 bool compl_generated = false; 2425 unsigned long flags1; 2426 2427 spin_lock_irqsave(&iwqp->iwscq->lock, flags1); 2428 if (irdma_cq_empty(iwqp->iwscq)) { 2429 unsigned long flags2; 2430 2431 spin_lock_irqsave(&iwqp->lock, flags2); 2432 while (IRDMA_RING_MORE_WORK(*sq_ring)) { 2433 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2434 if (!cmpl) { 2435 spin_unlock_irqrestore(&iwqp->lock, flags2); 2436 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2437 return; 2438 } 2439 2440 wqe_idx = sq_ring->tail; 2441 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2442 2443 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; 2444 cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled; 2445 sw_wqe = qp->sq_base[wqe_idx].elem; 2446 get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); 2447 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); 2448 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; 2449 /* remove the SQ WR by moving SQ tail */ 2450 IRDMA_RING_SET_TAIL(*sq_ring, 2451 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); 2452 2453 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { 2454 kfree(cmpl); 2455 continue; 2456 } 2457 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, 2458 "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n", 2459 __func__, cmpl->cpi.wr_id, qp->qp_id); 2460 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); 2461 compl_generated = true; 2462 } 2463 spin_unlock_irqrestore(&iwqp->lock, flags2); 2464 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2465 if (compl_generated) { 2466 irdma_comp_handler(iwqp->iwscq); 2467 compl_generated = false; 2468 } 2469 } else { 2470 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); 2471 irdma_sched_qp_flush_work(iwqp); 2472 } 2473 2474 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); 2475 if (irdma_cq_empty(iwqp->iwrcq)) { 2476 unsigned long flags2; 2477 2478 spin_lock_irqsave(&iwqp->lock, flags2); 2479 while (IRDMA_RING_MORE_WORK(*rq_ring)) { 2480 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); 2481 if (!cmpl) { 2482 spin_unlock_irqrestore(&iwqp->lock, flags2); 2483 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2484 return; 2485 } 2486 2487 wqe_idx = rq_ring->tail; 2488 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id); 2489 2490 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx]; 2491 cmpl->cpi.signaled = 1; 2492 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC; 2493 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ; 2494 /* remove the RQ WR by moving RQ tail */ 2495 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); 2496 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, 2497 "%s: adding wr_id = 0x%lx RQ Completion to list qp_id=%d, wqe_idx=%d\n", 2498 __func__, cmpl->cpi.wr_id, qp->qp_id, 2499 wqe_idx); 2500 2501 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); 2502 2503 compl_generated = true; 2504 } 2505 spin_unlock_irqrestore(&iwqp->lock, flags2); 2506 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2507 if (compl_generated) 2508 irdma_comp_handler(iwqp->iwrcq); 2509 } else { 2510 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); 2511 irdma_sched_qp_flush_work(iwqp); 2512 } 2513 } 2514 2515 /** 2516 * irdma_udqp_qs_change - change qs for UD QP in a worker thread 2517 * @iwqp: QP pointer 2518 * @user_prio: new user priority value 2519 * @qs_change: when false, only user priority changes, QS handle do not need to change 2520 */ 2521 static void 2522 irdma_udqp_qs_change(struct irdma_qp *iwqp, u8 user_prio, bool qs_change) 2523 { 2524 irdma_qp_rem_qos(&iwqp->sc_qp); 2525 if (qs_change) 2526 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri); 2527 2528 iwqp->ctx_info.user_pri = user_prio; 2529 iwqp->sc_qp.user_pri = user_prio; 2530 2531 if (qs_change) 2532 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio)) 2533 irdma_dev_warn(&iwqp->iwdev->ibdev, 2534 "WS add failed during %s, qp_id: %x user_pri: %x", 2535 __func__, iwqp->ibqp.qp_num, user_prio); 2536 irdma_qp_add_qos(&iwqp->sc_qp); 2537 } 2538 2539 void 2540 irdma_udqp_qs_worker(struct work_struct *work) 2541 { 2542 struct irdma_udqs_work *udqs_work = container_of(work, struct irdma_udqs_work, work); 2543 2544 irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change); 2545 if (udqs_work->qs_change) 2546 irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME); 2547 irdma_qp_rem_ref(&udqs_work->iwqp->ibqp); 2548 kfree(udqs_work); 2549 } 2550