1 /*
2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: RDMA Controller HW interface
29 */
30
31 #include <linux/interrupt.h>
32 #include <linux/spinlock.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/sched.h>
35 #include <linux/pci.h>
36 #include <linux/delay.h>
37 #include <linux/hardirq.h>
38 #include <linux/device.h>
39
40 #include "hsi_struct_def.h"
41 #include "qplib_tlv.h"
42 #include "qplib_res.h"
43 #include "qplib_sp.h"
44 #include "qplib_rcfw.h"
45 #include "bnxt_re.h"
46
47 static void bnxt_qplib_service_creq(unsigned long data);
48
__check_cmdq_stall(struct bnxt_qplib_rcfw * rcfw,u32 * cur_prod,u32 * cur_cons)49 int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,
50 u32 *cur_prod, u32 *cur_cons)
51 {
52 struct bnxt_qplib_cmdq_ctx *cmdq;
53
54 cmdq = &rcfw->cmdq;
55
56 if (*cur_prod == cmdq->hwq.prod &&
57 *cur_cons == cmdq->hwq.cons)
58 /* No activity on CMDQ or CREQ. FW down */
59 return -ETIMEDOUT;
60
61 *cur_prod = cmdq->hwq.prod;
62 *cur_cons = cmdq->hwq.cons;
63 return 0;
64 }
65
bnxt_qplib_map_rc(u8 opcode)66 static int bnxt_qplib_map_rc(u8 opcode)
67 {
68 switch (opcode) {
69 case CMDQ_BASE_OPCODE_DESTROY_QP:
70 case CMDQ_BASE_OPCODE_DESTROY_SRQ:
71 case CMDQ_BASE_OPCODE_DESTROY_CQ:
72 case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:
73 case CMDQ_BASE_OPCODE_DEREGISTER_MR:
74 case CMDQ_BASE_OPCODE_DELETE_GID:
75 case CMDQ_BASE_OPCODE_DESTROY_QP1:
76 case CMDQ_BASE_OPCODE_DESTROY_AH:
77 case CMDQ_BASE_OPCODE_DEINITIALIZE_FW:
78 case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC:
79 case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE:
80 return 0;
81 default:
82 return -ETIMEDOUT;
83 }
84 }
85
86 /**
87 * bnxt_re_is_fw_stalled - Check firmware health
88 * @rcfw - rcfw channel instance of rdev
89 * @cookie - cookie to track the command
90 *
91 * If firmware has not responded any rcfw command within
92 * rcfw->max_timeout, consider firmware as stalled.
93 *
94 * Returns:
95 * 0 if firmware is responding
96 * -ENODEV if firmware is not responding
97 */
bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw * rcfw,u16 cookie)98 static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
99 {
100 struct bnxt_qplib_cmdq_ctx *cmdq;
101 struct bnxt_qplib_crsqe *crsqe;
102
103 crsqe = &rcfw->crsqe_tbl[cookie];
104 cmdq = &rcfw->cmdq;
105
106 if (time_after(jiffies, cmdq->last_seen +
107 (rcfw->max_timeout * HZ))) {
108 dev_warn_ratelimited(&rcfw->pdev->dev,
109 "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%ld > %d) msec active %d\n",
110 __func__, cookie, crsqe->opcode,
111 (long)jiffies_to_msecs(jiffies - cmdq->last_seen),
112 rcfw->max_timeout * 1000,
113 crsqe->is_in_used);
114 return -ENODEV;
115 }
116
117 return 0;
118 }
119 /**
120 * __wait_for_resp - Don't hold the cpu context and wait for response
121 * @rcfw - rcfw channel instance of rdev
122 * @cookie - cookie to track the command
123 *
124 * Wait for command completion in sleepable context.
125 *
126 * Returns:
127 * 0 if command is completed by firmware.
128 * Non zero error code for rest of the case.
129 */
__wait_for_resp(struct bnxt_qplib_rcfw * rcfw,u16 cookie)130 static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
131 {
132 struct bnxt_qplib_cmdq_ctx *cmdq;
133 struct bnxt_qplib_crsqe *crsqe;
134 unsigned long issue_time;
135 int ret;
136
137 cmdq = &rcfw->cmdq;
138 issue_time = jiffies;
139 crsqe = &rcfw->crsqe_tbl[cookie];
140
141 do {
142 if (RCFW_NO_FW_ACCESS(rcfw))
143 return bnxt_qplib_map_rc(crsqe->opcode);
144 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
145 return -ETIMEDOUT;
146
147 /* Non zero means command completed */
148 ret = wait_event_timeout(cmdq->waitq,
149 !crsqe->is_in_used ||
150 RCFW_NO_FW_ACCESS(rcfw),
151 msecs_to_jiffies(rcfw->max_timeout * 1000));
152
153 if (!crsqe->is_in_used)
154 return 0;
155 /*
156 * Take care if interrupt miss or other cases like DBR drop
157 */
158 bnxt_qplib_service_creq((unsigned long)rcfw);
159 dev_warn_ratelimited(&rcfw->pdev->dev,
160 "Non-Blocking QPLIB: cmdq[%#x]=%#x waited (%lu) msec bit %d\n",
161 cookie, crsqe->opcode,
162 (long)jiffies_to_msecs(jiffies - issue_time),
163 crsqe->is_in_used);
164
165 if (!crsqe->is_in_used)
166 return 0;
167
168 ret = bnxt_re_is_fw_stalled(rcfw, cookie);
169 if (ret)
170 return ret;
171
172 } while (true);
173 };
174
175 /**
176 * __block_for_resp - hold the cpu context and wait for response
177 * @rcfw - rcfw channel instance of rdev
178 * @cookie - cookie to track the command
179 *
180 * This function will hold the cpu (non-sleepable context) and
181 * wait for command completion. Maximum holding interval is 8 second.
182 *
183 * Returns:
184 * -ETIMEOUT if command is not completed in specific time interval.
185 * 0 if command is completed by firmware.
186 */
__block_for_resp(struct bnxt_qplib_rcfw * rcfw,u16 cookie)187 static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
188 {
189 struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
190 struct bnxt_qplib_crsqe *crsqe;
191 unsigned long issue_time = 0;
192
193 issue_time = jiffies;
194 crsqe = &rcfw->crsqe_tbl[cookie];
195
196 do {
197 if (RCFW_NO_FW_ACCESS(rcfw))
198 return bnxt_qplib_map_rc(crsqe->opcode);
199 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
200 return -ETIMEDOUT;
201
202 udelay(1);
203
204 /* Below call is must since there can be a deadlock
205 * if interrupt is mapped to the same cpu
206 */
207 bnxt_qplib_service_creq((unsigned long)rcfw);
208 if (!crsqe->is_in_used)
209 return 0;
210
211 } while (time_before(jiffies, issue_time + (8 * HZ)));
212
213 dev_warn_ratelimited(&rcfw->pdev->dev,
214 "Blocking QPLIB: cmdq[%#x]=%#x taken (%lu) msec",
215 cookie, crsqe->opcode,
216 (long)jiffies_to_msecs(jiffies - issue_time));
217
218 return -ETIMEDOUT;
219 };
220
221 /* __send_message_no_waiter - get cookie and post the message.
222 * @rcfw - rcfw channel instance of rdev
223 * @msg - qplib message internal
224 *
225 * This function will just post and don't bother about completion.
226 * Current design of this function is -
227 * user must hold the completion queue hwq->lock.
228 * user must have used existing completion and free the resources.
229 * this function will not check queue full condition.
230 * this function will explicitly set is_waiter_alive=false.
231 * current use case is - send destroy_ah if create_ah is return
232 * after waiter of create_ah is lost. It can be extended for other
233 * use case as well.
234 *
235 * Returns: Nothing
236 *
237 */
__send_message_no_waiter(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_cmdqmsg * msg)238 static void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw,
239 struct bnxt_qplib_cmdqmsg *msg)
240 {
241 struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
242 struct bnxt_qplib_hwq *cmdq_hwq = &cmdq->hwq;
243 struct bnxt_qplib_crsqe *crsqe;
244 struct bnxt_qplib_cmdqe *cmdqe;
245 u32 sw_prod, cmdq_prod, bsize;
246 u16 cookie;
247 u8 *preq;
248
249 cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
250 __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
251 crsqe = &rcfw->crsqe_tbl[cookie];
252
253 /* Set cmd_size in terms of 16B slots in req. */
254 bsize = bnxt_qplib_set_cmd_slots(msg->req);
255 /* GET_CMD_SIZE would return number of slots in either case of tlv
256 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()
257 */
258 crsqe->send_timestamp = jiffies;
259 crsqe->is_internal_cmd = true;
260 crsqe->is_waiter_alive = false;
261 crsqe->is_in_used = true;
262 crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
263
264 preq = (u8 *)msg->req;
265 do {
266 /* Locate the next cmdq slot */
267 sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq);
268 cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL);
269 /* Copy a segment of the req cmd to the cmdq */
270 memset(cmdqe, 0, sizeof(*cmdqe));
271 memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
272 preq += min_t(u32, bsize, sizeof(*cmdqe));
273 bsize -= min_t(u32, bsize, sizeof(*cmdqe));
274 cmdq_hwq->prod++;
275 } while (bsize > 0);
276 cmdq->seq_num++;
277
278 cmdq_prod = cmdq_hwq->prod & 0xFFFF;
279 atomic_inc(&rcfw->timeout_send);
280 /* ring CMDQ DB */
281 wmb();
282 writel(cmdq_prod, cmdq->cmdq_mbox.prod);
283 writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
284 }
285
__send_message(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_cmdqmsg * msg)286 static int __send_message(struct bnxt_qplib_rcfw *rcfw,
287 struct bnxt_qplib_cmdqmsg *msg)
288 {
289 u32 bsize, free_slots, required_slots;
290 struct bnxt_qplib_cmdq_ctx *cmdq;
291 struct bnxt_qplib_crsqe *crsqe;
292 struct bnxt_qplib_cmdqe *cmdqe;
293 struct bnxt_qplib_hwq *cmdq_hwq;
294 u32 sw_prod, cmdq_prod;
295 struct pci_dev *pdev;
296 unsigned long flags;
297 u16 cookie;
298 u8 opcode;
299 u8 *preq;
300
301 cmdq = &rcfw->cmdq;
302 cmdq_hwq = &cmdq->hwq;
303 pdev = rcfw->pdev;
304 opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
305
306 /* Cmdq are in 16-byte units, each request can consume 1 or more
307 cmdqe */
308 spin_lock_irqsave(&cmdq_hwq->lock, flags);
309 required_slots = bnxt_qplib_get_cmd_slots(msg->req);
310 free_slots = HWQ_FREE_SLOTS(cmdq_hwq);
311 cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
312 crsqe = &rcfw->crsqe_tbl[cookie];
313
314 if (required_slots >= free_slots) {
315 dev_warn_ratelimited(&pdev->dev,
316 "QPLIB: RCFW: CMDQ is full req/free %d/%d!\n",
317 required_slots, free_slots);
318 rcfw->cmdq_full_dbg++;
319 spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
320 return -EAGAIN;
321 }
322
323 if (crsqe->is_in_used)
324 panic("QPLIB: Cookie was not requested %d\n",
325 cookie);
326
327 if (msg->block)
328 cookie |= RCFW_CMD_IS_BLOCKING;
329 __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
330
331 /* Set cmd_size in terms of 16B slots in req. */
332 bsize = bnxt_qplib_set_cmd_slots(msg->req);
333 /* GET_CMD_SIZE would return number of slots in either case of tlv
334 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()
335 */
336 crsqe->send_timestamp = jiffies;
337 crsqe->free_slots = free_slots;
338 crsqe->resp = (struct creq_qp_event *)msg->resp;
339 crsqe->resp->cookie = cpu_to_le16(cookie);
340 crsqe->is_internal_cmd = false;
341 crsqe->is_waiter_alive = true;
342 crsqe->is_in_used = true;
343 crsqe->opcode = opcode;
344 crsqe->requested_qp_state = msg->qp_state;
345
346 crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
347 if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) {
348 struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb;
349
350 __set_cmdq_base_resp_addr(msg->req, msg->req_sz,
351 cpu_to_le64(sbuf->dma_addr));
352 __set_cmdq_base_resp_size(msg->req, msg->req_sz,
353 ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS) /
354 BNXT_QPLIB_CMDQE_UNITS);
355 }
356
357 preq = (u8 *)msg->req;
358 do {
359 /* Locate the next cmdq slot */
360 sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq);
361 cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL);
362 /* Copy a segment of the req cmd to the cmdq */
363 memset(cmdqe, 0, sizeof(*cmdqe));
364 memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
365 preq += min_t(u32, bsize, sizeof(*cmdqe));
366 bsize -= min_t(u32, bsize, sizeof(*cmdqe));
367 cmdq_hwq->prod++;
368 } while (bsize > 0);
369 cmdq->seq_num++;
370
371 cmdq_prod = cmdq_hwq->prod & 0xFFFF;
372 if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) {
373 /* The very first doorbell write
374 * is required to set this flag
375 * which prompts the FW to reset
376 * its internal pointers
377 */
378 cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);
379 clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
380 }
381 /* ring CMDQ DB */
382 wmb();
383 writel(cmdq_prod, cmdq->cmdq_mbox.prod);
384 writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
385
386 dev_dbg(&pdev->dev, "QPLIB: RCFW sent request with 0x%x 0x%x 0x%x\n",
387 cmdq_prod, cmdq_hwq->prod, crsqe->req_size);
388 dev_dbg(&pdev->dev,
389 "QPLIB: opcode 0x%x with cookie 0x%x at cmdq/crsq 0x%p/0x%p\n",
390 opcode,
391 __get_cmdq_base_cookie(msg->req, msg->req_sz),
392 cmdqe, crsqe);
393 spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
394 /* Return the CREQ response pointer */
395 return 0;
396 }
397
398 /**
399 * __poll_for_resp - self poll completion for rcfw command
400 * @rcfw - rcfw channel instance of rdev
401 * @cookie - cookie to track the command
402 *
403 * It works same as __wait_for_resp except this function will
404 * do self polling in sort interval since interrupt is disabled.
405 * This function can not be called from non-sleepable context.
406 *
407 * Returns:
408 * -ETIMEOUT if command is not completed in specific time interval.
409 * 0 if command is completed by firmware.
410 */
__poll_for_resp(struct bnxt_qplib_rcfw * rcfw,u16 cookie)411 static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
412 {
413 struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
414 struct bnxt_qplib_crsqe *crsqe;
415 unsigned long issue_time;
416 int ret;
417
418 issue_time = jiffies;
419 crsqe = &rcfw->crsqe_tbl[cookie];
420
421 do {
422 if (RCFW_NO_FW_ACCESS(rcfw))
423 return bnxt_qplib_map_rc(crsqe->opcode);
424 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
425 return -ETIMEDOUT;
426
427 usleep_range(1000, 1001);
428
429 bnxt_qplib_service_creq((unsigned long)rcfw);
430 if (!crsqe->is_in_used)
431 return 0;
432
433 if (jiffies_to_msecs(jiffies - issue_time) >
434 (rcfw->max_timeout * 1000)) {
435 dev_warn_ratelimited(&rcfw->pdev->dev,
436 "Self Polling QPLIB: cmdq[%#x]=%#x taken (%lu) msec",
437 cookie, crsqe->opcode,
438 (long)jiffies_to_msecs(jiffies - issue_time));
439 ret = bnxt_re_is_fw_stalled(rcfw, cookie);
440 if (ret)
441 return ret;
442 }
443 } while (true);
444
445 };
446
__send_message_basic_sanity(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_cmdqmsg * msg,u8 opcode)447 static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw,
448 struct bnxt_qplib_cmdqmsg *msg, u8 opcode)
449 {
450 struct bnxt_qplib_cmdq_ctx *cmdq;
451
452 cmdq = &rcfw->cmdq;
453
454 /* Prevent posting if f/w is not in a state to process */
455 if (RCFW_NO_FW_ACCESS(rcfw))
456 return -ENXIO;
457
458 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
459 return -ETIMEDOUT;
460
461 if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
462 opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {
463 dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n");
464 return -EINVAL;
465 }
466
467 if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
468 (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
469 opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &&
470 opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {
471 dev_err(&rcfw->pdev->dev,
472 "QPLIB: RCFW not initialized, reject opcode 0x%x\n",
473 opcode);
474 return -ENOTSUPP;
475 }
476
477 return 0;
478 }
479
480 /* This function will just post and do not bother about completion */
__destroy_timedout_ah(struct bnxt_qplib_rcfw * rcfw,struct creq_create_ah_resp * create_ah_resp)481 static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw,
482 struct creq_create_ah_resp *create_ah_resp)
483 {
484 struct bnxt_qplib_cmdqmsg msg = {};
485 struct cmdq_destroy_ah req = {};
486
487 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH,
488 sizeof(req));
489 req.ah_cid = create_ah_resp->xid;
490 msg.req = (struct cmdq_base *)&req;
491 msg.req_sz = sizeof(req);
492 __send_message_no_waiter(rcfw, &msg);
493 dev_warn_ratelimited(&rcfw->pdev->dev,
494 "From %s: ah_cid = %d timeout_send %d\n", __func__,
495 req.ah_cid,
496 atomic_read(&rcfw->timeout_send));
497 }
498
499 /**
500 * __bnxt_qplib_rcfw_send_message - qplib interface to send
501 * and complete rcfw command.
502 * @rcfw - rcfw channel instance of rdev
503 * @msg - qplib message internal
504 *
505 * This function does not account shadow queue depth. It will send
506 * all the command unconditionally as long as send queue is not full.
507 *
508 * Returns:
509 * 0 if command completed by firmware.
510 * Non zero if the command is not completed by firmware.
511 */
__bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_cmdqmsg * msg)512 static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
513 struct bnxt_qplib_cmdqmsg *msg)
514 {
515 struct bnxt_qplib_crsqe *crsqe;
516 struct creq_qp_event *event;
517 unsigned long flags;
518 u16 cookie;
519 int rc = 0;
520 u8 opcode;
521
522 opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
523
524 rc = __send_message_basic_sanity(rcfw, msg, opcode);
525 if (rc)
526 return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc;
527
528 rc = __send_message(rcfw, msg);
529 if (rc)
530 return rc;
531
532 cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req,
533 msg->req_sz)) & RCFW_MAX_COOKIE_VALUE;
534
535
536 if (msg->block)
537 rc = __block_for_resp(rcfw, cookie);
538 else if (atomic_read(&rcfw->rcfw_intr_enabled))
539 rc = __wait_for_resp(rcfw, cookie);
540 else
541 rc = __poll_for_resp(rcfw, cookie);
542
543 if (rc) {
544 /* First check if it is FW stall.
545 * Use hwq.lock to avoid race with actual completion.
546 */
547 spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags);
548 crsqe = &rcfw->crsqe_tbl[cookie];
549 crsqe->is_waiter_alive = false;
550 if (rc == -ENODEV)
551 set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags);
552 spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags);
553
554 return -ETIMEDOUT;
555 }
556
557 event = (struct creq_qp_event *)msg->resp;
558 if (event->status) {
559 /* failed with status */
560 dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n",
561 cookie, opcode, GET_OPCODE_TYPE(opcode), event->status);
562 rc = -EFAULT;
563 /*
564 * Workaround to avoid errors in the stack during bond
565 * creation and deletion.
566 * Disable error returned for ADD_GID/DEL_GID
567 */
568 if (opcode == CMDQ_BASE_OPCODE_ADD_GID ||
569 opcode == CMDQ_BASE_OPCODE_DELETE_GID)
570 rc = 0;
571 }
572
573 dev_dbg(&pdev->dev, "QPLIB: %s:%d - op 0x%x (%s), cookie 0x%x -- Return: e->status 0x%x, rc = 0x%x\n",
574 __func__, __LINE__, opcode, GET_OPCODE_TYPE(opcode), cookie, event->status, rc);
575 return rc;
576 }
577
578 /**
579 * bnxt_qplib_rcfw_send_message - qplib interface to send
580 * and complete rcfw command.
581 * @rcfw - rcfw channel instance of rdev
582 * @msg - qplib message internal
583 *
584 * Driver interact with Firmware through rcfw channel/slow path in two ways.
585 * a. Blocking rcfw command send. In this path, driver cannot hold
586 * the context for longer period since it is holding cpu until
587 * command is not completed.
588 * b. Non-blocking rcfw command send. In this path, driver can hold the
589 * context for longer period. There may be many pending command waiting
590 * for completion because of non-blocking nature.
591 *
592 * Driver will use shadow queue depth. Current queue depth of 8K
593 * (due to size of rcfw message it can be actual ~4K rcfw outstanding)
594 * is not optimal for rcfw command processing in firmware.
595 * RCFW_CMD_NON_BLOCKING_SHADOW_QD is defined as 64.
596 * Restrict at max 64 Non-Blocking rcfw commands.
597 * Do not allow more than 64 non-blocking command to the Firmware.
598 * Allow all blocking commands until there is no queue full.
599 *
600 * Returns:
601 * 0 if command completed by firmware.
602 * Non zero if the command is not completed by firmware.
603 */
bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_cmdqmsg * msg)604 int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
605 struct bnxt_qplib_cmdqmsg *msg)
606 {
607 int ret;
608
609 if (!msg->block) {
610 down(&rcfw->rcfw_inflight);
611 ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);
612 up(&rcfw->rcfw_inflight);
613 } else {
614 ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);
615 }
616
617 return ret;
618 }
619
bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw * rcfw,struct bnxt_qplib_crsqe * crsqe)620 static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw,
621 struct bnxt_qplib_crsqe *crsqe)
622 {
623 u32 latency_msec, dest_stats_id;
624 u64 *dest_stats_ptr = NULL;
625
626 latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen -
627 crsqe->send_timestamp);
628 if (latency_msec/1000 < RCFW_MAX_LATENCY_SEC_SLAB_INDEX)
629 rcfw->rcfw_lat_slab_sec[latency_msec/1000]++;
630
631 if (!rcfw->sp_perf_stats_enabled)
632 return;
633
634 if (latency_msec < RCFW_MAX_LATENCY_MSEC_SLAB_INDEX)
635 rcfw->rcfw_lat_slab_msec[latency_msec]++;
636
637 switch (crsqe->opcode) {
638 case CMDQ_BASE_OPCODE_CREATE_QP:
639 dest_stats_id = rcfw->qp_create_stats_id++;
640 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
641 dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id];
642 break;
643 case CMDQ_BASE_OPCODE_DESTROY_QP:
644 dest_stats_id = rcfw->qp_destroy_stats_id++;
645 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
646 dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id];
647 break;
648 case CMDQ_BASE_OPCODE_REGISTER_MR:
649 dest_stats_id = rcfw->mr_create_stats_id++;
650 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
651 dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id];
652 break;
653 case CMDQ_BASE_OPCODE_DEREGISTER_MR:
654 case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:
655 dest_stats_id = rcfw->mr_destroy_stats_id++;
656 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
657 dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id];
658 break;
659 case CMDQ_BASE_OPCODE_MODIFY_QP:
660 if (crsqe->requested_qp_state != IB_QPS_ERR)
661 break;
662 dest_stats_id = rcfw->qp_modify_stats_id++;
663 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
664 dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id];
665 break;
666 default:
667 break;
668 }
669 if (dest_stats_ptr)
670 *dest_stats_ptr = max_t(unsigned long,
671 (rcfw->cmdq.last_seen - crsqe->send_timestamp), 1);
672
673 }
674
675 /* Completions */
bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw * rcfw,struct creq_qp_event * event,u32 * num_wait)676 static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
677 struct creq_qp_event *event,
678 u32 *num_wait)
679 {
680 struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq;
681 struct creq_cq_error_notification *cqerr;
682 struct creq_qp_error_notification *qperr;
683 struct bnxt_qplib_crsqe *crsqe;
684 struct bnxt_qplib_reftbl *tbl;
685 struct bnxt_qplib_qp *qp;
686 struct bnxt_qplib_cq *cq;
687 u16 cookie, blocked = 0;
688 struct pci_dev *pdev;
689 bool is_waiter_alive;
690 unsigned long flags;
691 u32 wait_cmds = 0;
692 u32 xid, qp_idx;
693 u32 req_size;
694 int rc = 0;
695
696 pdev = rcfw->pdev;
697 switch (event->event) {
698 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
699 tbl = &rcfw->res->reftbl.qpref;
700 qperr = (struct creq_qp_error_notification *)event;
701 xid = le32_to_cpu(qperr->xid);
702 qp_idx = map_qp_id_to_tbl_indx(xid, tbl);
703 spin_lock(&tbl->lock);
704 qp = tbl->rec[qp_idx].handle;
705 if (!qp) {
706 spin_unlock(&tbl->lock);
707 break;
708 }
709 bnxt_qplib_mark_qp_error(qp);
710 rc = rcfw->creq.aeq_handler(rcfw, event, qp);
711 spin_unlock(&tbl->lock);
712 /*
713 * Keeping these prints as debug to avoid flooding of log
714 * messages during modify QP to error state by applications
715 */
716 dev_dbg(&pdev->dev, "QPLIB: QP Error encountered!\n");
717 dev_dbg(&pdev->dev,
718 "QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n",
719 xid, qperr->req_err_state_reason,
720 qperr->res_err_state_reason);
721 break;
722 case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
723 tbl = &rcfw->res->reftbl.cqref;
724 cqerr = (struct creq_cq_error_notification *)event;
725 xid = le32_to_cpu(cqerr->xid);
726 spin_lock(&tbl->lock);
727 cq = tbl->rec[GET_TBL_INDEX(xid, tbl)].handle;
728 if (!cq) {
729 spin_unlock(&tbl->lock);
730 break;
731 }
732 rc = rcfw->creq.aeq_handler(rcfw, event, cq);
733 spin_unlock(&tbl->lock);
734 dev_dbg(&pdev->dev, "QPLIB: CQ error encountered!\n");
735 break;
736 default:
737 /*
738 * Command Response
739 * cmdq hwq lock needs to be acquired to synchronize
740 * the command send and completion reaping. This function
741 * is always called with creq hwq lock held. So there is no
742 * chance of deadlock here as the locking is in correct sequence.
743 * Using the nested variant of spin_lock to annotate
744 */
745 spin_lock_irqsave_nested(&cmdq_hwq->lock, flags,
746 SINGLE_DEPTH_NESTING);
747 cookie = le16_to_cpu(event->cookie);
748 blocked = cookie & RCFW_CMD_IS_BLOCKING;
749 cookie &= RCFW_MAX_COOKIE_VALUE;
750
751 crsqe = &rcfw->crsqe_tbl[cookie];
752
753 bnxt_re_add_perf_stats(rcfw, crsqe);
754
755 if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED,
756 &rcfw->cmdq.flags),
757 "QPLIB: Unreponsive rcfw channel detected.!!")) {
758 dev_info(&pdev->dev, "rcfw timedout: cookie = %#x,"
759 " latency_msec = %ld free_slots = %d\n", cookie,
760 (long)jiffies_to_msecs(rcfw->cmdq.last_seen -
761 crsqe->send_timestamp),
762 crsqe->free_slots);
763 spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
764 return rc;
765 }
766
767 if (crsqe->is_internal_cmd && !event->status)
768 atomic_dec(&rcfw->timeout_send);
769
770 if (crsqe->is_waiter_alive) {
771 if (crsqe->resp)
772 memcpy(crsqe->resp, event, sizeof(*event));
773 if (!blocked)
774 wait_cmds++;
775 }
776
777 req_size = crsqe->req_size;
778 is_waiter_alive = crsqe->is_waiter_alive;
779
780 crsqe->req_size = 0;
781 if (!crsqe->is_waiter_alive)
782 crsqe->resp = NULL;
783 crsqe->is_in_used = false;
784 /* Consumer is updated so that __send_message_no_waiter
785 * can never see queue full.
786 * It is safe since we are still holding cmdq_hwq->lock.
787 */
788 cmdq_hwq->cons += req_size;
789
790 /* This is a case to handle below scenario -
791 * Create AH is completed successfully by firmware,
792 * but completion took more time and driver already lost
793 * the context of create_ah from caller.
794 * We have already return failure for create_ah verbs,
795 * so let's destroy the same address vector since it is
796 * no more used in stack. We don't care about completion
797 * in __send_message_no_waiter.
798 * If destroy_ah is failued by firmware, there will be AH
799 * resource leak and relatively not critical + unlikely
800 * scenario. Current design is not to handle such case.
801 */
802 if (!is_waiter_alive && !event->status &&
803 event->event == CREQ_QP_EVENT_EVENT_CREATE_AH)
804 __destroy_timedout_ah(rcfw,
805 (struct creq_create_ah_resp *)
806 event);
807
808 spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
809 }
810 *num_wait += wait_cmds;
811 return rc;
812 }
813
814 /* SP - CREQ Completion handlers */
bnxt_qplib_service_creq(unsigned long data)815 static void bnxt_qplib_service_creq(unsigned long data)
816 {
817 struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;
818 struct bnxt_qplib_creq_ctx *creq = &rcfw->creq;
819 struct bnxt_qplib_res *res;
820 u32 type, budget = CREQ_ENTRY_POLL_BUDGET;
821 struct bnxt_qplib_hwq *creq_hwq = &creq->hwq;
822 struct creq_base *creqe;
823 struct pci_dev *pdev;
824 unsigned long flags;
825 u32 num_wakeup = 0;
826 int rc;
827
828 pdev = rcfw->pdev;
829 res = rcfw->res;
830 /* Service the CREQ until empty */
831 spin_lock_irqsave(&creq_hwq->lock, flags);
832 while (budget > 0) {
833 if (RCFW_NO_FW_ACCESS(rcfw)) {
834 spin_unlock_irqrestore(&creq_hwq->lock, flags);
835 return;
836 }
837 creqe = bnxt_qplib_get_qe(creq_hwq, creq_hwq->cons, NULL);
838 if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags))
839 break;
840 /* The valid test of the entry must be done first before
841 * reading any further.
842 */
843 dma_rmb();
844 type = creqe->type & CREQ_BASE_TYPE_MASK;
845 rcfw->cmdq.last_seen = jiffies;
846
847 switch (type) {
848 case CREQ_BASE_TYPE_QP_EVENT:
849 bnxt_qplib_process_qp_event
850 (rcfw,(struct creq_qp_event *)creqe,
851 &num_wakeup);
852 creq->stats.creq_qp_event_processed++;
853 break;
854 case CREQ_BASE_TYPE_FUNC_EVENT:
855 rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL);
856 if (rc)
857 dev_warn(&pdev->dev,
858 "QPLIB: async event type = 0x%x not handled",
859 type);
860 creq->stats.creq_func_event_processed++;
861 break;
862 default:
863 if (type != HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) {
864 dev_warn(&pdev->dev,
865 "QPLIB: op_event = 0x%x not handled\n",
866 type);
867 }
868 break;
869 }
870 budget--;
871 bnxt_qplib_hwq_incr_cons(creq_hwq->max_elements, &creq_hwq->cons,
872 1, &creq->creq_db.dbinfo.flags);
873 }
874 if (budget == CREQ_ENTRY_POLL_BUDGET &&
875 !CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) {
876 /* No completions received during this poll. Enable interrupt now */
877 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
878 creq->stats.creq_arm_count++;
879 dev_dbg(&pdev->dev, "QPLIB: Num of Func (0x%llx) \n",
880 creq->stats.creq_func_event_processed);
881 dev_dbg(&pdev->dev, "QPLIB: QP (0x%llx) events processed\n",
882 creq->stats.creq_qp_event_processed);
883 dev_dbg(&pdev->dev, "QPLIB: Armed:%#llx resched:%#llx \n",
884 creq->stats.creq_arm_count,
885 creq->stats.creq_tasklet_schedule_count);
886 } else if (creq->requested) {
887 /*
888 * Currently there is no bottom half implementation to process
889 * completions, all completions are processed in interrupt context
890 * only. So enable interrupts.
891 */
892 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
893 creq->stats.creq_tasklet_schedule_count++;
894 }
895 spin_unlock_irqrestore(&creq_hwq->lock, flags);
896 if (num_wakeup)
897 wake_up_all(&rcfw->cmdq.waitq);
898 }
899
bnxt_qplib_creq_irq(int irq,void * dev_instance)900 static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
901 {
902 struct bnxt_qplib_rcfw *rcfw = dev_instance;
903
904 bnxt_qplib_service_creq((unsigned long)rcfw);
905 return IRQ_HANDLED;
906 }
907
908 /* RCFW */
bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw * rcfw)909 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
910 {
911 struct creq_deinitialize_fw_resp resp = {};
912 struct cmdq_deinitialize_fw req = {};
913 struct bnxt_qplib_cmdqmsg msg = {};
914 int rc;
915
916 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEINITIALIZE_FW,
917 sizeof(req));
918 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
919 sizeof(req), sizeof(resp), 0);
920 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
921 if (rc)
922 return rc;
923 clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
924 return 0;
925 }
926
bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw * rcfw,int is_virtfn)927 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn)
928 {
929 struct creq_initialize_fw_resp resp = {};
930 struct cmdq_initialize_fw req = {};
931 struct bnxt_qplib_cmdqmsg msg = {};
932 struct bnxt_qplib_chip_ctx *cctx;
933 struct bnxt_qplib_ctx *hctx;
934 struct bnxt_qplib_res *res;
935 struct bnxt_qplib_hwq *hwq;
936 int rc;
937
938 res = rcfw->res;
939 cctx = res->cctx;
940 hctx = res->hctx;
941
942 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_INITIALIZE_FW,
943 sizeof(req));
944 /* Supply (log-base-2-of-host-page-size - base-page-shift)
945 * to bono to adjust the doorbell page sizes.
946 */
947 req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT -
948 RCFW_DBR_BASE_PAGE_SHIFT);
949 /*
950 * VFs need not setup the HW context area, PF
951 * shall setup this area for VF. Skipping the
952 * HW programming
953 */
954 if (is_virtfn || _is_chip_gen_p5_p7(cctx))
955 goto skip_ctx_setup;
956
957 hwq = &hctx->qp_ctx.hwq;
958 req.qpc_page_dir = cpu_to_le64(_get_base_addr(hwq));
959 req.number_of_qp = cpu_to_le32(hwq->max_elements);
960 req.qpc_pg_size_qpc_lvl = (_get_pte_pg_size(hwq) <<
961 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
962 (u8)hwq->level;
963
964 hwq = &hctx->mrw_ctx.hwq;
965 req.mrw_page_dir = cpu_to_le64(_get_base_addr(hwq));
966 req.number_of_mrw = cpu_to_le32(hwq->max_elements);
967 req.mrw_pg_size_mrw_lvl = (_get_pte_pg_size(hwq) <<
968 CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT) |
969 (u8)hwq->level;
970
971 hwq = &hctx->srq_ctx.hwq;
972 req.srq_page_dir = cpu_to_le64(_get_base_addr(hwq));
973 req.number_of_srq = cpu_to_le32(hwq->max_elements);
974 req.srq_pg_size_srq_lvl = (_get_pte_pg_size(hwq) <<
975 CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT) |
976 (u8)hwq->level;
977
978 hwq = &hctx->cq_ctx.hwq;
979 req.cq_page_dir = cpu_to_le64(_get_base_addr(hwq));
980 req.number_of_cq = cpu_to_le32(hwq->max_elements);
981 req.cq_pg_size_cq_lvl = (_get_pte_pg_size(hwq) <<
982 CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT) |
983 (u8)hwq->level;
984
985 hwq = &hctx->tim_ctx.hwq;
986 req.tim_page_dir = cpu_to_le64(_get_base_addr(hwq));
987 req.tim_pg_size_tim_lvl = (_get_pte_pg_size(hwq) <<
988 CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT) |
989 (u8)hwq->level;
990 hwq = &hctx->tqm_ctx.pde;
991 req.tqm_page_dir = cpu_to_le64(_get_base_addr(hwq));
992 req.tqm_pg_size_tqm_lvl = (_get_pte_pg_size(hwq) <<
993 CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT) |
994 (u8)hwq->level;
995 skip_ctx_setup:
996 if (BNXT_RE_HW_RETX(res->dattr->dev_cap_flags))
997 req.flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED;
998 req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id);
999 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
1000 sizeof(req), sizeof(resp), 0);
1001 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
1002 if (rc)
1003 return rc;
1004 set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
1005
1006 return 0;
1007 }
1008
bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res * res)1009 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res)
1010 {
1011 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1012
1013 vfree(rcfw->rcfw_lat_slab_msec);
1014 rcfw->rcfw_lat_slab_msec = NULL;
1015 vfree(rcfw->qp_create_stats);
1016 rcfw->qp_create_stats = NULL;
1017 vfree(rcfw->qp_destroy_stats);
1018 rcfw->qp_destroy_stats = NULL;
1019 vfree(rcfw->mr_create_stats);
1020 rcfw->mr_create_stats = NULL;
1021 vfree(rcfw->mr_destroy_stats);
1022 rcfw->mr_destroy_stats = NULL;
1023 vfree(rcfw->qp_modify_stats);
1024 rcfw->qp_modify_stats = NULL;
1025 rcfw->sp_perf_stats_enabled = false;
1026
1027 kfree(rcfw->crsqe_tbl);
1028 rcfw->crsqe_tbl = NULL;
1029
1030 bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);
1031 bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);
1032 rcfw->pdev = NULL;
1033 }
1034
bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res * res)1035 int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res)
1036 {
1037 struct bnxt_qplib_hwq_attr hwq_attr = {};
1038 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1039 struct bnxt_qplib_sg_info sginfo = {};
1040 struct bnxt_qplib_cmdq_ctx *cmdq;
1041 struct bnxt_qplib_creq_ctx *creq;
1042
1043 rcfw->pdev = res->pdev;
1044 rcfw->res = res;
1045 cmdq = &rcfw->cmdq;
1046 creq = &rcfw->creq;
1047
1048 sginfo.pgsize = PAGE_SIZE;
1049 sginfo.pgshft = PAGE_SHIFT;
1050
1051 hwq_attr.sginfo = &sginfo;
1052 hwq_attr.res = rcfw->res;
1053 hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT;
1054 hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS;
1055 hwq_attr.type = _get_hwq_type(res);
1056
1057 if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) {
1058 dev_err(&rcfw->pdev->dev,
1059 "QPLIB: HW channel CREQ allocation failed\n");
1060 return -ENOMEM;
1061 }
1062
1063 sginfo.pgsize = BNXT_QPLIB_CMDQE_PAGE_SIZE;
1064 hwq_attr.depth = BNXT_QPLIB_CMDQE_MAX_CNT & 0x7FFFFFFF;
1065 hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS;
1066 hwq_attr.type = HWQ_TYPE_CTX;
1067 if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) {
1068 dev_err(&rcfw->pdev->dev,
1069 "QPLIB: HW channel CMDQ allocation failed\n");
1070 goto fail_free_creq_hwq;
1071 }
1072
1073 rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements,
1074 sizeof(*rcfw->crsqe_tbl), GFP_KERNEL);
1075 if (!rcfw->crsqe_tbl) {
1076 dev_err(&rcfw->pdev->dev,
1077 "QPLIB: HW channel CRSQ allocation failed\n");
1078 goto fail_free_cmdq_hwq;
1079 }
1080
1081 rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout;
1082
1083 rcfw->sp_perf_stats_enabled = false;
1084 rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) *
1085 RCFW_MAX_LATENCY_MSEC_SLAB_INDEX);
1086 rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
1087 rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
1088 rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
1089 rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
1090 rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
1091
1092 if (rcfw->rcfw_lat_slab_msec &&
1093 rcfw->qp_create_stats &&
1094 rcfw->qp_destroy_stats &&
1095 rcfw->mr_create_stats &&
1096 rcfw->mr_destroy_stats &&
1097 rcfw->qp_modify_stats)
1098 rcfw->sp_perf_stats_enabled = true;
1099
1100 return 0;
1101 fail_free_cmdq_hwq:
1102 bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);
1103 fail_free_creq_hwq:
1104 bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);
1105 return -ENOMEM;
1106 }
1107
bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw * rcfw,bool kill)1108 void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)
1109 {
1110 struct bnxt_qplib_creq_ctx *creq;
1111 struct bnxt_qplib_res *res;
1112
1113 creq = &rcfw->creq;
1114 res = rcfw->res;
1115
1116 if (!creq->requested)
1117 return;
1118
1119 creq->requested = false;
1120 /* Mask h/w interrupts */
1121 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, false);
1122 /* Sync with last running IRQ-handler */
1123 synchronize_irq(creq->msix_vec);
1124 free_irq(creq->msix_vec, rcfw);
1125 kfree(creq->irq_name);
1126 creq->irq_name = NULL;
1127 /* rcfw_intr_enabled should not be greater than 1. Debug
1128 * print to check if that is the case
1129 */
1130 if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) {
1131 dev_err(&rcfw->pdev->dev,
1132 "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__,
1133 atomic_read(&rcfw->rcfw_intr_enabled));
1134 }
1135 atomic_set(&rcfw->rcfw_intr_enabled, 0);
1136 rcfw->num_irq_stopped++;
1137 }
1138
bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw * rcfw)1139 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
1140 {
1141 struct bnxt_qplib_creq_ctx *creq;
1142 struct bnxt_qplib_cmdq_ctx *cmdq;
1143
1144 creq = &rcfw->creq;
1145 cmdq = &rcfw->cmdq;
1146 /* Make sure the HW channel is stopped! */
1147 bnxt_qplib_rcfw_stop_irq(rcfw, true);
1148
1149 creq->creq_db.reg.bar_reg = NULL;
1150 creq->creq_db.db = NULL;
1151
1152 if (cmdq->cmdq_mbox.reg.bar_reg) {
1153 iounmap(cmdq->cmdq_mbox.reg.bar_reg);
1154 cmdq->cmdq_mbox.reg.bar_reg = NULL;
1155 cmdq->cmdq_mbox.prod = NULL;
1156 cmdq->cmdq_mbox.db = NULL;
1157 }
1158
1159 creq->aeq_handler = NULL;
1160 creq->msix_vec = 0;
1161 }
1162
bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw * rcfw,int msix_vector,bool need_init)1163 int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
1164 bool need_init)
1165 {
1166 struct bnxt_qplib_creq_ctx *creq;
1167 struct bnxt_qplib_res *res;
1168 int rc;
1169
1170 creq = &rcfw->creq;
1171 res = rcfw->res;
1172
1173 if (creq->requested)
1174 return -EFAULT;
1175
1176 creq->msix_vec = msix_vector;
1177
1178 creq->irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s\n",
1179 pci_name(res->pdev));
1180 if (!creq->irq_name)
1181 return -ENOMEM;
1182
1183 rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0,
1184 creq->irq_name, rcfw);
1185 if (rc) {
1186 kfree(creq->irq_name);
1187 creq->irq_name = NULL;
1188 return rc;
1189 }
1190 creq->requested = true;
1191
1192 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
1193
1194 rcfw->num_irq_started++;
1195 /* Debug print to check rcfw interrupt enable/disable is invoked
1196 * out of sequence
1197 */
1198 if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) {
1199 dev_err(&rcfw->pdev->dev,
1200 "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__,
1201 atomic_read(&rcfw->rcfw_intr_enabled));
1202 }
1203 atomic_inc(&rcfw->rcfw_intr_enabled);
1204 return 0;
1205 }
1206
bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw * rcfw)1207 static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw)
1208 {
1209 struct bnxt_qplib_cmdq_mbox *mbox;
1210 resource_size_t bar_reg;
1211 struct pci_dev *pdev;
1212
1213 pdev = rcfw->pdev;
1214 mbox = &rcfw->cmdq.cmdq_mbox;
1215
1216 mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION;
1217 mbox->reg.len = RCFW_COMM_SIZE;
1218 mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id);
1219 if (!mbox->reg.bar_base) {
1220 dev_err(&pdev->dev,
1221 "QPLIB: CMDQ BAR region %d resc start is 0!\n",
1222 mbox->reg.bar_id);
1223 return -ENOMEM;
1224 }
1225
1226 bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET;
1227 mbox->reg.len = RCFW_COMM_SIZE;
1228 mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len);
1229 if (!mbox->reg.bar_reg) {
1230 dev_err(&pdev->dev,
1231 "QPLIB: CMDQ BAR region %d mapping failed\n",
1232 mbox->reg.bar_id);
1233 return -ENOMEM;
1234 }
1235
1236 mbox->prod = (void __iomem *)((char *)mbox->reg.bar_reg +
1237 RCFW_PF_VF_COMM_PROD_OFFSET);
1238 mbox->db = (void __iomem *)((char *)mbox->reg.bar_reg +
1239 RCFW_COMM_TRIG_OFFSET);
1240 return 0;
1241 }
1242
bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw * rcfw,u32 reg_offt)1243 static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)
1244 {
1245 struct bnxt_qplib_creq_db *creq_db;
1246 struct bnxt_qplib_reg_desc *dbreg;
1247 struct bnxt_qplib_res *res;
1248
1249 res = rcfw->res;
1250 creq_db = &rcfw->creq.creq_db;
1251 dbreg = &res->dpi_tbl.ucreg;
1252
1253 creq_db->reg.bar_id = dbreg->bar_id;
1254 creq_db->reg.bar_base = dbreg->bar_base;
1255 creq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
1256 creq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :
1257 sizeof(u32);
1258
1259 creq_db->dbinfo.db = creq_db->reg.bar_reg;
1260 creq_db->dbinfo.hwq = &rcfw->creq.hwq;
1261 creq_db->dbinfo.xid = rcfw->creq.ring_id;
1262 creq_db->dbinfo.seed = rcfw->creq.ring_id;
1263 creq_db->dbinfo.flags = 0;
1264 spin_lock_init(&creq_db->dbinfo.lock);
1265 creq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
1266 creq_db->dbinfo.res = rcfw->res;
1267
1268 return 0;
1269 }
1270
bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw * rcfw)1271 static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw)
1272 {
1273 struct bnxt_qplib_cmdq_ctx *cmdq;
1274 struct bnxt_qplib_creq_ctx *creq;
1275 struct bnxt_qplib_cmdq_mbox *mbox;
1276 struct cmdq_init init = {0};
1277
1278 cmdq = &rcfw->cmdq;
1279 creq = &rcfw->creq;
1280 mbox = &cmdq->cmdq_mbox;
1281
1282 init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
1283 init.cmdq_size_cmdq_lvl = cpu_to_le16(
1284 ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) &
1285 CMDQ_INIT_CMDQ_SIZE_MASK) |
1286 ((cmdq->hwq.level << CMDQ_INIT_CMDQ_LVL_SFT) &
1287 CMDQ_INIT_CMDQ_LVL_MASK));
1288 init.creq_ring_id = cpu_to_le16(creq->ring_id);
1289 /* Write to the Bono mailbox register */
1290 __iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4);
1291 }
1292
bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw * rcfw,int msix_vector,int cp_bar_reg_off,aeq_handler_t aeq_handler)1293 int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
1294 int msix_vector,
1295 int cp_bar_reg_off,
1296 aeq_handler_t aeq_handler)
1297 {
1298 struct bnxt_qplib_cmdq_ctx *cmdq;
1299 struct bnxt_qplib_creq_ctx *creq;
1300 int rc;
1301
1302 cmdq = &rcfw->cmdq;
1303 creq = &rcfw->creq;
1304
1305 /* Clear to defaults */
1306 cmdq->seq_num = 0;
1307 set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
1308 init_waitqueue_head(&cmdq->waitq);
1309
1310 creq->stats.creq_qp_event_processed = 0;
1311 creq->stats.creq_func_event_processed = 0;
1312 creq->aeq_handler = aeq_handler;
1313
1314 rc = bnxt_qplib_map_cmdq_mbox(rcfw);
1315 if (rc)
1316 return rc;
1317
1318 rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off);
1319 if (rc)
1320 return rc;
1321
1322 rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);
1323 if (rc) {
1324 dev_err(&rcfw->pdev->dev,
1325 "QPLIB: Failed to request IRQ for CREQ rc = 0x%x\n", rc);
1326 bnxt_qplib_disable_rcfw_channel(rcfw);
1327 return rc;
1328 }
1329
1330 rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd,
1331 (unsigned int)RCFW_CMD_NON_BLOCKING_SHADOW_QD);
1332 sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd);
1333 dev_dbg(&rcfw->pdev->dev,
1334 "Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd);
1335 bnxt_qplib_start_rcfw(rcfw);
1336
1337 return 0;
1338 }
1339