1 /*
2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term
3 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Description: RDMA Controller HW interface (header)
29 */
30
31 #ifndef __BNXT_QPLIB_RCFW_H__
32 #define __BNXT_QPLIB_RCFW_H__
33
34 #include <linux/semaphore.h>
35 #include <linux/module.h>
36 #include <linux/netdevice.h>
37 #include <linux/mutex.h>
38 #include <linux/list.h>
39 #include <linux/rculist.h>
40 #include <linux/spinlock.h>
41 #include <linux/pci.h>
42 #include <net/ipv6.h>
43 #include <linux/if_ether.h>
44 #include <linux/debugfs.h>
45 #include <linux/seq_file.h>
46 #include <linux/interrupt.h>
47 #include <linux/vmalloc.h>
48 #include <linux/delay.h>
49
50 #include "qplib_tlv.h"
51
52 #define RCFW_CMDQ_TRIG_VAL 1
53 #define RCFW_COMM_PCI_BAR_REGION 0
54 #define RCFW_COMM_CONS_PCI_BAR_REGION 2
55 #define RCFW_COMM_BASE_OFFSET 0x600
56 #define RCFW_PF_VF_COMM_PROD_OFFSET 0xc
57 #define RCFW_COMM_TRIG_OFFSET 0x100
58 #define RCFW_COMM_SIZE 0x104
59
60 #define RCFW_DBR_PCI_BAR_REGION 2
61 #define RCFW_DBR_BASE_PAGE_SHIFT 12
62 #define RCFW_MAX_LATENCY_SEC_SLAB_INDEX 128
63 #define RCFW_MAX_LATENCY_MSEC_SLAB_INDEX 3000
64 #define RCFW_MAX_STAT_INDEX 0xFFFF
65 #define RCFW_FW_STALL_MAX_TIMEOUT 40
66
67 #define GET_OPCODE_TYPE(x) \
68 ((x) == 0x1 ? "CREATE_QP": \
69 ((x) == 0x2 ? "DESTROY_QP": \
70 ((x) == 0x3 ? "MODIFY_QP": \
71 ((x) == 0x4 ? "QUERY_QP": \
72 ((x) == 0x5 ? "CREATE_SRQ": \
73 ((x) == 0x6 ? "DESTROY_SRQ": \
74 ((x) == 0x8 ? "QUERY_SRQ": \
75 ((x) == 0x9 ? "CREATE_CQ": \
76 ((x) == 0xa ? "DESTROY_CQ": \
77 ((x) == 0xc ? "RESIZE_CQ": \
78 ((x) == 0xd ? "ALLOCATE_MRW": \
79 ((x) == 0xe ? "DEALLOCATE_KEY": \
80 ((x) == 0xf ? "REGISTER_MR": \
81 ((x) == 0x10 ? "DEREGISTER_MR": \
82 ((x) == 0x11 ? "ADD_GID": \
83 ((x) == 0x12 ? "DELETE_GID": \
84 ((x) == 0x17 ? "MODIFY_GID": \
85 ((x) == 0x18 ? "QUERY_GID": \
86 ((x) == 0x13 ? "CREATE_QP1": \
87 ((x) == 0x14 ? "DESTROY_QP1": \
88 ((x) == 0x15 ? "CREATE_AH": \
89 ((x) == 0x16 ? "DESTROY_AH": \
90 ((x) == 0x80 ? "INITIALIZE_FW": \
91 ((x) == 0x81 ? "DEINITIALIZE_FW": \
92 ((x) == 0x82 ? "STOP_FUNC": \
93 ((x) == 0x83 ? "QUERY_FUNC": \
94 ((x) == 0x84 ? "SET_FUNC_RESOURCES": \
95 ((x) == 0x85 ? "READ_CONTEXT": \
96 ((x) == 0x86 ? "VF_BACKCHANNEL_REQUEST": \
97 ((x) == 0x87 ? "READ_VF_MEMORY": \
98 ((x) == 0x88 ? "COMPLETE_VF_REQUEST": \
99 ((x) == 0x89 ? "EXTEND_CONTEXT_ARRRAY": \
100 ((x) == 0x8a ? "MAP_TC_TO_COS": \
101 ((x) == 0x8b ? "QUERY_VERSION": \
102 ((x) == 0x8c ? "MODIFY_ROCE_CC": \
103 ((x) == 0x8d ? "QUERY_ROCE_CC": \
104 ((x) == 0x8e ? "QUERY_ROCE_STATS": \
105 ((x) == 0x8f ? "SET_LINK_AGGR_MODE": \
106 ((x) == 0x90 ? "MODIFY_CQ": \
107 ((x) == 0x91 ? "QUERY_QP_EXTEND": \
108 ((x) == 0x92 ? "QUERY_ROCE_STATS_EXT": \
109 "Unknown OPCODE" \
110 )))))))))))))))))))))))))))))))))))))))))
111
112 extern unsigned int cmdq_shadow_qd;
113 /* Cmdq contains a fix number of a 16-Byte slots */
114 struct bnxt_qplib_cmdqe {
115 u8 data[16];
116 };
117 #define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe)
118
bnxt_qplib_rcfw_cmd_prep(void * r,u8 opcode,u8 cmd_size)119 static inline void bnxt_qplib_rcfw_cmd_prep(void *r, u8 opcode, u8 cmd_size)
120 {
121 struct cmdq_base *req = r;
122
123 req->opcode = opcode;
124 req->cmd_size = cmd_size;
125 }
126
127 /* Shadow queue depth for non blocking command */
128 #define RCFW_CMD_NON_BLOCKING_SHADOW_QD 64
129 #define RCFW_CMD_DEV_ERR_CHECK_TIME_MS 1000 /* 1 Second time out*/
130 #define RCFW_ERR_RETRY_COUNT (RCFW_CMD_WAIT_TIME_MS / RCFW_CMD_DEV_ERR_CHECK_TIME_MS)
131
132 /* CMDQ elements */
133 #define BNXT_QPLIB_CMDQE_MAX_CNT 8192
134 #define BNXT_QPLIB_CMDQE_BYTES (BNXT_QPLIB_CMDQE_MAX_CNT * \
135 BNXT_QPLIB_CMDQE_UNITS)
136 #define BNXT_QPLIB_CMDQE_NPAGES ((BNXT_QPLIB_CMDQE_BYTES % \
137 PAGE_SIZE) ? \
138 ((BNXT_QPLIB_CMDQE_BYTES / \
139 PAGE_SIZE) + 1) : \
140 (BNXT_QPLIB_CMDQE_BYTES / \
141 PAGE_SIZE))
142 #define BNXT_QPLIB_CMDQE_PAGE_SIZE (BNXT_QPLIB_CMDQE_NPAGES * \
143 PAGE_SIZE)
144
145 #define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT
146 #define RCFW_MAX_COOKIE_VALUE (BNXT_QPLIB_CMDQE_MAX_CNT - 1)
147 #define RCFW_CMD_IS_BLOCKING 0x8000
148 #define RCFW_NO_FW_ACCESS(rcfw) \
149 (test_bit(ERR_DEVICE_DETACHED, &(rcfw)->cmdq.flags) || \
150 pci_channel_offline((rcfw)->pdev))
151
152 /* Crsq buf is 1024-Byte */
153 struct bnxt_qplib_crsbe {
154 u8 data[1024];
155 };
156
157 /* Get the number of command units required for the req. The
158 * function returns correct value only if called before
159 * setting using bnxt_qplib_set_cmd_slots
160 */
bnxt_qplib_get_cmd_slots(struct cmdq_base * req)161 static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req)
162 {
163 u32 cmd_units = 0;
164
165 if (HAS_TLV_HEADER(req)) {
166 struct roce_tlv *tlv_req = (struct roce_tlv *)req;
167 cmd_units = tlv_req->total_size;
168 } else {
169 cmd_units = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
170 BNXT_QPLIB_CMDQE_UNITS;
171 }
172 return cmd_units;
173 }
174
175 /* Set the cmd_size to a factor of CMDQE unit */
bnxt_qplib_set_cmd_slots(struct cmdq_base * req)176 static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
177 {
178 u32 cmd_byte = 0;
179
180 if (HAS_TLV_HEADER(req)) {
181 struct roce_tlv *tlv_req = (struct roce_tlv *)req;
182 cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS;
183 } else {
184 cmd_byte = req->cmd_size;
185 req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
186 BNXT_QPLIB_CMDQE_UNITS;
187 }
188
189 return cmd_byte;
190 }
191
192 /* CREQ */
193 /* Allocate 1 per QP for async error notification for now */
194 #define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024)
195 #define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */
196
197 #define CREQ_CMP_VALID(hdr, pass) \
198 (!!((hdr)->v & CREQ_BASE_V) == \
199 !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
200
201 #define CREQ_ENTRY_POLL_BUDGET 8
202
203 typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *);
204
205 struct bnxt_qplib_crsqe {
206 struct creq_qp_event *resp;
207 u32 req_size;
208 bool is_waiter_alive;
209 bool is_internal_cmd;
210 bool is_in_used;
211
212 /* Free slots at the time of submission */
213 u32 free_slots;
214 unsigned long send_timestamp;
215 u8 opcode;
216 u8 requested_qp_state;
217 };
218
219 struct bnxt_qplib_rcfw_sbuf {
220 void *sb;
221 dma_addr_t dma_addr;
222 u32 size;
223 };
224
225 #define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF
226
227 #define FIRMWARE_INITIALIZED_FLAG (0)
228 #define FIRMWARE_FIRST_FLAG (31)
229 #define FIRMWARE_STALL_DETECTED (3)
230 #define ERR_DEVICE_DETACHED (4)
231 struct bnxt_qplib_cmdq_mbox {
232 struct bnxt_qplib_reg_desc reg;
233 void __iomem *prod;
234 void __iomem *db;
235 };
236
237 struct bnxt_qplib_cmdq_ctx {
238 struct bnxt_qplib_hwq hwq;
239 struct bnxt_qplib_cmdq_mbox cmdq_mbox;
240 wait_queue_head_t waitq;
241 unsigned long flags;
242 unsigned long last_seen;
243 u32 seq_num;
244 };
245
246 struct bnxt_qplib_creq_db {
247 struct bnxt_qplib_reg_desc reg;
248 void __iomem *db;
249 struct bnxt_qplib_db_info dbinfo;
250 };
251
252 struct bnxt_qplib_creq_stat {
253 u64 creq_arm_count;
254 u64 creq_tasklet_schedule_count;
255 u64 creq_qp_event_processed;
256 u64 creq_func_event_processed;
257 };
258
259 struct bnxt_qplib_creq_ctx {
260 struct bnxt_qplib_hwq hwq;
261 struct bnxt_qplib_creq_db creq_db;
262 struct bnxt_qplib_creq_stat stats;
263 aeq_handler_t aeq_handler;
264 u16 ring_id;
265 int msix_vec;
266 bool requested;
267 char *irq_name;
268 };
269
270 /* RCFW Communication Channels */
271 #define BNXT_QPLIB_RCFW_SEND_RETRY_COUNT 4000
272 struct bnxt_qplib_rcfw {
273 struct pci_dev *pdev;
274 struct bnxt_qplib_res *res;
275 struct bnxt_qplib_cmdq_ctx cmdq;
276 struct bnxt_qplib_creq_ctx creq;
277 struct bnxt_qplib_crsqe *crsqe_tbl;
278 u32 rcfw_lat_slab_sec[RCFW_MAX_LATENCY_SEC_SLAB_INDEX];
279
280 /* Slow path Perf Stats */
281 bool sp_perf_stats_enabled;
282 u32 *rcfw_lat_slab_msec;
283 u64 *qp_create_stats;
284 u64 *qp_destroy_stats;
285 u64 *qp_modify_stats;
286 u64 *mr_create_stats;
287 u64 *mr_destroy_stats;
288 u32 qp_create_stats_id;
289 u32 qp_destroy_stats_id;
290 u32 qp_modify_stats_id;
291 u32 mr_create_stats_id;
292 u32 mr_destroy_stats_id;
293 bool init_oos_stats;
294 u64 oos_prev;
295 u32 num_irq_stopped;
296 u32 num_irq_started;
297 u32 poll_in_intr_en;
298 u32 poll_in_intr_dis;
299 atomic_t rcfw_intr_enabled;
300 u32 cmdq_full_dbg;
301 struct semaphore rcfw_inflight;
302 unsigned int curr_shadow_qd;
303 atomic_t timeout_send;
304 /* cached from chip cctx for quick reference in slow path */
305 u16 max_timeout;
306 };
307
308 struct bnxt_qplib_cmdqmsg {
309 struct cmdq_base *req;
310 struct creq_base *resp;
311 void *sb;
312 u32 req_sz;
313 u32 res_sz;
314 u8 block;
315 u8 qp_state;
316 };
317
bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg * msg,void * req,void * resp,void * sb,u32 req_sz,u32 res_sz,u8 block)318 static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg,
319 void *req, void *resp, void *sb,
320 u32 req_sz, u32 res_sz, u8 block)
321 {
322 msg->req = req;
323 msg->resp = resp;
324 msg->sb = sb;
325 msg->req_sz = req_sz;
326 msg->res_sz = res_sz;
327 msg->block = block;
328 }
329
330 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res);
331 int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res);
332 void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill);
333 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
334 int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
335 bool need_init);
336 int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
337 int msix_vector,
338 int cp_bar_reg_off,
339 aeq_handler_t aeq_handler);
340
341 struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(
342 struct bnxt_qplib_rcfw *rcfw,
343 u32 size);
344 void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw,
345 struct bnxt_qplib_rcfw_sbuf *sbuf);
346 int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
347 struct bnxt_qplib_cmdqmsg *msg);
348
349 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
350 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn);
351 void bnxt_qplib_mark_qp_error(void *qp_handle);
352 int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,
353 u32 *cur_prod, u32 *cur_cons);
354 #endif
355