1 /*
2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #ifndef __MLX5_EN_H__
33 #define __MLX5_EN_H__
34
35 #include <linux/if_vlan.h>
36 #include <linux/etherdevice.h>
37 #include <linux/timecounter.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/crash_dump.h>
40 #include <linux/mlx5/driver.h>
41 #include <linux/mlx5/qp.h>
42 #include <linux/mlx5/cq.h>
43 #include <linux/mlx5/port.h>
44 #include <linux/mlx5/vport.h>
45 #include <linux/mlx5/transobj.h>
46 #include <linux/mlx5/fs.h>
47 #include <linux/rhashtable.h>
48 #include <net/udp_tunnel.h>
49 #include <net/switchdev.h>
50 #include <net/xdp.h>
51 #include <linux/dim.h>
52 #include <linux/bits.h>
53 #include "wq.h"
54 #include "mlx5_core.h"
55 #include "en_stats.h"
56 #include "en/dcbnl.h"
57 #include "en/fs.h"
58 #include "en/qos.h"
59 #include "lib/hv_vhca.h"
60 #include "lib/clock.h"
61
62 extern const struct net_device_ops mlx5e_netdev_ops;
63 struct page_pool;
64
65 #define MLX5E_METADATA_ETHER_TYPE (0x8CE4)
66 #define MLX5E_METADATA_ETHER_LEN 8
67
68 #define MLX5_SET_CFG(p, f, v) MLX5_SET(create_flow_group_in, p, f, v)
69
70 #define MLX5E_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
71
72 #define MLX5E_HW2SW_MTU(params, hwmtu) ((hwmtu) - ((params)->hard_mtu))
73 #define MLX5E_SW2HW_MTU(params, swmtu) ((swmtu) + ((params)->hard_mtu))
74
75 #define MLX5E_MAX_NUM_TC 8
76
77 #define MLX5_RX_HEADROOM NET_SKB_PAD
78 #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
79 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
80
81 #define MLX5E_RX_MAX_HEAD (256)
82
83 #define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
84 (6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
85 #define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \
86 max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req)
87 #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
88 MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
89
90 #define MLX5_MPWRQ_LOG_WQE_SZ 18
91 #define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
92 MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0)
93 #define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
94
95 #define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8))
96 #define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2)
97 #define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts)))
98 /* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between
99 * WQEs, This page will absorb write overflow by the hardware, when
100 * receiving packets larger than MTU. These oversize packets are
101 * dropped by the driver at a later stage.
102 */
103 #define MLX5E_REQUIRED_WQE_MTTS (MLX5_ALIGN_MTTS(MLX5_MPWRQ_PAGES_PER_WQE + 1))
104 #define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS)
105 #define MLX5E_MAX_RQ_NUM_MTTS \
106 ((1 << 16) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
107 #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
108 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \
109 (ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS))
110 #define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \
111 (MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \
112 (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU))
113
114 #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
115 #define MLX5E_LOG_MAX_RX_WQE_BULK \
116 (ilog2(PAGE_SIZE / roundup_pow_of_two(MLX5E_MIN_SKB_FRAG_SZ)))
117
118 #define MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE 0x6
119 #define MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE 0xa
120 #define MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE 0xd
121
122 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK)
123 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa
124 #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \
125 MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW)
126
127 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
128
129 #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
130 #define MLX5E_DEFAULT_LRO_TIMEOUT 32
131 #define MLX5E_LRO_TIMEOUT_ARR_SIZE 4
132
133 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10
134 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3
135 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20
136 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC 0x10
137 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE 0x10
138 #define MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS 0x20
139 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES 0x80
140 #define MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW 0x2
141
142 #define MLX5E_LOG_INDIR_RQT_SIZE 0x8
143 #define MLX5E_INDIR_RQT_SIZE BIT(MLX5E_LOG_INDIR_RQT_SIZE)
144 #define MLX5E_MIN_NUM_CHANNELS 0x1
145 #define MLX5E_MAX_NUM_CHANNELS (MLX5E_INDIR_RQT_SIZE / 2)
146 #define MLX5E_MAX_NUM_SQS (MLX5E_MAX_NUM_CHANNELS * MLX5E_MAX_NUM_TC)
147 #define MLX5E_TX_CQ_POLL_BUDGET 128
148 #define MLX5E_TX_XSK_POLL_BUDGET 64
149 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */
150
151 #define MLX5E_UMR_WQE_INLINE_SZ \
152 (sizeof(struct mlx5e_umr_wqe) + \
153 ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(struct mlx5_mtt), \
154 MLX5_UMR_MTT_ALIGNMENT))
155 #define MLX5E_UMR_WQEBBS \
156 (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
157
158 #define MLX5E_MSG_LEVEL NETIF_MSG_LINK
159
160 #define mlx5e_dbg(mlevel, priv, format, ...) \
161 do { \
162 if (NETIF_MSG_##mlevel & (priv)->msglevel) \
163 netdev_warn(priv->netdev, format, \
164 ##__VA_ARGS__); \
165 } while (0)
166
167 #define mlx5e_state_dereference(priv, p) \
168 rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock))
169
170 enum mlx5e_rq_group {
171 MLX5E_RQ_GROUP_REGULAR,
172 MLX5E_RQ_GROUP_XSK,
173 #define MLX5E_NUM_RQ_GROUPS(g) (1 + MLX5E_RQ_GROUP_##g)
174 };
175
mlx5e_get_num_lag_ports(struct mlx5_core_dev * mdev)176 static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev)
177 {
178 if (mlx5_lag_is_lacp_owner(mdev))
179 return 1;
180
181 return clamp_t(u8, MLX5_CAP_GEN(mdev, num_lag_ports), 1, MLX5_MAX_PORTS);
182 }
183
mlx5_min_rx_wqes(int wq_type,u32 wq_size)184 static inline u16 mlx5_min_rx_wqes(int wq_type, u32 wq_size)
185 {
186 switch (wq_type) {
187 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
188 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES_MPW,
189 wq_size / 2);
190 default:
191 return min_t(u16, MLX5E_PARAMS_DEFAULT_MIN_RX_WQES,
192 wq_size / 2);
193 }
194 }
195
196 /* Use this function to get max num channels (rxqs/txqs) only to create netdev */
mlx5e_get_max_num_channels(struct mlx5_core_dev * mdev)197 static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
198 {
199 return is_kdump_kernel() ?
200 MLX5E_MIN_NUM_CHANNELS :
201 min_t(int, mlx5_comp_vectors_count(mdev), MLX5E_MAX_NUM_CHANNELS);
202 }
203
204 struct mlx5e_tx_wqe {
205 struct mlx5_wqe_ctrl_seg ctrl;
206 struct mlx5_wqe_eth_seg eth;
207 struct mlx5_wqe_data_seg data[0];
208 };
209
210 struct mlx5e_rx_wqe_ll {
211 struct mlx5_wqe_srq_next_seg next;
212 struct mlx5_wqe_data_seg data[];
213 };
214
215 struct mlx5e_rx_wqe_cyc {
216 struct mlx5_wqe_data_seg data[0];
217 };
218
219 struct mlx5e_umr_wqe {
220 struct mlx5_wqe_ctrl_seg ctrl;
221 struct mlx5_wqe_umr_ctrl_seg uctrl;
222 struct mlx5_mkey_seg mkc;
223 struct mlx5_mtt inline_mtts[0];
224 };
225
226 extern const char mlx5e_self_tests[][ETH_GSTRING_LEN];
227
228 enum mlx5e_priv_flag {
229 MLX5E_PFLAG_RX_CQE_BASED_MODER,
230 MLX5E_PFLAG_TX_CQE_BASED_MODER,
231 MLX5E_PFLAG_RX_CQE_COMPRESS,
232 MLX5E_PFLAG_RX_STRIDING_RQ,
233 MLX5E_PFLAG_RX_NO_CSUM_COMPLETE,
234 MLX5E_PFLAG_XDP_TX_MPWQE,
235 MLX5E_PFLAG_SKB_TX_MPWQE,
236 MLX5E_PFLAG_TX_PORT_TS,
237 MLX5E_NUM_PFLAGS, /* Keep last */
238 };
239
240 #define MLX5E_SET_PFLAG(params, pflag, enable) \
241 do { \
242 if (enable) \
243 (params)->pflags |= BIT(pflag); \
244 else \
245 (params)->pflags &= ~(BIT(pflag)); \
246 } while (0)
247
248 #define MLX5E_GET_PFLAG(params, pflag) (!!((params)->pflags & (BIT(pflag))))
249
250 struct mlx5e_params {
251 u8 log_sq_size;
252 u8 rq_wq_type;
253 u8 log_rq_mtu_frames;
254 u16 num_channels;
255 u8 num_tc;
256 bool rx_cqe_compress_def;
257 bool tunneled_offload_en;
258 struct dim_cq_moder rx_cq_moderation;
259 struct dim_cq_moder tx_cq_moderation;
260 bool lro_en;
261 u8 tx_min_inline_mode;
262 bool vlan_strip_disable;
263 bool scatter_fcs_en;
264 bool rx_dim_enabled;
265 bool tx_dim_enabled;
266 u32 lro_timeout;
267 u32 pflags;
268 struct bpf_prog *xdp_prog;
269 struct mlx5e_xsk *xsk;
270 unsigned int sw_mtu;
271 int hard_mtu;
272 bool ptp_rx;
273 };
274
275 enum {
276 MLX5E_RQ_STATE_ENABLED,
277 MLX5E_RQ_STATE_RECOVERING,
278 MLX5E_RQ_STATE_AM,
279 MLX5E_RQ_STATE_NO_CSUM_COMPLETE,
280 MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */
281 MLX5E_RQ_STATE_FPGA_TLS, /* FPGA TLS enabled */
282 MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX /* set when mini_cqe_resp_stride_index cap is used */
283 };
284
285 struct mlx5e_cq {
286 /* data path - accessed per cqe */
287 struct mlx5_cqwq wq;
288
289 /* data path - accessed per napi poll */
290 u16 event_ctr;
291 struct napi_struct *napi;
292 struct mlx5_core_cq mcq;
293 struct mlx5e_ch_stats *ch_stats;
294
295 /* control */
296 struct net_device *netdev;
297 struct mlx5_core_dev *mdev;
298 struct mlx5e_priv *priv;
299 struct mlx5_wq_ctrl wq_ctrl;
300 } ____cacheline_aligned_in_smp;
301
302 struct mlx5e_cq_decomp {
303 /* cqe decompression */
304 struct mlx5_cqe64 title;
305 struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
306 u8 mini_arr_idx;
307 u16 left;
308 u16 wqe_counter;
309 } ____cacheline_aligned_in_smp;
310
311 enum mlx5e_dma_map_type {
312 MLX5E_DMA_MAP_SINGLE,
313 MLX5E_DMA_MAP_PAGE
314 };
315
316 struct mlx5e_sq_dma {
317 dma_addr_t addr;
318 u32 size;
319 enum mlx5e_dma_map_type type;
320 };
321
322 enum {
323 MLX5E_SQ_STATE_ENABLED,
324 MLX5E_SQ_STATE_MPWQE,
325 MLX5E_SQ_STATE_RECOVERING,
326 MLX5E_SQ_STATE_IPSEC,
327 MLX5E_SQ_STATE_AM,
328 MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE,
329 MLX5E_SQ_STATE_PENDING_XSK_TX,
330 MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC,
331 };
332
333 struct mlx5e_tx_mpwqe {
334 /* Current MPWQE session */
335 struct mlx5e_tx_wqe *wqe;
336 u32 bytes_count;
337 u8 ds_count;
338 u8 pkt_count;
339 u8 inline_on;
340 };
341
342 struct mlx5e_skb_fifo {
343 struct sk_buff **fifo;
344 u16 *pc;
345 u16 *cc;
346 u16 mask;
347 };
348
349 struct mlx5e_ptpsq;
350
351 struct mlx5e_txqsq {
352 /* data path */
353
354 /* dirtied @completion */
355 u16 cc;
356 u16 skb_fifo_cc;
357 u32 dma_fifo_cc;
358 struct dim dim; /* Adaptive Moderation */
359
360 /* dirtied @xmit */
361 u16 pc ____cacheline_aligned_in_smp;
362 u16 skb_fifo_pc;
363 u32 dma_fifo_pc;
364 struct mlx5e_tx_mpwqe mpwqe;
365
366 struct mlx5e_cq cq;
367
368 /* read only */
369 struct mlx5_wq_cyc wq;
370 u32 dma_fifo_mask;
371 struct mlx5e_sq_stats *stats;
372 struct {
373 struct mlx5e_sq_dma *dma_fifo;
374 struct mlx5e_skb_fifo skb_fifo;
375 struct mlx5e_tx_wqe_info *wqe_info;
376 } db;
377 void __iomem *uar_map;
378 struct netdev_queue *txq;
379 u32 sqn;
380 u16 stop_room;
381 u8 min_inline_mode;
382 struct device *pdev;
383 __be32 mkey_be;
384 unsigned long state;
385 unsigned int hw_mtu;
386 struct hwtstamp_config *tstamp;
387 struct mlx5_clock *clock;
388 struct net_device *netdev;
389 struct mlx5_core_dev *mdev;
390 struct mlx5e_priv *priv;
391
392 /* control path */
393 struct mlx5_wq_ctrl wq_ctrl;
394 int ch_ix;
395 int txq_ix;
396 u32 rate_limit;
397 struct work_struct recover_work;
398 struct mlx5e_ptpsq *ptpsq;
399 cqe_ts_to_ns ptp_cyc2time;
400 } ____cacheline_aligned_in_smp;
401
402 struct mlx5e_dma_info {
403 dma_addr_t addr;
404 union {
405 struct page *page;
406 struct xdp_buff *xsk;
407 };
408 };
409
410 /* XDP packets can be transmitted in different ways. On completion, we need to
411 * distinguish between them to clean up things in a proper way.
412 */
413 enum mlx5e_xdp_xmit_mode {
414 /* An xdp_frame was transmitted due to either XDP_REDIRECT from another
415 * device or XDP_TX from an XSK RQ. The frame has to be unmapped and
416 * returned.
417 */
418 MLX5E_XDP_XMIT_MODE_FRAME,
419
420 /* The xdp_frame was created in place as a result of XDP_TX from a
421 * regular RQ. No DMA remapping happened, and the page belongs to us.
422 */
423 MLX5E_XDP_XMIT_MODE_PAGE,
424
425 /* No xdp_frame was created at all, the transmit happened from a UMEM
426 * page. The UMEM Completion Ring producer pointer has to be increased.
427 */
428 MLX5E_XDP_XMIT_MODE_XSK,
429 };
430
431 struct mlx5e_xdp_info {
432 enum mlx5e_xdp_xmit_mode mode;
433 union {
434 struct {
435 struct xdp_frame *xdpf;
436 dma_addr_t dma_addr;
437 } frame;
438 struct {
439 struct mlx5e_rq *rq;
440 struct mlx5e_dma_info di;
441 } page;
442 };
443 };
444
445 struct mlx5e_xmit_data {
446 dma_addr_t dma_addr;
447 void *data;
448 u32 len;
449 };
450
451 struct mlx5e_xdp_info_fifo {
452 struct mlx5e_xdp_info *xi;
453 u32 *cc;
454 u32 *pc;
455 u32 mask;
456 };
457
458 struct mlx5e_xdpsq;
459 typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *);
460 typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *,
461 struct mlx5e_xmit_data *,
462 struct mlx5e_xdp_info *,
463 int);
464
465 struct mlx5e_xdpsq {
466 /* data path */
467
468 /* dirtied @completion */
469 u32 xdpi_fifo_cc;
470 u16 cc;
471
472 /* dirtied @xmit */
473 u32 xdpi_fifo_pc ____cacheline_aligned_in_smp;
474 u16 pc;
475 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
476 struct mlx5e_tx_mpwqe mpwqe;
477
478 struct mlx5e_cq cq;
479
480 /* read only */
481 struct xsk_buff_pool *xsk_pool;
482 struct mlx5_wq_cyc wq;
483 struct mlx5e_xdpsq_stats *stats;
484 mlx5e_fp_xmit_xdp_frame_check xmit_xdp_frame_check;
485 mlx5e_fp_xmit_xdp_frame xmit_xdp_frame;
486 struct {
487 struct mlx5e_xdp_wqe_info *wqe_info;
488 struct mlx5e_xdp_info_fifo xdpi_fifo;
489 } db;
490 void __iomem *uar_map;
491 u32 sqn;
492 struct device *pdev;
493 __be32 mkey_be;
494 u8 min_inline_mode;
495 unsigned long state;
496 unsigned int hw_mtu;
497
498 /* control path */
499 struct mlx5_wq_ctrl wq_ctrl;
500 struct mlx5e_channel *channel;
501 } ____cacheline_aligned_in_smp;
502
503 struct mlx5e_ktls_resync_resp;
504
505 struct mlx5e_icosq {
506 /* data path */
507 u16 cc;
508 u16 pc;
509
510 struct mlx5_wqe_ctrl_seg *doorbell_cseg;
511 struct mlx5e_cq cq;
512
513 /* write@xmit, read@completion */
514 struct {
515 struct mlx5e_icosq_wqe_info *wqe_info;
516 } db;
517
518 /* read only */
519 struct mlx5_wq_cyc wq;
520 void __iomem *uar_map;
521 u32 sqn;
522 u16 reserved_room;
523 unsigned long state;
524 struct mlx5e_ktls_resync_resp *ktls_resync;
525
526 /* control path */
527 struct mlx5_wq_ctrl wq_ctrl;
528 struct mlx5e_channel *channel;
529
530 struct work_struct recover_work;
531 } ____cacheline_aligned_in_smp;
532
533 struct mlx5e_wqe_frag_info {
534 struct mlx5e_dma_info *di;
535 u32 offset;
536 bool last_in_page;
537 };
538
539 struct mlx5e_umr_dma_info {
540 struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE];
541 };
542
543 struct mlx5e_mpw_info {
544 struct mlx5e_umr_dma_info umr;
545 u16 consumed_strides;
546 DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
547 };
548
549 #define MLX5E_MAX_RX_FRAGS 4
550
551 /* a single cache unit is capable to serve one napi call (for non-striding rq)
552 * or a MPWQE (for striding rq).
553 */
554 #define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
555 MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
556 #define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
557 struct mlx5e_page_cache {
558 u32 head;
559 u32 tail;
560 struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE];
561 };
562
563 struct mlx5e_rq;
564 typedef void (*mlx5e_fp_handle_rx_cqe)(struct mlx5e_rq*, struct mlx5_cqe64*);
565 typedef struct sk_buff *
566 (*mlx5e_fp_skb_from_cqe_mpwrq)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
567 u16 cqe_bcnt, u32 head_offset, u32 page_idx);
568 typedef struct sk_buff *
569 (*mlx5e_fp_skb_from_cqe)(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
570 struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt);
571 typedef bool (*mlx5e_fp_post_rx_wqes)(struct mlx5e_rq *rq);
572 typedef void (*mlx5e_fp_dealloc_wqe)(struct mlx5e_rq*, u16);
573
574 int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool xsk);
575 void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params);
576
577 enum mlx5e_rq_flag {
578 MLX5E_RQ_FLAG_XDP_XMIT,
579 MLX5E_RQ_FLAG_XDP_REDIRECT,
580 };
581
582 struct mlx5e_rq_frag_info {
583 int frag_size;
584 int frag_stride;
585 };
586
587 struct mlx5e_rq_frags_info {
588 struct mlx5e_rq_frag_info arr[MLX5E_MAX_RX_FRAGS];
589 u8 num_frags;
590 u8 log_num_frags;
591 u8 wqe_bulk;
592 };
593
594 struct mlx5e_rq {
595 /* data path */
596 union {
597 struct {
598 struct mlx5_wq_cyc wq;
599 struct mlx5e_wqe_frag_info *frags;
600 struct mlx5e_dma_info *di;
601 struct mlx5e_rq_frags_info info;
602 mlx5e_fp_skb_from_cqe skb_from_cqe;
603 } wqe;
604 struct {
605 struct mlx5_wq_ll wq;
606 struct mlx5e_umr_wqe umr_wqe;
607 struct mlx5e_mpw_info *info;
608 mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
609 u16 num_strides;
610 u16 actual_wq_head;
611 u8 log_stride_sz;
612 u8 umr_in_progress;
613 u8 umr_last_bulk;
614 u8 umr_completed;
615 } mpwqe;
616 };
617 struct {
618 u16 headroom;
619 u32 frame0_sz;
620 u8 map_dir; /* dma map direction */
621 } buff;
622
623 struct device *pdev;
624 struct net_device *netdev;
625 struct mlx5e_rq_stats *stats;
626 struct mlx5e_cq cq;
627 struct mlx5e_cq_decomp cqd;
628 struct mlx5e_page_cache page_cache;
629 struct hwtstamp_config *tstamp;
630 struct mlx5_clock *clock;
631 struct mlx5e_icosq *icosq;
632 struct mlx5e_priv *priv;
633
634 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
635 mlx5e_fp_post_rx_wqes post_wqes;
636 mlx5e_fp_dealloc_wqe dealloc_wqe;
637
638 unsigned long state;
639 int ix;
640 unsigned int hw_mtu;
641
642 struct dim dim; /* Dynamic Interrupt Moderation */
643
644 /* XDP */
645 struct bpf_prog __rcu *xdp_prog;
646 struct mlx5e_xdpsq *xdpsq;
647 DECLARE_BITMAP(flags, 8);
648 struct page_pool *page_pool;
649
650 /* AF_XDP zero-copy */
651 struct xsk_buff_pool *xsk_pool;
652
653 struct work_struct recover_work;
654
655 /* control */
656 struct mlx5_wq_ctrl wq_ctrl;
657 __be32 mkey_be;
658 u8 wq_type;
659 u32 rqn;
660 struct mlx5_core_dev *mdev;
661 struct mlx5_core_mkey umr_mkey;
662 struct mlx5e_dma_info wqe_overflow;
663
664 /* XDP read-mostly */
665 struct xdp_rxq_info xdp_rxq;
666 cqe_ts_to_ns ptp_cyc2time;
667 } ____cacheline_aligned_in_smp;
668
669 enum mlx5e_channel_state {
670 MLX5E_CHANNEL_STATE_XSK,
671 MLX5E_CHANNEL_NUM_STATES
672 };
673
674 struct mlx5e_channel {
675 /* data path */
676 struct mlx5e_rq rq;
677 struct mlx5e_xdpsq rq_xdpsq;
678 struct mlx5e_txqsq sq[MLX5E_MAX_NUM_TC];
679 struct mlx5e_icosq icosq; /* internal control operations */
680 struct mlx5e_txqsq __rcu * __rcu *qos_sqs;
681 bool xdp;
682 struct napi_struct napi;
683 struct device *pdev;
684 struct net_device *netdev;
685 __be32 mkey_be;
686 u16 qos_sqs_size;
687 u8 num_tc;
688 u8 lag_port;
689
690 /* XDP_REDIRECT */
691 struct mlx5e_xdpsq xdpsq;
692
693 /* AF_XDP zero-copy */
694 struct mlx5e_rq xskrq;
695 struct mlx5e_xdpsq xsksq;
696
697 /* Async ICOSQ */
698 struct mlx5e_icosq async_icosq;
699 /* async_icosq can be accessed from any CPU - the spinlock protects it. */
700 spinlock_t async_icosq_lock;
701
702 /* data path - accessed per napi poll */
703 const struct cpumask *aff_mask;
704 struct mlx5e_ch_stats *stats;
705
706 /* control */
707 struct mlx5e_priv *priv;
708 struct mlx5_core_dev *mdev;
709 struct hwtstamp_config *tstamp;
710 DECLARE_BITMAP(state, MLX5E_CHANNEL_NUM_STATES);
711 int ix;
712 int cpu;
713 };
714
715 struct mlx5e_ptp;
716
717 struct mlx5e_channels {
718 struct mlx5e_channel **c;
719 struct mlx5e_ptp *ptp;
720 unsigned int num;
721 struct mlx5e_params params;
722 };
723
724 struct mlx5e_channel_stats {
725 struct mlx5e_ch_stats ch;
726 struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
727 struct mlx5e_rq_stats rq;
728 struct mlx5e_rq_stats xskrq;
729 struct mlx5e_xdpsq_stats rq_xdpsq;
730 struct mlx5e_xdpsq_stats xdpsq;
731 struct mlx5e_xdpsq_stats xsksq;
732 } ____cacheline_aligned_in_smp;
733
734 struct mlx5e_ptp_stats {
735 struct mlx5e_ch_stats ch;
736 struct mlx5e_sq_stats sq[MLX5E_MAX_NUM_TC];
737 struct mlx5e_ptp_cq_stats cq[MLX5E_MAX_NUM_TC];
738 struct mlx5e_rq_stats rq;
739 } ____cacheline_aligned_in_smp;
740
741 enum {
742 MLX5E_STATE_OPENED,
743 MLX5E_STATE_DESTROYING,
744 MLX5E_STATE_XDP_TX_ENABLED,
745 MLX5E_STATE_XDP_ACTIVE,
746 };
747
748 struct mlx5e_rqt {
749 u32 rqtn;
750 bool enabled;
751 };
752
753 struct mlx5e_tir {
754 u32 tirn;
755 struct mlx5e_rqt rqt;
756 struct list_head list;
757 };
758
759 enum {
760 MLX5E_TC_PRIO = 0,
761 MLX5E_NIC_PRIO
762 };
763
764 struct mlx5e_rss_params {
765 u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE];
766 u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS];
767 u8 toeplitz_hash_key[40];
768 u8 hfunc;
769 };
770
771 struct mlx5e_modify_sq_param {
772 int curr_state;
773 int next_state;
774 int rl_update;
775 int rl_index;
776 bool qos_update;
777 u16 qos_queue_group_id;
778 };
779
780 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
781 struct mlx5e_hv_vhca_stats_agent {
782 struct mlx5_hv_vhca_agent *agent;
783 struct delayed_work work;
784 u16 delay;
785 void *buf;
786 };
787 #endif
788
789 struct mlx5e_xsk {
790 /* XSK buffer pools are stored separately from channels,
791 * because we don't want to lose them when channels are
792 * recreated. The kernel also stores buffer pool, but it doesn't
793 * distinguish between zero-copy and non-zero-copy UMEMs, so
794 * rely on our mechanism.
795 */
796 struct xsk_buff_pool **pools;
797 u16 refcnt;
798 bool ever_used;
799 };
800
801 /* Temporary storage for variables that are allocated when struct mlx5e_priv is
802 * initialized, and used where we can't allocate them because that functions
803 * must not fail. Use with care and make sure the same variable is not used
804 * simultaneously by multiple users.
805 */
806 struct mlx5e_scratchpad {
807 cpumask_var_t cpumask;
808 };
809
810 struct mlx5e_htb {
811 DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES));
812 DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES);
813 struct mlx5e_sq_stats **qos_sq_stats;
814 u16 max_qos_sqs;
815 u16 maj_id;
816 u16 defcls;
817 };
818
819 struct mlx5e_trap;
820
821 struct mlx5e_priv {
822 /* priv data path fields - start */
823 /* +1 for port ptp ts */
824 struct mlx5e_txqsq *txq2sq[(MLX5E_MAX_NUM_CHANNELS + 1) * MLX5E_MAX_NUM_TC +
825 MLX5E_QOS_MAX_LEAF_NODES];
826 int channel_tc2realtxq[MLX5E_MAX_NUM_CHANNELS][MLX5E_MAX_NUM_TC];
827 int port_ptp_tc2realtxq[MLX5E_MAX_NUM_TC];
828 #ifdef CONFIG_MLX5_CORE_EN_DCB
829 struct mlx5e_dcbx_dp dcbx_dp;
830 #endif
831 /* priv data path fields - end */
832
833 u32 msglevel;
834 unsigned long state;
835 struct mutex state_lock; /* Protects Interface state */
836 struct mlx5e_rq drop_rq;
837
838 struct mlx5e_channels channels;
839 u32 tisn[MLX5_MAX_PORTS][MLX5E_MAX_NUM_TC];
840 struct mlx5e_rqt indir_rqt;
841 struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS];
842 struct mlx5e_tir inner_indir_tir[MLX5E_NUM_INDIR_TIRS];
843 struct mlx5e_tir direct_tir[MLX5E_MAX_NUM_CHANNELS];
844 struct mlx5e_tir xsk_tir[MLX5E_MAX_NUM_CHANNELS];
845 struct mlx5e_tir ptp_tir;
846 struct mlx5e_rss_params rss_params;
847 u32 tx_rates[MLX5E_MAX_NUM_SQS];
848
849 struct mlx5e_flow_steering fs;
850
851 struct workqueue_struct *wq;
852 struct work_struct update_carrier_work;
853 struct work_struct set_rx_mode_work;
854 struct work_struct tx_timeout_work;
855 struct work_struct update_stats_work;
856 struct work_struct monitor_counters_work;
857 struct mlx5_nb monitor_counters_nb;
858
859 struct mlx5_core_dev *mdev;
860 struct net_device *netdev;
861 struct mlx5e_trap *en_trap;
862 struct mlx5e_stats stats;
863 struct mlx5e_channel_stats channel_stats[MLX5E_MAX_NUM_CHANNELS];
864 struct mlx5e_channel_stats trap_stats;
865 struct mlx5e_ptp_stats ptp_stats;
866 u16 max_nch;
867 u8 max_opened_tc;
868 bool tx_ptp_opened;
869 bool rx_ptp_opened;
870 struct hwtstamp_config tstamp;
871 u16 q_counter;
872 u16 drop_rq_q_counter;
873 struct notifier_block events_nb;
874 struct notifier_block blocking_events_nb;
875 int num_tc_x_num_ch;
876
877 struct udp_tunnel_nic_info nic_info;
878 #ifdef CONFIG_MLX5_CORE_EN_DCB
879 struct mlx5e_dcbx dcbx;
880 #endif
881
882 const struct mlx5e_profile *profile;
883 void *ppriv;
884 #ifdef CONFIG_MLX5_EN_IPSEC
885 struct mlx5e_ipsec *ipsec;
886 #endif
887 #ifdef CONFIG_MLX5_EN_TLS
888 struct mlx5e_tls *tls;
889 #endif
890 struct devlink_health_reporter *tx_reporter;
891 struct devlink_health_reporter *rx_reporter;
892 struct mlx5e_xsk xsk;
893 #if IS_ENABLED(CONFIG_PCI_HYPERV_INTERFACE)
894 struct mlx5e_hv_vhca_stats_agent stats_agent;
895 #endif
896 struct mlx5e_scratchpad scratchpad;
897 struct mlx5e_htb htb;
898 };
899
900 struct mlx5e_rx_handlers {
901 mlx5e_fp_handle_rx_cqe handle_rx_cqe;
902 mlx5e_fp_handle_rx_cqe handle_rx_cqe_mpwqe;
903 };
904
905 extern const struct mlx5e_rx_handlers mlx5e_rx_handlers_nic;
906
907 struct mlx5e_profile {
908 int (*init)(struct mlx5_core_dev *mdev,
909 struct net_device *netdev);
910 void (*cleanup)(struct mlx5e_priv *priv);
911 int (*init_rx)(struct mlx5e_priv *priv);
912 void (*cleanup_rx)(struct mlx5e_priv *priv);
913 int (*init_tx)(struct mlx5e_priv *priv);
914 void (*cleanup_tx)(struct mlx5e_priv *priv);
915 void (*enable)(struct mlx5e_priv *priv);
916 void (*disable)(struct mlx5e_priv *priv);
917 int (*update_rx)(struct mlx5e_priv *priv);
918 void (*update_stats)(struct mlx5e_priv *priv);
919 void (*update_carrier)(struct mlx5e_priv *priv);
920 unsigned int (*stats_grps_num)(struct mlx5e_priv *priv);
921 mlx5e_stats_grp_t *stats_grps;
922 const struct mlx5e_rx_handlers *rx_handlers;
923 int max_tc;
924 u8 rq_groups;
925 bool rx_ptp_support;
926 };
927
928 void mlx5e_build_ptys2ethtool_map(void);
929
930 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
931
932 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
933 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s);
934
935 void mlx5e_init_l2_addr(struct mlx5e_priv *priv);
936 int mlx5e_self_test_num(struct mlx5e_priv *priv);
937 void mlx5e_self_test(struct net_device *ndev, struct ethtool_test *etest,
938 u64 *buf);
939 void mlx5e_set_rx_mode_work(struct work_struct *work);
940
941 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr);
942 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr);
943 int mlx5e_modify_rx_cqe_compression_locked(struct mlx5e_priv *priv, bool val);
944
945 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __always_unused __be16 proto,
946 u16 vid);
947 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __always_unused __be16 proto,
948 u16 vid);
949 void mlx5e_timestamp_init(struct mlx5e_priv *priv);
950
951 struct mlx5e_redirect_rqt_param {
952 bool is_rss;
953 union {
954 u32 rqn; /* Direct RQN (Non-RSS) */
955 struct {
956 u8 hfunc;
957 struct mlx5e_channels *channels;
958 } rss; /* RSS data */
959 };
960 };
961
962 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
963 struct mlx5e_redirect_rqt_param rrp);
964 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params,
965 const struct mlx5e_tirc_config *ttconfig,
966 void *tirc, bool inner);
967 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in);
968 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt);
969
970 struct mlx5e_xsk_param;
971
972 struct mlx5e_rq_param;
973 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
974 struct mlx5e_xsk_param *xsk, int node,
975 struct mlx5e_rq *rq);
976 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
977 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
978 void mlx5e_close_rq(struct mlx5e_rq *rq);
979 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param);
980 void mlx5e_destroy_rq(struct mlx5e_rq *rq);
981
982 struct mlx5e_sq_param;
983 int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
984 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq);
985 void mlx5e_close_icosq(struct mlx5e_icosq *sq);
986 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
987 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
988 struct mlx5e_xdpsq *sq, bool is_redirect);
989 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq);
990
991 struct mlx5e_create_cq_param {
992 struct napi_struct *napi;
993 struct mlx5e_ch_stats *ch_stats;
994 int node;
995 int ix;
996 };
997
998 struct mlx5e_cq_param;
999 int mlx5e_open_cq(struct mlx5e_priv *priv, struct dim_cq_moder moder,
1000 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
1001 struct mlx5e_cq *cq);
1002 void mlx5e_close_cq(struct mlx5e_cq *cq);
1003
1004 int mlx5e_open_locked(struct net_device *netdev);
1005 int mlx5e_close_locked(struct net_device *netdev);
1006
1007 int mlx5e_open_channels(struct mlx5e_priv *priv,
1008 struct mlx5e_channels *chs);
1009 void mlx5e_close_channels(struct mlx5e_channels *chs);
1010
1011 /* Function pointer to be used to modify HW or kernel settings while
1012 * switching channels
1013 */
1014 typedef int (*mlx5e_fp_preactivate)(struct mlx5e_priv *priv, void *context);
1015 #define MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(fn) \
1016 int fn##_ctx(struct mlx5e_priv *priv, void *context) \
1017 { \
1018 return fn(priv); \
1019 }
1020 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv);
1021 int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
1022 struct mlx5e_params *new_params,
1023 mlx5e_fp_preactivate preactivate,
1024 void *context, bool reset);
1025 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv);
1026 int mlx5e_num_channels_changed(struct mlx5e_priv *priv);
1027 int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context);
1028 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
1029 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
1030 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx);
1031
1032 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
1033 int num_channels);
1034
1035 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state);
1036 void mlx5e_activate_rq(struct mlx5e_rq *rq);
1037 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
1038 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
1039 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq);
1040
1041 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1042 struct mlx5e_modify_sq_param *p);
1043 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1044 struct mlx5e_params *params, struct mlx5e_sq_param *param,
1045 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, u16 qos_qid);
1046 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq);
1047 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq);
1048 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq);
1049 void mlx5e_tx_disable_queue(struct netdev_queue *txq);
1050 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa);
1051 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq);
1052 struct mlx5e_create_sq_param;
1053 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1054 struct mlx5e_sq_param *param,
1055 struct mlx5e_create_sq_param *csp,
1056 u16 qos_queue_group_id,
1057 u32 *sqn);
1058 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work);
1059 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq);
1060
mlx5_tx_swp_supported(struct mlx5_core_dev * mdev)1061 static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)
1062 {
1063 return MLX5_CAP_ETH(mdev, swp) &&
1064 MLX5_CAP_ETH(mdev, swp_csum) && MLX5_CAP_ETH(mdev, swp_lso);
1065 }
1066
1067 extern const struct ethtool_ops mlx5e_ethtool_ops;
1068
1069 int mlx5e_create_tir(struct mlx5_core_dev *mdev, struct mlx5e_tir *tir,
1070 u32 *in);
1071 void mlx5e_destroy_tir(struct mlx5_core_dev *mdev,
1072 struct mlx5e_tir *tir);
1073 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev);
1074 void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
1075 int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb,
1076 bool enable_mc_lb);
1077 void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc);
1078
1079 /* common netdev helpers */
1080 void mlx5e_create_q_counters(struct mlx5e_priv *priv);
1081 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv);
1082 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
1083 struct mlx5e_rq *drop_rq);
1084 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
1085 int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node);
1086 void mlx5e_free_di_list(struct mlx5e_rq *rq);
1087
1088 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv);
1089
1090 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc);
1091 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv);
1092
1093 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n);
1094 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n);
1095 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n);
1096 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs, int n);
1097 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt);
1098
1099 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn);
1100 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
1101
1102 int mlx5e_create_tises(struct mlx5e_priv *priv);
1103 void mlx5e_destroy_tises(struct mlx5e_priv *priv);
1104 int mlx5e_update_nic_rx(struct mlx5e_priv *priv);
1105 void mlx5e_update_carrier(struct mlx5e_priv *priv);
1106 int mlx5e_close(struct net_device *netdev);
1107 int mlx5e_open(struct net_device *netdev);
1108
1109 void mlx5e_queue_update_stats(struct mlx5e_priv *priv);
1110 int mlx5e_bits_invert(unsigned long a, int size);
1111
1112 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv);
1113 int mlx5e_set_dev_port_mtu_ctx(struct mlx5e_priv *priv, void *context);
1114 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
1115 mlx5e_fp_preactivate preactivate);
1116 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv);
1117
1118 /* ethtool helpers */
1119 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
1120 struct ethtool_drvinfo *drvinfo);
1121 void mlx5e_ethtool_get_strings(struct mlx5e_priv *priv,
1122 uint32_t stringset, uint8_t *data);
1123 int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset);
1124 void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv,
1125 struct ethtool_stats *stats, u64 *data);
1126 void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
1127 struct ethtool_ringparam *param);
1128 int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv,
1129 struct ethtool_ringparam *param);
1130 void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv,
1131 struct ethtool_channels *ch);
1132 int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
1133 struct ethtool_channels *ch);
1134 int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv,
1135 struct ethtool_coalesce *coal);
1136 int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv,
1137 struct ethtool_coalesce *coal);
1138 int mlx5e_ethtool_get_link_ksettings(struct mlx5e_priv *priv,
1139 struct ethtool_link_ksettings *link_ksettings);
1140 int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv,
1141 const struct ethtool_link_ksettings *link_ksettings);
1142 int mlx5e_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, u8 *hfunc);
1143 int mlx5e_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key,
1144 const u8 hfunc);
1145 int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1146 u32 *rule_locs);
1147 int mlx5e_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *cmd);
1148 u32 mlx5e_ethtool_get_rxfh_key_size(struct mlx5e_priv *priv);
1149 u32 mlx5e_ethtool_get_rxfh_indir_size(struct mlx5e_priv *priv);
1150 int mlx5e_ethtool_get_ts_info(struct mlx5e_priv *priv,
1151 struct ethtool_ts_info *info);
1152 int mlx5e_ethtool_flash_device(struct mlx5e_priv *priv,
1153 struct ethtool_flash *flash);
1154 void mlx5e_ethtool_get_pauseparam(struct mlx5e_priv *priv,
1155 struct ethtool_pauseparam *pauseparam);
1156 int mlx5e_ethtool_set_pauseparam(struct mlx5e_priv *priv,
1157 struct ethtool_pauseparam *pauseparam);
1158
1159 /* mlx5e generic netdev management API */
1160 static inline unsigned int
mlx5e_calc_max_nch(struct mlx5e_priv * priv,const struct mlx5e_profile * profile)1161 mlx5e_calc_max_nch(struct mlx5e_priv *priv, const struct mlx5e_profile *profile)
1162 {
1163 return priv->netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1);
1164 }
1165
1166 int mlx5e_priv_init(struct mlx5e_priv *priv,
1167 struct net_device *netdev,
1168 struct mlx5_core_dev *mdev);
1169 void mlx5e_priv_cleanup(struct mlx5e_priv *priv);
1170 struct net_device *
1171 mlx5e_create_netdev(struct mlx5_core_dev *mdev, unsigned int txqs, unsigned int rxqs);
1172 int mlx5e_attach_netdev(struct mlx5e_priv *priv);
1173 void mlx5e_detach_netdev(struct mlx5e_priv *priv);
1174 void mlx5e_destroy_netdev(struct mlx5e_priv *priv);
1175 int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
1176 const struct mlx5e_profile *new_profile, void *new_ppriv);
1177 void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv);
1178 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv);
1179 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu);
1180 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params,
1181 u16 num_channels);
1182 void mlx5e_rx_dim_work(struct work_struct *work);
1183 void mlx5e_tx_dim_work(struct work_struct *work);
1184
1185 netdev_features_t mlx5e_features_check(struct sk_buff *skb,
1186 struct net_device *netdev,
1187 netdev_features_t features);
1188 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features);
1189 #ifdef CONFIG_MLX5_ESWITCH
1190 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac);
1191 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, int max_tx_rate);
1192 int mlx5e_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivi);
1193 int mlx5e_get_vf_stats(struct net_device *dev, int vf, struct ifla_vf_stats *vf_stats);
1194 #endif
1195 #endif /* __MLX5_EN_H__ */
1196