1 /****************************************************************************** 2 3 Copyright (c) 2013-2018, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #ifndef _IXL_H_ 36 #define _IXL_H_ 37 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_rss.h" 41 #include "opt_ixl.h" 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/buf_ring.h> 46 #include <sys/mbuf.h> 47 #include <sys/protosw.h> 48 #include <sys/socket.h> 49 #include <sys/malloc.h> 50 #include <sys/kernel.h> 51 #include <sys/module.h> 52 #include <sys/sockio.h> 53 #include <sys/eventhandler.h> 54 #include <sys/syslog.h> 55 #include <sys/priv.h> 56 #include <sys/bitstring.h> 57 58 #include <net/if.h> 59 #include <net/if_var.h> 60 #include <net/if_arp.h> 61 #include <net/bpf.h> 62 #include <net/ethernet.h> 63 #include <net/if_dl.h> 64 #include <net/if_media.h> 65 #include <net/iflib.h> 66 67 #include <net/bpf.h> 68 #include <net/if_types.h> 69 #include <net/if_vlan_var.h> 70 71 #include <netinet/in_systm.h> 72 #include <netinet/in.h> 73 #include <netinet/if_ether.h> 74 #include <netinet/ip.h> 75 #include <netinet/ip6.h> 76 #include <netinet/tcp.h> 77 #include <netinet/tcp_lro.h> 78 #include <netinet/udp.h> 79 #include <netinet/sctp.h> 80 81 #include <machine/in_cksum.h> 82 83 #include <sys/bus.h> 84 #include <machine/bus.h> 85 #include <sys/rman.h> 86 #include <machine/resource.h> 87 #include <vm/vm.h> 88 #include <vm/pmap.h> 89 #include <machine/clock.h> 90 #include <dev/pci/pcivar.h> 91 #include <dev/pci/pcireg.h> 92 #include <sys/proc.h> 93 #include <sys/sysctl.h> 94 #include <sys/endian.h> 95 #include <sys/taskqueue.h> 96 #include <sys/pcpu.h> 97 #include <sys/smp.h> 98 #include <sys/sbuf.h> 99 #include <machine/smp.h> 100 #include <machine/stdarg.h> 101 102 #ifdef RSS 103 #include <net/rss_config.h> 104 #include <netinet/in_rss.h> 105 #endif 106 107 #include "ifdi_if.h" 108 #include "i40e_type.h" 109 #include "i40e_prototype.h" 110 #include "ixl_debug.h" 111 112 #define PVIDV(vendor, devid, name) \ 113 PVID(vendor, devid, name " - " IXL_DRIVER_VERSION_STRING) 114 115 /* Tunables */ 116 117 /* 118 * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the 119 * number of tx/rx descriptors allocated by the driver. Increasing this 120 * value allows the driver to queue more operations. 121 * 122 * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes. 123 * The driver currently always uses 32 byte Rx descriptors. 124 */ 125 #define IXL_DEFAULT_RING 1024 126 #define IXL_MAX_RING 4096 127 #define IXL_MIN_RING 64 128 #define IXL_RING_INCREMENT 32 129 130 #define IXL_AQ_LEN 256 131 #define IXL_AQ_LEN_MAX 1024 132 133 /* Alignment for rings */ 134 #define DBA_ALIGN 128 135 136 #define MAX_MULTICAST_ADDR 128 137 138 #define IXL_MSIX_BAR 3 139 #define IXL_ADM_LIMIT 2 140 #define IXL_TSO_SIZE ((255*1024)-1) 141 #define IXL_TX_BUF_SZ ((u32) 1514) 142 #define IXL_AQ_BUF_SZ ((u32) 4096) 143 #define IXL_RX_ITR 0 144 #define IXL_TX_ITR 1 145 #define IXL_ITR_NONE 3 146 #define IXL_QUEUE_EOL 0x7FF 147 #define IXL_MIN_FRAME 17 148 #define IXL_MAX_FRAME 9728 149 #define IXL_MAX_TX_SEGS 8 150 #define IXL_MAX_RX_SEGS 5 151 #define IXL_MAX_TSO_SEGS 128 152 #define IXL_SPARSE_CHAIN 7 153 #define IXL_MIN_TSO_MSS 64 154 #define IXL_MAX_TSO_MSS 9668 155 #define IXL_MAX_DMA_SEG_SIZE ((16 * 1024) - 1) 156 157 #define IXL_RSS_KEY_SIZE_REG 13 158 #define IXL_RSS_KEY_SIZE (IXL_RSS_KEY_SIZE_REG * 4) 159 #define IXL_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */ 160 #define IXL_RSS_VSI_LUT_ENTRY_MASK 0x3F 161 #define IXL_RSS_VF_LUT_ENTRY_MASK 0xF 162 163 #define IXL_VF_MAX_BUFFER 0x3F80 164 #define IXL_VF_MAX_HDR_BUFFER 0x840 165 #define IXL_VF_MAX_FRAME 0x3FFF 166 167 /* ERJ: hardware can support ~2k (SW5+) filters between all functions */ 168 #define IXL_MAX_FILTERS 256 169 170 #define IXL_NVM_VERSION_LO_SHIFT 0 171 #define IXL_NVM_VERSION_LO_MASK (0xff << IXL_NVM_VERSION_LO_SHIFT) 172 #define IXL_NVM_VERSION_HI_SHIFT 12 173 #define IXL_NVM_VERSION_HI_MASK (0xf << IXL_NVM_VERSION_HI_SHIFT) 174 175 /* 176 * Interrupt Moderation parameters 177 * Multiply ITR values by 2 for real ITR value 178 */ 179 #define IXL_MAX_ITR 0x0FF0 180 #define IXL_ITR_100K 0x0005 181 #define IXL_ITR_20K 0x0019 182 #define IXL_ITR_8K 0x003E 183 #define IXL_ITR_4K 0x007A 184 #define IXL_ITR_1K 0x01F4 185 #define IXL_ITR_DYNAMIC 0x8000 186 #define IXL_LOW_LATENCY 0 187 #define IXL_AVE_LATENCY 1 188 #define IXL_BULK_LATENCY 2 189 190 /* MacVlan Flags */ 191 #define IXL_FILTER_VLAN (u16)(1 << 0) 192 #define IXL_FILTER_MC (u16)(1 << 1) 193 194 /* used in the vlan field of the filter when not a vlan */ 195 #define IXL_VLAN_ANY -1 196 197 /* Maximum number of MAC/VLAN filters supported by HW */ 198 #define IXL_MAX_VLAN_FILTERS 256 199 200 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) 201 #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) 202 #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) 203 204 /* Misc flags for ixl_vsi.flags */ 205 #define IXL_FLAGS_KEEP_TSO4 (1 << 0) 206 #define IXL_FLAGS_KEEP_TSO6 (1 << 1) 207 #define IXL_FLAGS_USES_MSIX (1 << 2) 208 #define IXL_FLAGS_IS_VF (1 << 3) 209 210 #define IXL_VSI_IS_PF(v) ((v->flags & IXL_FLAGS_IS_VF) == 0) 211 #define IXL_VSI_IS_VF(v) ((v->flags & IXL_FLAGS_IS_VF) != 0) 212 213 #define IXL_VF_RESET_TIMEOUT 100 214 215 #define IXL_VSI_DATA_PORT 0x01 216 217 #define IAVF_MAX_QUEUES 16 218 #define IXL_MAX_VSI_QUEUES (2 * (I40E_VSILAN_QTABLE_MAX_INDEX + 1)) 219 220 #define IXL_RX_CTX_BASE_UNITS 128 221 #define IXL_TX_CTX_BASE_UNITS 128 222 223 #define IXL_PF_PCI_CIAA_VF_DEVICE_STATUS 0xAA 224 225 #define IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK 0x20 226 227 #define IXL_GLGEN_VFLRSTAT_INDEX(glb_vf) ((glb_vf) / 32) 228 #define IXL_GLGEN_VFLRSTAT_MASK(glb_vf) (1 << ((glb_vf) % 32)) 229 230 #define IXL_MAX_ITR_IDX 3 231 232 #define IXL_END_OF_INTR_LNKLST 0x7FF 233 234 #define IXL_DEFAULT_RSS_HENA_BASE (\ 235 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_UDP) | \ 236 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP) | \ 237 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ 238 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ 239 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4) | \ 240 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_UDP) | \ 241 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP) | \ 242 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ 243 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ 244 BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6) | \ 245 BIT_ULL(I40E_FILTER_PCTYPE_L2_PAYLOAD)) 246 247 #define IXL_DEFAULT_RSS_HENA_XL710 IXL_DEFAULT_RSS_HENA_BASE 248 249 #define IXL_DEFAULT_RSS_HENA_X722 (\ 250 IXL_DEFAULT_RSS_HENA_BASE | \ 251 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ 252 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ 253 BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ 254 BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ 255 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ 256 BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) 257 258 #define IXL_CAPS \ 259 (IFCAP_TSO4 | IFCAP_TSO6 | \ 260 IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | \ 261 IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | \ 262 IFCAP_VLAN_HWFILTER | IFCAP_VLAN_HWTSO | \ 263 IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM | \ 264 IFCAP_VLAN_MTU | IFCAP_JUMBO_MTU | IFCAP_LRO) 265 266 #define IXL_CSUM_TCP \ 267 (CSUM_IP_TCP|CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP6_TCP) 268 #define IXL_CSUM_UDP \ 269 (CSUM_IP_UDP|CSUM_IP6_UDP) 270 #define IXL_CSUM_SCTP \ 271 (CSUM_IP_SCTP|CSUM_IP6_SCTP) 272 #define IXL_CSUM_IPV4 \ 273 (CSUM_IP|CSUM_IP_TSO) 274 275 /* Pre-11 counter(9) compatibility */ 276 #define IXL_SET_IPACKETS(vsi, count) (vsi)->ipackets = (count) 277 #define IXL_SET_IERRORS(vsi, count) (vsi)->ierrors = (count) 278 #define IXL_SET_OPACKETS(vsi, count) (vsi)->opackets = (count) 279 #define IXL_SET_OERRORS(vsi, count) (vsi)->oerrors = (count) 280 #define IXL_SET_COLLISIONS(vsi, count) /* Do nothing; collisions is always 0. */ 281 #define IXL_SET_IBYTES(vsi, count) (vsi)->ibytes = (count) 282 #define IXL_SET_OBYTES(vsi, count) (vsi)->obytes = (count) 283 #define IXL_SET_IMCASTS(vsi, count) (vsi)->imcasts = (count) 284 #define IXL_SET_OMCASTS(vsi, count) (vsi)->omcasts = (count) 285 #define IXL_SET_IQDROPS(vsi, count) (vsi)->iqdrops = (count) 286 #define IXL_SET_OQDROPS(vsi, count) (vsi)->oqdrops = (count) 287 #define IXL_SET_NOPROTO(vsi, count) (vsi)->noproto = (count) 288 289 /* For stats sysctl naming */ 290 #define IXL_QUEUE_NAME_LEN 32 291 292 MALLOC_DECLARE(M_IXL); 293 294 #define IXL_DEV_ERR(_dev, _format, ...) \ 295 device_printf(_dev, "%s: " _format " (%s:%d)\n", __func__, ##__VA_ARGS__, __FILE__, __LINE__) 296 297 /* 298 ***************************************************************************** 299 * vendor_info_array 300 * 301 * This array contains the list of Subvendor/Subdevice IDs on which the driver 302 * should load. 303 * 304 ***************************************************************************** 305 */ 306 typedef struct _ixl_vendor_info_t { 307 unsigned int vendor_id; 308 unsigned int device_id; 309 unsigned int subvendor_id; 310 unsigned int subdevice_id; 311 unsigned int index; 312 } ixl_vendor_info_t; 313 314 /* 315 ** This struct has multiple uses, multicast 316 ** addresses, vlans, and mac filters all use it. 317 */ 318 struct ixl_mac_filter { 319 LIST_ENTRY(ixl_mac_filter) ftle; 320 u8 macaddr[ETHER_ADDR_LEN]; 321 s16 vlan; 322 u16 flags; 323 }; 324 325 /* 326 * The Transmit ring control struct 327 */ 328 struct tx_ring { 329 struct ixl_tx_queue *que; 330 u32 tail; 331 struct i40e_tx_desc *tx_base; 332 u64 tx_paddr; 333 u32 latency; 334 u32 packets; 335 u32 me; 336 /* 337 * For reporting completed packet status 338 * in descriptor writeback mode 339 */ 340 qidx_t *tx_rsq; 341 qidx_t tx_rs_cidx; 342 qidx_t tx_rs_pidx; 343 qidx_t tx_cidx_processed; 344 345 /* Used for Dynamic ITR calculation */ 346 u32 itr; 347 u32 bytes; 348 349 /* Soft Stats */ 350 u64 tx_bytes; 351 u64 tx_packets; 352 u64 mss_too_small; 353 }; 354 355 356 /* 357 * The Receive ring control struct 358 */ 359 struct rx_ring { 360 struct ixl_rx_queue *que; 361 union i40e_rx_desc *rx_base; 362 uint64_t rx_paddr; 363 bool discard; 364 u32 itr; 365 u32 latency; 366 u32 mbuf_sz; 367 u32 tail; 368 u32 me; 369 370 /* Used for Dynamic ITR calculation */ 371 u32 packets; 372 u32 bytes; 373 374 /* Soft stats */ 375 u64 rx_packets; 376 u64 rx_bytes; 377 u64 desc_errs; 378 u64 csum_errs; 379 }; 380 381 /* 382 ** Driver queue structs 383 */ 384 struct ixl_tx_queue { 385 struct ixl_vsi *vsi; 386 struct tx_ring txr; 387 struct if_irq que_irq; 388 u32 msix; 389 /* Stats */ 390 u64 irqs; 391 u64 tso; 392 }; 393 394 struct ixl_rx_queue { 395 struct ixl_vsi *vsi; 396 struct rx_ring rxr; 397 struct if_irq que_irq; 398 u32 msix; /* This queue's MSIX vector */ 399 /* Stats */ 400 u64 irqs; 401 }; 402 403 /* 404 ** Virtual Station Interface 405 */ 406 LIST_HEAD(ixl_ftl_head, ixl_mac_filter); 407 struct ixl_vsi { 408 if_ctx_t ctx; 409 if_softc_ctx_t shared; 410 if_t ifp; 411 device_t dev; 412 struct i40e_hw *hw; 413 struct ifmedia *media; 414 415 int num_rx_queues; 416 int num_tx_queues; 417 418 void *back; 419 enum i40e_vsi_type type; 420 int id; 421 u32 rx_itr_setting; 422 u32 tx_itr_setting; 423 bool enable_head_writeback; 424 425 u16 vsi_num; 426 bool link_active; 427 u16 seid; 428 u16 uplink_seid; 429 u16 downlink_seid; 430 431 struct ixl_tx_queue *tx_queues; /* TX queue array */ 432 struct ixl_rx_queue *rx_queues; /* RX queue array */ 433 struct if_irq irq; 434 u32 link_speed; 435 436 /* MAC/VLAN Filter list */ 437 struct ixl_ftl_head ftl; 438 u16 num_macs; 439 u64 num_hw_filters; 440 441 /* Contains readylist & stat counter id */ 442 struct i40e_aqc_vsi_properties_data info; 443 444 #define IXL_VLANS_MAP_LEN EVL_VLID_MASK + 1 445 bitstr_t bit_decl(vlans_map, IXL_VLANS_MAP_LEN); 446 u16 num_vlans; 447 448 /* Per-VSI stats from hardware */ 449 struct i40e_eth_stats eth_stats; 450 struct i40e_eth_stats eth_stats_offsets; 451 bool stat_offsets_loaded; 452 /* VSI stat counters */ 453 u64 ipackets; 454 u64 ierrors; 455 u64 opackets; 456 u64 oerrors; 457 u64 ibytes; 458 u64 obytes; 459 u64 imcasts; 460 u64 omcasts; 461 u64 iqdrops; 462 u64 oqdrops; 463 u64 noproto; 464 465 /* Misc. */ 466 u64 flags; 467 /* Stats sysctls for this VSI */ 468 struct sysctl_oid *vsi_node; 469 struct sysctl_ctx_list sysctl_ctx; 470 }; 471 472 struct ixl_add_maddr_arg { 473 struct ixl_ftl_head to_add; 474 struct ixl_vsi *vsi; 475 }; 476 477 /* 478 ** Compare two ethernet addresses 479 */ 480 static inline bool 481 ixl_ether_is_equal(const u8 *ea1, const u8 *ea2) 482 { 483 return (bcmp(ea1, ea2, ETHER_ADDR_LEN) == 0); 484 } 485 486 /* 487 * Return next largest power of 2, unsigned 488 * 489 * Public domain, from Bit Twiddling Hacks 490 */ 491 static inline u32 492 next_power_of_two(u32 n) 493 { 494 n--; 495 n |= n >> 1; 496 n |= n >> 2; 497 n |= n >> 4; 498 n |= n >> 8; 499 n |= n >> 16; 500 n++; 501 502 /* Next power of two > 0 is 1 */ 503 n += (n == 0); 504 505 return (n); 506 } 507 508 /* 509 * Info for stats sysctls 510 */ 511 struct ixl_sysctl_info { 512 u64 *stat; 513 char *name; 514 char *description; 515 }; 516 517 extern const uint8_t ixl_bcast_addr[ETHER_ADDR_LEN]; 518 519 /* Common function prototypes between PF/VF driver */ 520 void ixl_debug_core(device_t dev, u32 enabled_mask, u32 mask, char *fmt, ...); 521 void ixl_init_tx_ring(struct ixl_vsi *vsi, struct ixl_tx_queue *que); 522 void ixl_get_default_rss_key(u32 *); 523 const char * i40e_vc_stat_str(struct i40e_hw *hw, 524 enum virtchnl_status_code stat_err); 525 void ixl_init_tx_rsqs(struct ixl_vsi *vsi); 526 void ixl_init_tx_cidx(struct ixl_vsi *vsi); 527 u64 ixl_max_vc_speed_to_value(u8 link_speeds); 528 void ixl_add_vsi_sysctls(device_t dev, struct ixl_vsi *vsi, 529 struct sysctl_ctx_list *ctx, const char *sysctl_name); 530 void ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, 531 struct sysctl_oid_list *child, 532 struct i40e_eth_stats *eth_stats); 533 void ixl_vsi_add_queues_stats(struct ixl_vsi *vsi, 534 struct sysctl_ctx_list *ctx); 535 #endif /* _IXL_H_ */ 536