1# Chelsio T5 Factory Default configuration file. 2# 3# Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 4# 5# DO NOT MODIFY THIS FILE UNDER ANY CIRCUMSTANCES. MODIFICATION OF 6# THIS FILE WILL RESULT IN A NON-FUNCTIONAL T4 ADAPTER AND MAY RESULT 7# IN PHYSICAL DAMAGE TO T4 ADAPTERS. 8 9# This file provides the default, power-on configuration for 4-port T4-based 10# adapters shipped from the factory. These defaults are designed to address 11# the needs of the vast majority of T4 customers. The basic idea is to have 12# a default configuration which allows a customer to plug a T4 adapter in and 13# have it work regardless of OS, driver or application except in the most 14# unusual and/or demanding customer applications. 15# 16# Many of the T4 resources which are described by this configuration are 17# finite. This requires balancing the configuration/operation needs of 18# device drivers across OSes and a large number of customer application. 19# 20# Some of the more important resources to allocate and their constaints are: 21# 1. Virtual Interfaces: 128. 22# 2. Ingress Queues with Free Lists: 1024. PCI-E SR-IOV Virtual Functions 23# must use a power of 2 Ingress Queues. 24# 3. Egress Queues: 128K. PCI-E SR-IOV Virtual Functions must use a 25# power of 2 Egress Queues. 26# 4. MSI-X Vectors: 1088. A complication here is that the PCI-E SR-IOV 27# Virtual Functions based off of a Physical Function all get the 28# same umber of MSI-X Vectors as the base Physical Function. 29# Additionally, regardless of whether Virtual Functions are enabled or 30# not, their MSI-X "needs" are counted by the PCI-E implementation. 31# And finally, all Physical Funcations capable of supporting Virtual 32# Functions (PF0-3) must have the same number of configured TotalVFs in 33# their SR-IOV Capabilities. 34# 5. Multi-Port Support (MPS) TCAM: 336 entries to support MAC destination 35# address matching on Ingress Packets. 36# 37# Some of the important OS/Driver resource needs are: 38# 6. Some OS Drivers will manage all resources through a single Physical 39# Function (currently PF0 but it could be any Physical Function). Thus, 40# this "Unified PF" will need to have enough resources allocated to it 41# to allow for this. And because of the MSI-X resource allocation 42# constraints mentioned above, this probably means we'll either have to 43# severely limit the TotalVFs if we continue to use PF0 as the Unified PF 44# or we'll need to move the Unified PF into the PF4-7 range since those 45# Physical Functions don't have any Virtual Functions associated with 46# them. 47# 7. Some OS Drivers will manage different ports and functions (NIC, 48# storage, etc.) on different Physical Functions. For example, NIC 49# functions for ports 0-3 on PF0-3, FCoE on PF4, iSCSI on PF5, etc. 50# 51# Some of the customer application needs which need to be accommodated: 52# 8. Some customers will want to support large CPU count systems with 53# good scaling. Thus, we'll need to accommodate a number of 54# Ingress Queues and MSI-X Vectors to allow up to some number of CPUs 55# to be involved per port and per application function. For example, 56# in the case where all ports and application functions will be 57# managed via a single Unified PF and we want to accommodate scaling up 58# to 8 CPUs, we would want: 59# 60# 4 ports * 61# 3 application functions (NIC, FCoE, iSCSI) per port * 62# 8 Ingress Queue/MSI-X Vectors per application function 63# 64# for a total of 96 Ingress Queues and MSI-X Vectors on the Unified PF. 65# (Plus a few for Firmware Event Queues, etc.) 66# 67# 9. Some customers will want to use T4's PCI-E SR-IOV Capability to allow 68# Virtual Machines to directly access T4 functionality via SR-IOV 69# Virtual Functions and "PCI Device Passthrough" -- this is especially 70# true for the NIC application functionality. (Note that there is 71# currently no ability to use the TOE, FCoE, iSCSI, etc. via Virtual 72# Functions so this is in fact solely limited to NIC.) 73# 74 75 76# Global configuration settings. 77# 78[global] 79 rss_glb_config_mode = basicvirtual 80 rss_glb_config_options = tnlmapen,hashtoeplitz,tnlalllkp 81 82 # PCIE_MA_RSP register 83 pcie_ma_rsp_timervalue = 500 # the timer value in units of us 84 reg[0x59c4] = 0x3/0x3 # enable the timers 85 86 # PL_TIMEOUT register 87 pl_timeout_value = 200 # the timeout value in units of us 88 89 # The following Scatter Gather Engine (SGE) settings assume a 4KB Host 90 # Page Size and a 64B L1 Cache Line Size. It programs the 91 # EgrStatusPageSize and IngPadBoundary to 64B and the PktShift to 2. 92 # If a Master PF Driver finds itself on a machine with different 93 # parameters, then the Master PF Driver is responsible for initializing 94 # these parameters to appropriate values. 95 # 96 # Notes: 97 # 1. The Free List Buffer Sizes below are raw and the firmware will 98 # round them up to the Ingress Padding Boundary. 99 # 2. The SGE Timer Values below are expressed below in microseconds. 100 # The firmware will convert these values to Core Clock Ticks when 101 # it processes the configuration parameters. 102 # 103 reg[0x1008] = 0x40810/0x21c70 # SGE_CONTROL 104 reg[0x100c] = 0x22222222 # SGE_HOST_PAGE_SIZE 105 reg[0x10a0] = 0x01040810 # SGE_INGRESS_RX_THRESHOLD 106 reg[0x1044] = 4096 # SGE_FL_BUFFER_SIZE0 107 reg[0x1048] = 65536 # SGE_FL_BUFFER_SIZE1 108 reg[0x104c] = 1536 # SGE_FL_BUFFER_SIZE2 109 reg[0x1050] = 9024 # SGE_FL_BUFFER_SIZE3 110 reg[0x1054] = 9216 # SGE_FL_BUFFER_SIZE4 111 reg[0x1058] = 2048 # SGE_FL_BUFFER_SIZE5 112 reg[0x105c] = 128 # SGE_FL_BUFFER_SIZE6 113 reg[0x1060] = 8192 # SGE_FL_BUFFER_SIZE7 114 reg[0x1064] = 16384 # SGE_FL_BUFFER_SIZE8 115 reg[0x10a4] = 0xa000a000/0xf000f000 # SGE_DBFIFO_STATUS 116 reg[0x10a8] = 0x402000/0x402000 # SGE_DOORBELL_CONTROL 117 118 # SGE_THROTTLE_CONTROL 119 bar2throttlecount = 500 # bar2throttlecount in us 120 121 sge_timer_value = 5, 10, 20, 50, 100, 200 # SGE_TIMER_VALUE* in usecs 122 123 124 reg[0x1124] = 0x00000400/0x00000400 # SGE_CONTROL2, enable VFIFO; if 125 # SGE_VFIFO_SIZE is not set, then 126 # firmware will set it up in function 127 # of number of egress queues used 128 129 reg[0x1130] = 0x00d5ffeb # SGE_DBP_FETCH_THRESHOLD, fetch 130 # threshold set to queue depth 131 # minus 128-entries for FL and HP 132 # queues, and 0xfff for LP which 133 # prompts the firmware to set it up 134 # in function of egress queues 135 # used 136 137 reg[0x113c] = 0x0002ffc0 # SGE_VFIFO_SIZE, set to 0x2ffc0 which 138 # prompts the firmware to set it up in 139 # function of number of egress queues 140 # used 141 142 reg[0x7dc0] = 0x062f8849 # TP_SHIFT_CNT 143 144 # Selection of tuples for LE filter lookup, fields (and widths which 145 # must sum to <= 36): { IP Fragment (1), MPS Match Type (3), 146 # IP Protocol (8), [Inner] VLAN (17), Port (3), FCoE (1) } 147 # 148 filterMode = fragmentation, mpshittype, protocol, vlan, port, fcoe, srvrsram 149 150 # Percentage of dynamic memory (in either the EDRAM or external MEM) 151 # to use for TP RX payload 152 tp_pmrx = 30, 512 153 154 # TP RX payload page size 155 tp_pmrx_pagesize = 64K 156 157 # TP number of RX channels 158 tp_nrxch = 0 # 0 (auto) = 1 159 160 # Percentage of dynamic memory (in either the EDRAM or external MEM) 161 # to use for TP TX payload 162 tp_pmtx = 50, 512 163 164 # TP TX payload page size 165 tp_pmtx_pagesize = 64K 166 167 # TP number of TX channels 168 tp_ntxch = 0 # 0 (auto) = equal number of ports 169 170 reg[0x19c04] = 0x00400000/0x00400000 # LE Server SRAM Enable 171 172# Some "definitions" to make the rest of this a bit more readable. We support 173# 4 ports, 3 functions (NIC, FCoE and iSCSI), scaling up to 8 "CPU Queue Sets" 174# per function per port ... 175# 176# NMSIX = 1088 # available MSI-X Vectors 177# NVI = 128 # available Virtual Interfaces 178# NMPSTCAM = 336 # MPS TCAM entries 179# 180# NPORTS = 4 # ports 181# NCPUS = 8 # CPUs we want to support scalably 182# NFUNCS = 3 # functions per port (NIC, FCoE, iSCSI) 183 184# Breakdown of Virtual Interface/Queue/Interrupt resources for the "Unified 185# PF" which many OS Drivers will use to manage most or all functions. 186# 187# Each Ingress Queue can use one MSI-X interrupt but some Ingress Queues can 188# use Forwarded Interrupt Ingress Queues. For these latter, an Ingress Queue 189# would be created and the Queue ID of a Forwarded Interrupt Ingress Queue 190# will be specified as the "Ingress Queue Asynchronous Destination Index." 191# Thus, the number of MSI-X Vectors assigned to the Unified PF will be less 192# than or equal to the number of Ingress Queues ... 193# 194# NVI_NIC = 4 # NIC access to NPORTS 195# NFLIQ_NIC = 32 # NIC Ingress Queues with Free Lists 196# NETHCTRL_NIC = 32 # NIC Ethernet Control/TX Queues 197# NEQ_NIC = 64 # NIC Egress Queues (FL, ETHCTRL/TX) 198# NMPSTCAM_NIC = 16 # NIC MPS TCAM Entries (NPORTS*4) 199# NMSIX_NIC = 32 # NIC MSI-X Interrupt Vectors (FLIQ) 200# 201# NVI_OFLD = 0 # Offload uses NIC function to access ports 202# NFLIQ_OFLD = 16 # Offload Ingress Queues with Free Lists 203# NETHCTRL_OFLD = 0 # Offload Ethernet Control/TX Queues 204# NEQ_OFLD = 16 # Offload Egress Queues (FL) 205# NMPSTCAM_OFLD = 0 # Offload MPS TCAM Entries (uses NIC's) 206# NMSIX_OFLD = 16 # Offload MSI-X Interrupt Vectors (FLIQ) 207# 208# NVI_RDMA = 0 # RDMA uses NIC function to access ports 209# NFLIQ_RDMA = 4 # RDMA Ingress Queues with Free Lists 210# NETHCTRL_RDMA = 0 # RDMA Ethernet Control/TX Queues 211# NEQ_RDMA = 4 # RDMA Egress Queues (FL) 212# NMPSTCAM_RDMA = 0 # RDMA MPS TCAM Entries (uses NIC's) 213# NMSIX_RDMA = 4 # RDMA MSI-X Interrupt Vectors (FLIQ) 214# 215# NEQ_WD = 128 # Wire Direct TX Queues and FLs 216# NETHCTRL_WD = 64 # Wire Direct TX Queues 217# NFLIQ_WD = 64 ` # Wire Direct Ingress Queues with Free Lists 218# 219# NVI_ISCSI = 4 # ISCSI access to NPORTS 220# NFLIQ_ISCSI = 4 # ISCSI Ingress Queues with Free Lists 221# NETHCTRL_ISCSI = 0 # ISCSI Ethernet Control/TX Queues 222# NEQ_ISCSI = 4 # ISCSI Egress Queues (FL) 223# NMPSTCAM_ISCSI = 4 # ISCSI MPS TCAM Entries (NPORTS) 224# NMSIX_ISCSI = 4 # ISCSI MSI-X Interrupt Vectors (FLIQ) 225# 226# NVI_FCOE = 4 # FCOE access to NPORTS 227# NFLIQ_FCOE = 34 # FCOE Ingress Queues with Free Lists 228# NETHCTRL_FCOE = 32 # FCOE Ethernet Control/TX Queues 229# NEQ_FCOE = 66 # FCOE Egress Queues (FL) 230# NMPSTCAM_FCOE = 32 # FCOE MPS TCAM Entries (NPORTS) 231# NMSIX_FCOE = 34 # FCOE MSI-X Interrupt Vectors (FLIQ) 232 233# Two extra Ingress Queues per function for Firmware Events and Forwarded 234# Interrupts, and two extra interrupts per function for Firmware Events (or a 235# Forwarded Interrupt Queue) and General Interrupts per function. 236# 237# NFLIQ_EXTRA = 6 # "extra" Ingress Queues 2*NFUNCS (Firmware and 238# # Forwarded Interrupts 239# NMSIX_EXTRA = 6 # extra interrupts 2*NFUNCS (Firmware and 240# # General Interrupts 241 242# Microsoft HyperV resources. The HyperV Virtual Ingress Queues will have 243# their interrupts forwarded to another set of Forwarded Interrupt Queues. 244# 245# NVI_HYPERV = 16 # VMs we want to support 246# NVIIQ_HYPERV = 2 # Virtual Ingress Queues with Free Lists per VM 247# NFLIQ_HYPERV = 40 # VIQs + NCPUS Forwarded Interrupt Queues 248# NEQ_HYPERV = 32 # VIQs Free Lists 249# NMPSTCAM_HYPERV = 16 # MPS TCAM Entries (NVI_HYPERV) 250# NMSIX_HYPERV = 8 # NCPUS Forwarded Interrupt Queues 251 252# Adding all of the above Unified PF resource needs together: (NIC + OFLD + 253# RDMA + ISCSI + FCOE + EXTRA + HYPERV) 254# 255# NVI_UNIFIED = 28 256# NFLIQ_UNIFIED = 106 257# NETHCTRL_UNIFIED = 32 258# NEQ_UNIFIED = 124 259# NMPSTCAM_UNIFIED = 40 260# 261# The sum of all the MSI-X resources above is 74 MSI-X Vectors but we'll round 262# that up to 128 to make sure the Unified PF doesn't run out of resources. 263# 264# NMSIX_UNIFIED = 128 265# 266# The Storage PFs could need up to NPORTS*NCPUS + NMSIX_EXTRA MSI-X Vectors 267# which is 34 but they're probably safe with 32. 268# 269# NMSIX_STORAGE = 32 270 271# Note: The UnifiedPF is PF4 which doesn't have any Virtual Functions 272# associated with it. Thus, the MSI-X Vector allocations we give to the 273# UnifiedPF aren't inherited by any Virtual Functions. As a result we can 274# provision many more Virtual Functions than we can if the UnifiedPF were 275# one of PF0-3. 276# 277 278# All of the below PCI-E parameters are actually stored in various *_init.txt 279# files. We include them below essentially as comments. 280# 281# For PF0-3 we assign 8 vectors each for NIC Ingress Queues of the associated 282# ports 0-3. 283# 284# For PF4, the Unified PF, we give it an MSI-X Table Size as outlined above. 285# 286# For PF5-6 we assign enough MSI-X Vectors to support FCoE and iSCSI 287# storage applications across all four possible ports. 288# 289# Additionally, since the UnifiedPF isn't one of the per-port Physical 290# Functions, we give the UnifiedPF and the PF0-3 Physical Functions 291# different PCI Device IDs which will allow Unified and Per-Port Drivers 292# to directly select the type of Physical Function to which they wish to be 293# attached. 294# 295# Note that the actual values used for the PCI-E Intelectual Property will be 296# 1 less than those below since that's the way it "counts" things. For 297# readability, we use the number we actually mean ... 298# 299# PF0_INT = 8 # NCPUS 300# PF1_INT = 8 # NCPUS 301# PF2_INT = 8 # NCPUS 302# PF3_INT = 8 # NCPUS 303# PF0_3_INT = 32 # PF0_INT + PF1_INT + PF2_INT + PF3_INT 304# 305# PF4_INT = 128 # NMSIX_UNIFIED 306# PF5_INT = 32 # NMSIX_STORAGE 307# PF6_INT = 32 # NMSIX_STORAGE 308# PF7_INT = 0 # Nothing Assigned 309# PF4_7_INT = 192 # PF4_INT + PF5_INT + PF6_INT + PF7_INT 310# 311# PF0_7_INT = 224 # PF0_3_INT + PF4_7_INT 312# 313# With the above we can get 17 VFs/PF0-3 (limited by 336 MPS TCAM entries) 314# but we'll lower that to 16 to make our total 64 and a nice power of 2 ... 315# 316# NVF = 16 317 318# For those OSes which manage different ports on different PFs, we need 319# only enough resources to support a single port's NIC application functions 320# on PF0-3. The below assumes that we're only doing NIC with NCPUS "Queue 321# Sets" for ports 0-3. The FCoE and iSCSI functions for such OSes will be 322# managed on the "storage PFs" (see below). 323# 324 325# Some OS Drivers manage all application functions for all ports via PF4. 326# Thus we need to provide a large number of resources here. For Egress 327# Queues we need to account for both TX Queues as well as Free List Queues 328# (because the host is responsible for producing Free List Buffers for the 329# hardware to consume). 330# 331[function "0"] 332 wx_caps = all # write/execute permissions for all commands 333 r_caps = all # read permissions for all commands 334 nvi = 28 # NVI_UNIFIED 335 niqflint = 170 # NFLIQ_UNIFIED + NLFIQ_WD 336 nethctrl = 96 # NETHCTRL_UNIFIED + NETHCTRL_WD 337 neq = 252 # NEQ_UNIFIED + NEQ_WD 338 nexactf = 40 # NMPSTCAM_UNIFIED 339 cmask = all # access to all channels 340 pmask = all # access to all four ports ... 341 nroute = 32 # number of routing region entries 342 nclip = 32 # number of clip region entries 343 nfilter = 48 # number of filter region entries 344 nserver = 32 # number of server region entries 345 nhash = 2048 # number of hash region entries 346 protocol = nic_vm, ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu 347 tp_l2t = 3072 348 tp_ddp = 2 349 tp_ddp_iscsi = 2 350 tp_stag = 2 351 tp_pbl = 5 352 tp_rq = 7 353 354# We have FCoE and iSCSI storage functions on PF5 and PF6 each of which may 355# need to have Virtual Interfaces on each of the four ports with up to NCPUS 356# "Queue Sets" each. 357# 358[function "1"] 359 wx_caps = all # write/execute permissions for all commands 360 r_caps = all # read permissions for all commands 361 nvi = 4 # NPORTS 362 niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA 363 nethctrl = 32 # NPORTS*NCPUS 364 neq = 66 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX) + 2 (EXTRA) 365 nexactf = 32 # NPORTS + adding 28 exact entries for FCoE 366 # which is OK since < MIN(SUM PF0..3, PF4) 367 # and we never load PF0..3 and PF4 concurrently 368 cmask = all # access to all channels 369 pmask = all # access to all four ports ... 370 nhash = 2048 371 protocol = fcoe_initiator 372 tp_ddp = 2 373 fcoe_nfcf = 16 374 fcoe_nvnp = 32 375 fcoe_nssn = 1024 376 377# The following function, 1023, is not an actual PCIE function but is used to 378# configure and reserve firmware internal resources that come from the global 379# resource pool. 380# 381[function "1023"] 382 wx_caps = all # write/execute permissions for all commands 383 r_caps = all # read permissions for all commands 384 nvi = 4 # NVI_UNIFIED 385 cmask = all # access to all channels 386 pmask = all # access to all four ports ... 387 nexactf = 8 # NPORTS + DCBX + 388 nfilter = 16 # number of filter region entries 389 390# For Virtual functions, we only allow NIC functionality and we only allow 391# access to one port (1 << PF). Note that because of limitations in the 392# Scatter Gather Engine (SGE) hardware which checks writes to VF KDOORBELL 393# and GTS registers, the number of Ingress and Egress Queues must be a power 394# of 2. 395# 396[function "0/*"] # NVF 397 wx_caps = 0x82 # DMAQ | VF 398 r_caps = 0x86 # DMAQ | VF | PORT 399 nvi = 1 # 1 port 400 niqflint = 4 # 2 "Queue Sets" + NXIQ 401 nethctrl = 2 # 2 "Queue Sets" 402 neq = 4 # 2 "Queue Sets" * 2 403 nexactf = 4 404 cmask = all # access to all channels 405 pmask = 0x1 # access to only one port ... 406 407[function "1/*"] # NVF 408 wx_caps = 0x82 # DMAQ | VF 409 r_caps = 0x86 # DMAQ | VF | PORT 410 nvi = 1 # 1 port 411 niqflint = 4 # 2 "Queue Sets" + NXIQ 412 nethctrl = 2 # 2 "Queue Sets" 413 neq = 4 # 2 "Queue Sets" * 2 414 nexactf = 4 415 cmask = all # access to all channels 416 pmask = 0x2 # access to only one port ... 417 418# MPS features a 196608 bytes ingress buffer that is used for ingress buffering 419# for packets from the wire as well as the loopback path of the L2 switch. The 420# folling params control how the buffer memory is distributed and the L2 flow 421# control settings: 422# 423# bg_mem: %-age of mem to use for port/buffer group 424# lpbk_mem: %-age of port/bg mem to use for loopback 425# hwm: high watermark; bytes available when starting to send pause 426# frames (in units of 0.1 MTU) 427# lwm: low watermark; bytes remaining when sending 'unpause' frame 428# (in inuits of 0.1 MTU) 429# dwm: minimum delta between high and low watermark (in units of 100 430# Bytes) 431# 432[port "0"] 433 dcb = ppp, dcbx # configure for DCB PPP and enable DCBX offload 434 bg_mem = 25 435 lpbk_mem = 25 436 hwm = 30 437 lwm = 15 438 dwm = 30 439 440[port "1"] 441 dcb = ppp, dcbx 442 bg_mem = 25 443 lpbk_mem = 25 444 hwm = 30 445 lwm = 15 446 dwm = 30 447 448[port "2"] 449 dcb = ppp, dcbx 450 bg_mem = 25 451 lpbk_mem = 25 452 hwm = 30 453 lwm = 15 454 dwm = 30 455 456[port "3"] 457 dcb = ppp, dcbx 458 bg_mem = 25 459 lpbk_mem = 25 460 hwm = 30 461 lwm = 15 462 dwm = 30 463 464[fini] 465 version = 0x1425000d 466 checksum = 0x22f1530b 467 468# Total resources used by above allocations: 469# Virtual Interfaces: 104 470# Ingress Queues/w Free Lists and Interrupts: 526 471# Egress Queues: 702 472# MPS TCAM Entries: 336 473# MSI-X Vectors: 736 474# Virtual Functions: 64 475# 476# 477