1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2011 Chelsio Communications, Inc.
5 * All rights reserved.
6 * Written by: Navdeep Parhar <np@FreeBSD.org>
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include <sys/cdefs.h>
31 #include "opt_ddb.h"
32 #include "opt_inet.h"
33 #include "opt_inet6.h"
34 #include "opt_kern_tls.h"
35 #include "opt_ratelimit.h"
36 #include "opt_rss.h"
37
38 #include <sys/param.h>
39 #include <sys/conf.h>
40 #include <sys/priv.h>
41 #include <sys/kernel.h>
42 #include <sys/bus.h>
43 #include <sys/eventhandler.h>
44 #include <sys/module.h>
45 #include <sys/malloc.h>
46 #include <sys/queue.h>
47 #include <sys/taskqueue.h>
48 #include <sys/pciio.h>
49 #include <dev/pci/pcireg.h>
50 #include <dev/pci/pcivar.h>
51 #include <dev/pci/pci_private.h>
52 #include <sys/firmware.h>
53 #include <sys/sbuf.h>
54 #include <sys/smp.h>
55 #include <sys/socket.h>
56 #include <sys/sockio.h>
57 #include <sys/sysctl.h>
58 #include <net/ethernet.h>
59 #include <net/if.h>
60 #include <net/if_types.h>
61 #include <net/if_dl.h>
62 #include <net/if_vlan_var.h>
63 #ifdef RSS
64 #include <net/rss_config.h>
65 #endif
66 #include <netinet/in.h>
67 #include <netinet/ip.h>
68 #ifdef KERN_TLS
69 #include <netinet/tcp_seq.h>
70 #endif
71 #if defined(__i386__) || defined(__amd64__)
72 #include <machine/md_var.h>
73 #include <machine/cputypes.h>
74 #include <vm/vm.h>
75 #include <vm/pmap.h>
76 #endif
77 #ifdef DDB
78 #include <ddb/ddb.h>
79 #include <ddb/db_lex.h>
80 #endif
81
82 #include "common/common.h"
83 #include "common/t4_msg.h"
84 #include "common/t4_regs.h"
85 #include "common/t4_regs_values.h"
86 #include "cudbg/cudbg.h"
87 #include "t4_clip.h"
88 #include "t4_ioctl.h"
89 #include "t4_l2t.h"
90 #include "t4_mp_ring.h"
91 #include "t4_if.h"
92 #include "t4_smt.h"
93
94 /* T4 bus driver interface */
95 static int t4_probe(device_t);
96 static int t4_attach(device_t);
97 static int t4_detach(device_t);
98 static int t4_child_location(device_t, device_t, struct sbuf *);
99 static int t4_ready(device_t);
100 static int t4_read_port_device(device_t, int, device_t *);
101 static int t4_suspend(device_t);
102 static int t4_resume(device_t);
103 static int t4_reset_prepare(device_t, device_t);
104 static int t4_reset_post(device_t, device_t);
105 static device_method_t t4_methods[] = {
106 DEVMETHOD(device_probe, t4_probe),
107 DEVMETHOD(device_attach, t4_attach),
108 DEVMETHOD(device_detach, t4_detach),
109 DEVMETHOD(device_suspend, t4_suspend),
110 DEVMETHOD(device_resume, t4_resume),
111
112 DEVMETHOD(bus_child_location, t4_child_location),
113 DEVMETHOD(bus_reset_prepare, t4_reset_prepare),
114 DEVMETHOD(bus_reset_post, t4_reset_post),
115
116 DEVMETHOD(t4_is_main_ready, t4_ready),
117 DEVMETHOD(t4_read_port_device, t4_read_port_device),
118
119 DEVMETHOD_END
120 };
121 static driver_t t4_driver = {
122 "t4nex",
123 t4_methods,
124 sizeof(struct adapter)
125 };
126
127
128 /* T4 port (cxgbe) interface */
129 static int cxgbe_probe(device_t);
130 static int cxgbe_attach(device_t);
131 static int cxgbe_detach(device_t);
132 device_method_t cxgbe_methods[] = {
133 DEVMETHOD(device_probe, cxgbe_probe),
134 DEVMETHOD(device_attach, cxgbe_attach),
135 DEVMETHOD(device_detach, cxgbe_detach),
136 { 0, 0 }
137 };
138 static driver_t cxgbe_driver = {
139 "cxgbe",
140 cxgbe_methods,
141 sizeof(struct port_info)
142 };
143
144 /* T4 VI (vcxgbe) interface */
145 static int vcxgbe_probe(device_t);
146 static int vcxgbe_attach(device_t);
147 static int vcxgbe_detach(device_t);
148 static device_method_t vcxgbe_methods[] = {
149 DEVMETHOD(device_probe, vcxgbe_probe),
150 DEVMETHOD(device_attach, vcxgbe_attach),
151 DEVMETHOD(device_detach, vcxgbe_detach),
152 { 0, 0 }
153 };
154 static driver_t vcxgbe_driver = {
155 "vcxgbe",
156 vcxgbe_methods,
157 sizeof(struct vi_info)
158 };
159
160 static d_ioctl_t t4_ioctl;
161
162 static struct cdevsw t4_cdevsw = {
163 .d_version = D_VERSION,
164 .d_ioctl = t4_ioctl,
165 .d_name = "t4nex",
166 };
167
168 /* T5 bus driver interface */
169 static int t5_probe(device_t);
170 static device_method_t t5_methods[] = {
171 DEVMETHOD(device_probe, t5_probe),
172 DEVMETHOD(device_attach, t4_attach),
173 DEVMETHOD(device_detach, t4_detach),
174 DEVMETHOD(device_suspend, t4_suspend),
175 DEVMETHOD(device_resume, t4_resume),
176
177 DEVMETHOD(bus_child_location, t4_child_location),
178 DEVMETHOD(bus_reset_prepare, t4_reset_prepare),
179 DEVMETHOD(bus_reset_post, t4_reset_post),
180
181 DEVMETHOD(t4_is_main_ready, t4_ready),
182 DEVMETHOD(t4_read_port_device, t4_read_port_device),
183
184 DEVMETHOD_END
185 };
186 static driver_t t5_driver = {
187 "t5nex",
188 t5_methods,
189 sizeof(struct adapter)
190 };
191
192
193 /* T5 port (cxl) interface */
194 static driver_t cxl_driver = {
195 "cxl",
196 cxgbe_methods,
197 sizeof(struct port_info)
198 };
199
200 /* T5 VI (vcxl) interface */
201 static driver_t vcxl_driver = {
202 "vcxl",
203 vcxgbe_methods,
204 sizeof(struct vi_info)
205 };
206
207 /* T6 bus driver interface */
208 static int t6_probe(device_t);
209 static device_method_t t6_methods[] = {
210 DEVMETHOD(device_probe, t6_probe),
211 DEVMETHOD(device_attach, t4_attach),
212 DEVMETHOD(device_detach, t4_detach),
213 DEVMETHOD(device_suspend, t4_suspend),
214 DEVMETHOD(device_resume, t4_resume),
215
216 DEVMETHOD(bus_child_location, t4_child_location),
217 DEVMETHOD(bus_reset_prepare, t4_reset_prepare),
218 DEVMETHOD(bus_reset_post, t4_reset_post),
219
220 DEVMETHOD(t4_is_main_ready, t4_ready),
221 DEVMETHOD(t4_read_port_device, t4_read_port_device),
222
223 DEVMETHOD_END
224 };
225 static driver_t t6_driver = {
226 "t6nex",
227 t6_methods,
228 sizeof(struct adapter)
229 };
230
231
232 /* T6 port (cc) interface */
233 static driver_t cc_driver = {
234 "cc",
235 cxgbe_methods,
236 sizeof(struct port_info)
237 };
238
239 /* T6 VI (vcc) interface */
240 static driver_t vcc_driver = {
241 "vcc",
242 vcxgbe_methods,
243 sizeof(struct vi_info)
244 };
245
246 /* ifnet interface */
247 static void cxgbe_init(void *);
248 static int cxgbe_ioctl(if_t, unsigned long, caddr_t);
249 static int cxgbe_transmit(if_t, struct mbuf *);
250 static void cxgbe_qflush(if_t);
251 #if defined(KERN_TLS) || defined(RATELIMIT)
252 static int cxgbe_snd_tag_alloc(if_t, union if_snd_tag_alloc_params *,
253 struct m_snd_tag **);
254 #endif
255
256 MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
257
258 /*
259 * Correct lock order when you need to acquire multiple locks is t4_list_lock,
260 * then ADAPTER_LOCK, then t4_uld_list_lock.
261 */
262 static struct sx t4_list_lock;
263 SLIST_HEAD(, adapter) t4_list;
264 #ifdef TCP_OFFLOAD
265 static struct sx t4_uld_list_lock;
266 SLIST_HEAD(, uld_info) t4_uld_list;
267 #endif
268
269 /*
270 * Tunables. See tweak_tunables() too.
271 *
272 * Each tunable is set to a default value here if it's known at compile-time.
273 * Otherwise it is set to -n as an indication to tweak_tunables() that it should
274 * provide a reasonable default (upto n) when the driver is loaded.
275 *
276 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to
277 * T5 are under hw.cxl.
278 */
279 SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
280 "cxgbe(4) parameters");
281 SYSCTL_NODE(_hw, OID_AUTO, cxl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
282 "cxgbe(4) T5+ parameters");
283 SYSCTL_NODE(_hw_cxgbe, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
284 "cxgbe(4) TOE parameters");
285
286 /*
287 * Number of queues for tx and rx, NIC and offload.
288 */
289 #define NTXQ 16
290 int t4_ntxq = -NTXQ;
291 SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq, CTLFLAG_RDTUN, &t4_ntxq, 0,
292 "Number of TX queues per port");
293 TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq); /* Old name, undocumented */
294
295 #define NRXQ 8
296 int t4_nrxq = -NRXQ;
297 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq, CTLFLAG_RDTUN, &t4_nrxq, 0,
298 "Number of RX queues per port");
299 TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq); /* Old name, undocumented */
300
301 #define NTXQ_VI 1
302 static int t4_ntxq_vi = -NTXQ_VI;
303 SYSCTL_INT(_hw_cxgbe, OID_AUTO, ntxq_vi, CTLFLAG_RDTUN, &t4_ntxq_vi, 0,
304 "Number of TX queues per VI");
305
306 #define NRXQ_VI 1
307 static int t4_nrxq_vi = -NRXQ_VI;
308 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nrxq_vi, CTLFLAG_RDTUN, &t4_nrxq_vi, 0,
309 "Number of RX queues per VI");
310
311 static int t4_rsrv_noflowq = 0;
312 SYSCTL_INT(_hw_cxgbe, OID_AUTO, rsrv_noflowq, CTLFLAG_RDTUN, &t4_rsrv_noflowq,
313 0, "Reserve TX queue 0 of each VI for non-flowid packets");
314
315 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
316 #define NOFLDTXQ 8
317 static int t4_nofldtxq = -NOFLDTXQ;
318 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq, CTLFLAG_RDTUN, &t4_nofldtxq, 0,
319 "Number of offload TX queues per port");
320
321 #define NOFLDRXQ 2
322 static int t4_nofldrxq = -NOFLDRXQ;
323 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq, CTLFLAG_RDTUN, &t4_nofldrxq, 0,
324 "Number of offload RX queues per port");
325
326 #define NOFLDTXQ_VI 1
327 static int t4_nofldtxq_vi = -NOFLDTXQ_VI;
328 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldtxq_vi, CTLFLAG_RDTUN, &t4_nofldtxq_vi, 0,
329 "Number of offload TX queues per VI");
330
331 #define NOFLDRXQ_VI 1
332 static int t4_nofldrxq_vi = -NOFLDRXQ_VI;
333 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nofldrxq_vi, CTLFLAG_RDTUN, &t4_nofldrxq_vi, 0,
334 "Number of offload RX queues per VI");
335
336 #define TMR_IDX_OFLD 1
337 int t4_tmr_idx_ofld = TMR_IDX_OFLD;
338 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_ofld, CTLFLAG_RDTUN,
339 &t4_tmr_idx_ofld, 0, "Holdoff timer index for offload queues");
340
341 #define PKTC_IDX_OFLD (-1)
342 int t4_pktc_idx_ofld = PKTC_IDX_OFLD;
343 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_ofld, CTLFLAG_RDTUN,
344 &t4_pktc_idx_ofld, 0, "holdoff packet counter index for offload queues");
345
346 /* 0 means chip/fw default, non-zero number is value in microseconds */
347 static u_long t4_toe_keepalive_idle = 0;
348 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_idle, CTLFLAG_RDTUN,
349 &t4_toe_keepalive_idle, 0, "TOE keepalive idle timer (us)");
350
351 /* 0 means chip/fw default, non-zero number is value in microseconds */
352 static u_long t4_toe_keepalive_interval = 0;
353 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, keepalive_interval, CTLFLAG_RDTUN,
354 &t4_toe_keepalive_interval, 0, "TOE keepalive interval timer (us)");
355
356 /* 0 means chip/fw default, non-zero number is # of keepalives before abort */
357 static int t4_toe_keepalive_count = 0;
358 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, keepalive_count, CTLFLAG_RDTUN,
359 &t4_toe_keepalive_count, 0, "Number of TOE keepalive probes before abort");
360
361 /* 0 means chip/fw default, non-zero number is value in microseconds */
362 static u_long t4_toe_rexmt_min = 0;
363 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_min, CTLFLAG_RDTUN,
364 &t4_toe_rexmt_min, 0, "Minimum TOE retransmit interval (us)");
365
366 /* 0 means chip/fw default, non-zero number is value in microseconds */
367 static u_long t4_toe_rexmt_max = 0;
368 SYSCTL_ULONG(_hw_cxgbe_toe, OID_AUTO, rexmt_max, CTLFLAG_RDTUN,
369 &t4_toe_rexmt_max, 0, "Maximum TOE retransmit interval (us)");
370
371 /* 0 means chip/fw default, non-zero number is # of rexmt before abort */
372 static int t4_toe_rexmt_count = 0;
373 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, rexmt_count, CTLFLAG_RDTUN,
374 &t4_toe_rexmt_count, 0, "Number of TOE retransmissions before abort");
375
376 /* -1 means chip/fw default, other values are raw backoff values to use */
377 static int t4_toe_rexmt_backoff[16] = {
378 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
379 };
380 SYSCTL_NODE(_hw_cxgbe_toe, OID_AUTO, rexmt_backoff,
381 CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
382 "cxgbe(4) TOE retransmit backoff values");
383 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 0, CTLFLAG_RDTUN,
384 &t4_toe_rexmt_backoff[0], 0, "");
385 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 1, CTLFLAG_RDTUN,
386 &t4_toe_rexmt_backoff[1], 0, "");
387 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 2, CTLFLAG_RDTUN,
388 &t4_toe_rexmt_backoff[2], 0, "");
389 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 3, CTLFLAG_RDTUN,
390 &t4_toe_rexmt_backoff[3], 0, "");
391 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 4, CTLFLAG_RDTUN,
392 &t4_toe_rexmt_backoff[4], 0, "");
393 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 5, CTLFLAG_RDTUN,
394 &t4_toe_rexmt_backoff[5], 0, "");
395 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 6, CTLFLAG_RDTUN,
396 &t4_toe_rexmt_backoff[6], 0, "");
397 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 7, CTLFLAG_RDTUN,
398 &t4_toe_rexmt_backoff[7], 0, "");
399 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 8, CTLFLAG_RDTUN,
400 &t4_toe_rexmt_backoff[8], 0, "");
401 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 9, CTLFLAG_RDTUN,
402 &t4_toe_rexmt_backoff[9], 0, "");
403 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 10, CTLFLAG_RDTUN,
404 &t4_toe_rexmt_backoff[10], 0, "");
405 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 11, CTLFLAG_RDTUN,
406 &t4_toe_rexmt_backoff[11], 0, "");
407 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 12, CTLFLAG_RDTUN,
408 &t4_toe_rexmt_backoff[12], 0, "");
409 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 13, CTLFLAG_RDTUN,
410 &t4_toe_rexmt_backoff[13], 0, "");
411 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 14, CTLFLAG_RDTUN,
412 &t4_toe_rexmt_backoff[14], 0, "");
413 SYSCTL_INT(_hw_cxgbe_toe_rexmt_backoff, OID_AUTO, 15, CTLFLAG_RDTUN,
414 &t4_toe_rexmt_backoff[15], 0, "");
415
416 int t4_ddp_rcvbuf_len = 256 * 1024;
417 SYSCTL_INT(_hw_cxgbe_toe, OID_AUTO, ddp_rcvbuf_len, CTLFLAG_RWTUN,
418 &t4_ddp_rcvbuf_len, 0, "length of each DDP RX buffer");
419
420 unsigned int t4_ddp_rcvbuf_cache = 4;
421 SYSCTL_UINT(_hw_cxgbe_toe, OID_AUTO, ddp_rcvbuf_cache, CTLFLAG_RWTUN,
422 &t4_ddp_rcvbuf_cache, 0,
423 "maximum number of free DDP RX buffers to cache per connection");
424 #endif
425
426 #ifdef DEV_NETMAP
427 #define NN_MAIN_VI (1 << 0) /* Native netmap on the main VI */
428 #define NN_EXTRA_VI (1 << 1) /* Native netmap on the extra VI(s) */
429 static int t4_native_netmap = NN_EXTRA_VI;
430 SYSCTL_INT(_hw_cxgbe, OID_AUTO, native_netmap, CTLFLAG_RDTUN, &t4_native_netmap,
431 0, "Native netmap support. bit 0 = main VI, bit 1 = extra VIs");
432
433 #define NNMTXQ 8
434 static int t4_nnmtxq = -NNMTXQ;
435 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq, CTLFLAG_RDTUN, &t4_nnmtxq, 0,
436 "Number of netmap TX queues");
437
438 #define NNMRXQ 8
439 static int t4_nnmrxq = -NNMRXQ;
440 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq, CTLFLAG_RDTUN, &t4_nnmrxq, 0,
441 "Number of netmap RX queues");
442
443 #define NNMTXQ_VI 2
444 static int t4_nnmtxq_vi = -NNMTXQ_VI;
445 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmtxq_vi, CTLFLAG_RDTUN, &t4_nnmtxq_vi, 0,
446 "Number of netmap TX queues per VI");
447
448 #define NNMRXQ_VI 2
449 static int t4_nnmrxq_vi = -NNMRXQ_VI;
450 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nnmrxq_vi, CTLFLAG_RDTUN, &t4_nnmrxq_vi, 0,
451 "Number of netmap RX queues per VI");
452 #endif
453
454 /*
455 * Holdoff parameters for ports.
456 */
457 #define TMR_IDX 1
458 int t4_tmr_idx = TMR_IDX;
459 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx, CTLFLAG_RDTUN, &t4_tmr_idx,
460 0, "Holdoff timer index");
461 TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx); /* Old name */
462
463 #define PKTC_IDX (-1)
464 int t4_pktc_idx = PKTC_IDX;
465 SYSCTL_INT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx, CTLFLAG_RDTUN, &t4_pktc_idx,
466 0, "Holdoff packet counter index");
467 TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx); /* Old name */
468
469 /*
470 * Size (# of entries) of each tx and rx queue.
471 */
472 unsigned int t4_qsize_txq = TX_EQ_QSIZE;
473 SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN, &t4_qsize_txq, 0,
474 "Number of descriptors in each TX queue");
475
476 unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
477 SYSCTL_INT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN, &t4_qsize_rxq, 0,
478 "Number of descriptors in each RX queue");
479
480 /*
481 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
482 */
483 int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
484 SYSCTL_INT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &t4_intr_types,
485 0, "Interrupt types allowed (bit 0 = INTx, 1 = MSI, 2 = MSI-X)");
486
487 /*
488 * Configuration file. All the _CF names here are special.
489 */
490 #define DEFAULT_CF "default"
491 #define BUILTIN_CF "built-in"
492 #define FLASH_CF "flash"
493 #define UWIRE_CF "uwire"
494 #define FPGA_CF "fpga"
495 static char t4_cfg_file[32] = DEFAULT_CF;
496 SYSCTL_STRING(_hw_cxgbe, OID_AUTO, config_file, CTLFLAG_RDTUN, t4_cfg_file,
497 sizeof(t4_cfg_file), "Firmware configuration file");
498
499 /*
500 * PAUSE settings (bit 0, 1, 2 = rx_pause, tx_pause, pause_autoneg respectively).
501 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them.
502 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water
503 * mark or when signalled to do so, 0 to never emit PAUSE.
504 * pause_autoneg = 1 means PAUSE will be negotiated if possible and the
505 * negotiated settings will override rx_pause/tx_pause.
506 * Otherwise rx_pause/tx_pause are applied forcibly.
507 */
508 static int t4_pause_settings = PAUSE_RX | PAUSE_TX | PAUSE_AUTONEG;
509 SYSCTL_INT(_hw_cxgbe, OID_AUTO, pause_settings, CTLFLAG_RDTUN,
510 &t4_pause_settings, 0,
511 "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)");
512
513 /*
514 * Forward Error Correction settings (bit 0, 1 = RS, BASER respectively).
515 * -1 to run with the firmware default. Same as FEC_AUTO (bit 5)
516 * 0 to disable FEC.
517 */
518 static int t4_fec = -1;
519 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fec, CTLFLAG_RDTUN, &t4_fec, 0,
520 "Forward Error Correction (bit 0 = RS, bit 1 = BASER_RS)");
521
522 /*
523 * Controls when the driver sets the FORCE_FEC bit in the L1_CFG32 that it
524 * issues to the firmware. If the firmware doesn't support FORCE_FEC then the
525 * driver runs as if this is set to 0.
526 * -1 to set FORCE_FEC iff requested_fec != AUTO. Multiple FEC bits are okay.
527 * 0 to never set FORCE_FEC. requested_fec = AUTO means use the hint from the
528 * transceiver. Multiple FEC bits may not be okay but will be passed on to
529 * the firmware anyway (may result in l1cfg errors with old firmwares).
530 * 1 to always set FORCE_FEC. Multiple FEC bits are okay. requested_fec = AUTO
531 * means set all FEC bits that are valid for the speed.
532 */
533 static int t4_force_fec = -1;
534 SYSCTL_INT(_hw_cxgbe, OID_AUTO, force_fec, CTLFLAG_RDTUN, &t4_force_fec, 0,
535 "Controls the use of FORCE_FEC bit in L1 configuration.");
536
537 /*
538 * Link autonegotiation.
539 * -1 to run with the firmware default.
540 * 0 to disable.
541 * 1 to enable.
542 */
543 static int t4_autoneg = -1;
544 SYSCTL_INT(_hw_cxgbe, OID_AUTO, autoneg, CTLFLAG_RDTUN, &t4_autoneg, 0,
545 "Link autonegotiation");
546
547 /*
548 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed,
549 * encouraged respectively). '-n' is the same as 'n' except the firmware
550 * version used in the checks is read from the firmware bundled with the driver.
551 */
552 static int t4_fw_install = 1;
553 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fw_install, CTLFLAG_RDTUN, &t4_fw_install, 0,
554 "Firmware auto-install (0 = prohibited, 1 = allowed, 2 = encouraged)");
555
556 /*
557 * ASIC features that will be used. Disable the ones you don't want so that the
558 * chip resources aren't wasted on features that will not be used.
559 */
560 static int t4_nbmcaps_allowed = 0;
561 SYSCTL_INT(_hw_cxgbe, OID_AUTO, nbmcaps_allowed, CTLFLAG_RDTUN,
562 &t4_nbmcaps_allowed, 0, "Default NBM capabilities");
563
564 static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */
565 SYSCTL_INT(_hw_cxgbe, OID_AUTO, linkcaps_allowed, CTLFLAG_RDTUN,
566 &t4_linkcaps_allowed, 0, "Default link capabilities");
567
568 static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS |
569 FW_CAPS_CONFIG_SWITCH_EGRESS;
570 SYSCTL_INT(_hw_cxgbe, OID_AUTO, switchcaps_allowed, CTLFLAG_RDTUN,
571 &t4_switchcaps_allowed, 0, "Default switch capabilities");
572
573 #ifdef RATELIMIT
574 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC |
575 FW_CAPS_CONFIG_NIC_HASHFILTER | FW_CAPS_CONFIG_NIC_ETHOFLD;
576 #else
577 static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC |
578 FW_CAPS_CONFIG_NIC_HASHFILTER;
579 #endif
580 SYSCTL_INT(_hw_cxgbe, OID_AUTO, niccaps_allowed, CTLFLAG_RDTUN,
581 &t4_niccaps_allowed, 0, "Default NIC capabilities");
582
583 static int t4_toecaps_allowed = -1;
584 SYSCTL_INT(_hw_cxgbe, OID_AUTO, toecaps_allowed, CTLFLAG_RDTUN,
585 &t4_toecaps_allowed, 0, "Default TCP offload capabilities");
586
587 static int t4_rdmacaps_allowed = -1;
588 SYSCTL_INT(_hw_cxgbe, OID_AUTO, rdmacaps_allowed, CTLFLAG_RDTUN,
589 &t4_rdmacaps_allowed, 0, "Default RDMA capabilities");
590
591 static int t4_cryptocaps_allowed = -1;
592 SYSCTL_INT(_hw_cxgbe, OID_AUTO, cryptocaps_allowed, CTLFLAG_RDTUN,
593 &t4_cryptocaps_allowed, 0, "Default crypto capabilities");
594
595 static int t4_iscsicaps_allowed = -1;
596 SYSCTL_INT(_hw_cxgbe, OID_AUTO, iscsicaps_allowed, CTLFLAG_RDTUN,
597 &t4_iscsicaps_allowed, 0, "Default iSCSI capabilities");
598
599 static int t4_fcoecaps_allowed = 0;
600 SYSCTL_INT(_hw_cxgbe, OID_AUTO, fcoecaps_allowed, CTLFLAG_RDTUN,
601 &t4_fcoecaps_allowed, 0, "Default FCoE capabilities");
602
603 static int t5_write_combine = 0;
604 SYSCTL_INT(_hw_cxl, OID_AUTO, write_combine, CTLFLAG_RDTUN, &t5_write_combine,
605 0, "Use WC instead of UC for BAR2");
606
607 static int t4_num_vis = 1;
608 SYSCTL_INT(_hw_cxgbe, OID_AUTO, num_vis, CTLFLAG_RDTUN, &t4_num_vis, 0,
609 "Number of VIs per port");
610
611 /*
612 * PCIe Relaxed Ordering.
613 * -1: driver should figure out a good value.
614 * 0: disable RO.
615 * 1: enable RO.
616 * 2: leave RO alone.
617 */
618 static int pcie_relaxed_ordering = -1;
619 SYSCTL_INT(_hw_cxgbe, OID_AUTO, pcie_relaxed_ordering, CTLFLAG_RDTUN,
620 &pcie_relaxed_ordering, 0,
621 "PCIe Relaxed Ordering: 0 = disable, 1 = enable, 2 = leave alone");
622
623 static int t4_panic_on_fatal_err = 0;
624 SYSCTL_INT(_hw_cxgbe, OID_AUTO, panic_on_fatal_err, CTLFLAG_RWTUN,
625 &t4_panic_on_fatal_err, 0, "panic on fatal errors");
626
627 static int t4_reset_on_fatal_err = 0;
628 SYSCTL_INT(_hw_cxgbe, OID_AUTO, reset_on_fatal_err, CTLFLAG_RWTUN,
629 &t4_reset_on_fatal_err, 0, "reset adapter on fatal errors");
630
631 static int t4_clock_gate_on_suspend = 0;
632 SYSCTL_INT(_hw_cxgbe, OID_AUTO, clock_gate_on_suspend, CTLFLAG_RWTUN,
633 &t4_clock_gate_on_suspend, 0, "gate the clock on suspend");
634
635 static int t4_tx_vm_wr = 0;
636 SYSCTL_INT(_hw_cxgbe, OID_AUTO, tx_vm_wr, CTLFLAG_RWTUN, &t4_tx_vm_wr, 0,
637 "Use VM work requests to transmit packets.");
638
639 /*
640 * Set to non-zero to enable the attack filter. A packet that matches any of
641 * these conditions will get dropped on ingress:
642 * 1) IP && source address == destination address.
643 * 2) TCP/IP && source address is not a unicast address.
644 * 3) TCP/IP && destination address is not a unicast address.
645 * 4) IP && source address is loopback (127.x.y.z).
646 * 5) IP && destination address is loopback (127.x.y.z).
647 * 6) IPv6 && source address == destination address.
648 * 7) IPv6 && source address is not a unicast address.
649 * 8) IPv6 && source address is loopback (::1/128).
650 * 9) IPv6 && destination address is loopback (::1/128).
651 * 10) IPv6 && source address is unspecified (::/128).
652 * 11) IPv6 && destination address is unspecified (::/128).
653 * 12) TCP/IPv6 && source address is multicast (ff00::/8).
654 * 13) TCP/IPv6 && destination address is multicast (ff00::/8).
655 */
656 static int t4_attack_filter = 0;
657 SYSCTL_INT(_hw_cxgbe, OID_AUTO, attack_filter, CTLFLAG_RDTUN,
658 &t4_attack_filter, 0, "Drop suspicious traffic");
659
660 static int t4_drop_ip_fragments = 0;
661 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_ip_fragments, CTLFLAG_RDTUN,
662 &t4_drop_ip_fragments, 0, "Drop IP fragments");
663
664 static int t4_drop_pkts_with_l2_errors = 1;
665 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l2_errors, CTLFLAG_RDTUN,
666 &t4_drop_pkts_with_l2_errors, 0,
667 "Drop all frames with Layer 2 length or checksum errors");
668
669 static int t4_drop_pkts_with_l3_errors = 0;
670 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l3_errors, CTLFLAG_RDTUN,
671 &t4_drop_pkts_with_l3_errors, 0,
672 "Drop all frames with IP version, length, or checksum errors");
673
674 static int t4_drop_pkts_with_l4_errors = 0;
675 SYSCTL_INT(_hw_cxgbe, OID_AUTO, drop_pkts_with_l4_errors, CTLFLAG_RDTUN,
676 &t4_drop_pkts_with_l4_errors, 0,
677 "Drop all frames with Layer 4 length, checksum, or other errors");
678
679 #ifdef TCP_OFFLOAD
680 /*
681 * TOE tunables.
682 */
683 static int t4_cop_managed_offloading = 0;
684 SYSCTL_INT(_hw_cxgbe, OID_AUTO, cop_managed_offloading, CTLFLAG_RDTUN,
685 &t4_cop_managed_offloading, 0,
686 "COP (Connection Offload Policy) controls all TOE offload");
687 #endif
688
689 #ifdef KERN_TLS
690 /*
691 * This enables KERN_TLS for all adapters if set.
692 */
693 static int t4_kern_tls = 0;
694 SYSCTL_INT(_hw_cxgbe, OID_AUTO, kern_tls, CTLFLAG_RDTUN, &t4_kern_tls, 0,
695 "Enable KERN_TLS mode for T6 adapters");
696
697 SYSCTL_NODE(_hw_cxgbe, OID_AUTO, tls, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
698 "cxgbe(4) KERN_TLS parameters");
699
700 static int t4_tls_inline_keys = 0;
701 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, inline_keys, CTLFLAG_RDTUN,
702 &t4_tls_inline_keys, 0,
703 "Always pass TLS keys in work requests (1) or attempt to store TLS keys "
704 "in card memory.");
705
706 static int t4_tls_combo_wrs = 0;
707 SYSCTL_INT(_hw_cxgbe_tls, OID_AUTO, combo_wrs, CTLFLAG_RDTUN, &t4_tls_combo_wrs,
708 0, "Attempt to combine TCB field updates with TLS record work requests.");
709 #endif
710
711 /* Functions used by VIs to obtain unique MAC addresses for each VI. */
712 static int vi_mac_funcs[] = {
713 FW_VI_FUNC_ETH,
714 FW_VI_FUNC_OFLD,
715 FW_VI_FUNC_IWARP,
716 FW_VI_FUNC_OPENISCSI,
717 FW_VI_FUNC_OPENFCOE,
718 FW_VI_FUNC_FOISCSI,
719 FW_VI_FUNC_FOFCOE,
720 };
721
722 struct intrs_and_queues {
723 uint16_t intr_type; /* INTx, MSI, or MSI-X */
724 uint16_t num_vis; /* number of VIs for each port */
725 uint16_t nirq; /* Total # of vectors */
726 uint16_t ntxq; /* # of NIC txq's for each port */
727 uint16_t nrxq; /* # of NIC rxq's for each port */
728 uint16_t nofldtxq; /* # of TOE/ETHOFLD txq's for each port */
729 uint16_t nofldrxq; /* # of TOE rxq's for each port */
730 uint16_t nnmtxq; /* # of netmap txq's */
731 uint16_t nnmrxq; /* # of netmap rxq's */
732
733 /* The vcxgbe/vcxl interfaces use these and not the ones above. */
734 uint16_t ntxq_vi; /* # of NIC txq's */
735 uint16_t nrxq_vi; /* # of NIC rxq's */
736 uint16_t nofldtxq_vi; /* # of TOE txq's */
737 uint16_t nofldrxq_vi; /* # of TOE rxq's */
738 uint16_t nnmtxq_vi; /* # of netmap txq's */
739 uint16_t nnmrxq_vi; /* # of netmap rxq's */
740 };
741
742 static void setup_memwin(struct adapter *);
743 static void position_memwin(struct adapter *, int, uint32_t);
744 static int validate_mem_range(struct adapter *, uint32_t, uint32_t);
745 static int fwmtype_to_hwmtype(int);
746 static int validate_mt_off_len(struct adapter *, int, uint32_t, uint32_t,
747 uint32_t *);
748 static int fixup_devlog_params(struct adapter *);
749 static int cfg_itype_and_nqueues(struct adapter *, struct intrs_and_queues *);
750 static int contact_firmware(struct adapter *);
751 static int partition_resources(struct adapter *);
752 static int get_params__pre_init(struct adapter *);
753 static int set_params__pre_init(struct adapter *);
754 static int get_params__post_init(struct adapter *);
755 static int set_params__post_init(struct adapter *);
756 static void t4_set_desc(struct adapter *);
757 static bool fixed_ifmedia(struct port_info *);
758 static void build_medialist(struct port_info *);
759 static void init_link_config(struct port_info *);
760 static int fixup_link_config(struct port_info *);
761 static int apply_link_config(struct port_info *);
762 static int cxgbe_init_synchronized(struct vi_info *);
763 static int cxgbe_uninit_synchronized(struct vi_info *);
764 static int adapter_full_init(struct adapter *);
765 static void adapter_full_uninit(struct adapter *);
766 static int vi_full_init(struct vi_info *);
767 static void vi_full_uninit(struct vi_info *);
768 static int alloc_extra_vi(struct adapter *, struct port_info *, struct vi_info *);
769 static void quiesce_txq(struct sge_txq *);
770 static void quiesce_wrq(struct sge_wrq *);
771 static void quiesce_iq_fl(struct adapter *, struct sge_iq *, struct sge_fl *);
772 static void quiesce_vi(struct vi_info *);
773 static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
774 driver_intr_t *, void *, char *);
775 static int t4_free_irq(struct adapter *, struct irq *);
776 static void t4_init_atid_table(struct adapter *);
777 static void t4_free_atid_table(struct adapter *);
778 static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
779 static void vi_refresh_stats(struct vi_info *);
780 static void cxgbe_refresh_stats(struct vi_info *);
781 static void cxgbe_tick(void *);
782 static void vi_tick(void *);
783 static void cxgbe_sysctls(struct port_info *);
784 static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
785 static int sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS);
786 static int sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS);
787 static int sysctl_btphy(SYSCTL_HANDLER_ARGS);
788 static int sysctl_noflowq(SYSCTL_HANDLER_ARGS);
789 static int sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS);
790 static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
791 static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
792 static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
793 static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
794 static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS);
795 static int sysctl_link_fec(SYSCTL_HANDLER_ARGS);
796 static int sysctl_requested_fec(SYSCTL_HANDLER_ARGS);
797 static int sysctl_module_fec(SYSCTL_HANDLER_ARGS);
798 static int sysctl_autoneg(SYSCTL_HANDLER_ARGS);
799 static int sysctl_force_fec(SYSCTL_HANDLER_ARGS);
800 static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
801 static int sysctl_temperature(SYSCTL_HANDLER_ARGS);
802 static int sysctl_vdd(SYSCTL_HANDLER_ARGS);
803 static int sysctl_reset_sensor(SYSCTL_HANDLER_ARGS);
804 static int sysctl_loadavg(SYSCTL_HANDLER_ARGS);
805 static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
806 static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS);
807 static int sysctl_cim_la(SYSCTL_HANDLER_ARGS);
808 static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS);
809 static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS);
810 static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS);
811 static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
812 static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
813 static int sysctl_tid_stats(SYSCTL_HANDLER_ARGS);
814 static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
815 static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
816 static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
817 static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
818 static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS);
819 static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
820 static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS);
821 static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS);
822 static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
823 static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
824 static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
825 static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
826 static int sysctl_tids(SYSCTL_HANDLER_ARGS);
827 static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
828 static int sysctl_tnl_stats(SYSCTL_HANDLER_ARGS);
829 static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS);
830 static int sysctl_tp_la(SYSCTL_HANDLER_ARGS);
831 static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
832 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
833 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
834 static int sysctl_cpus(SYSCTL_HANDLER_ARGS);
835 static int sysctl_reset(SYSCTL_HANDLER_ARGS);
836 #ifdef TCP_OFFLOAD
837 static int sysctl_tls(SYSCTL_HANDLER_ARGS);
838 static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS);
839 static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS);
840 static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS);
841 static int sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS);
842 static int sysctl_tp_backoff(SYSCTL_HANDLER_ARGS);
843 static int sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS);
844 static int sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS);
845 #endif
846 static int get_sge_context(struct adapter *, struct t4_sge_context *);
847 static int load_fw(struct adapter *, struct t4_data *);
848 static int load_cfg(struct adapter *, struct t4_data *);
849 static int load_boot(struct adapter *, struct t4_bootrom *);
850 static int load_bootcfg(struct adapter *, struct t4_data *);
851 static int cudbg_dump(struct adapter *, struct t4_cudbg_dump *);
852 static void free_offload_policy(struct t4_offload_policy *);
853 static int set_offload_policy(struct adapter *, struct t4_offload_policy *);
854 static int read_card_mem(struct adapter *, int, struct t4_mem_range *);
855 static int read_i2c(struct adapter *, struct t4_i2c_data *);
856 static int clear_stats(struct adapter *, u_int);
857 static int hold_clip_addr(struct adapter *, struct t4_clip_addr *);
858 static int release_clip_addr(struct adapter *, struct t4_clip_addr *);
859 #ifdef TCP_OFFLOAD
860 static int toe_capability(struct vi_info *, bool);
861 static int t4_deactivate_all_uld(struct adapter *);
862 static void t4_async_event(struct adapter *);
863 #endif
864 #ifdef KERN_TLS
865 static int ktls_capability(struct adapter *, bool);
866 #endif
867 static int mod_event(module_t, int, void *);
868 static int notify_siblings(device_t, int);
869 static uint64_t vi_get_counter(if_t, ift_counter);
870 static uint64_t cxgbe_get_counter(if_t, ift_counter);
871 static void enable_vxlan_rx(struct adapter *);
872 static void reset_adapter_task(void *, int);
873 static void fatal_error_task(void *, int);
874 static void dump_devlog(struct adapter *);
875 static void dump_cim_regs(struct adapter *);
876 static void dump_cimla(struct adapter *);
877
878 struct {
879 uint16_t device;
880 char *desc;
881 } t4_pciids[] = {
882 {0xa000, "Chelsio Terminator 4 FPGA"},
883 {0x4400, "Chelsio T440-dbg"},
884 {0x4401, "Chelsio T420-CR"},
885 {0x4402, "Chelsio T422-CR"},
886 {0x4403, "Chelsio T440-CR"},
887 {0x4404, "Chelsio T420-BCH"},
888 {0x4405, "Chelsio T440-BCH"},
889 {0x4406, "Chelsio T440-CH"},
890 {0x4407, "Chelsio T420-SO"},
891 {0x4408, "Chelsio T420-CX"},
892 {0x4409, "Chelsio T420-BT"},
893 {0x440a, "Chelsio T404-BT"},
894 {0x440e, "Chelsio T440-LP-CR"},
895 }, t5_pciids[] = {
896 {0xb000, "Chelsio Terminator 5 FPGA"},
897 {0x5400, "Chelsio T580-dbg"},
898 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */
899 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */
900 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */
901 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */
902 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */
903 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */
904 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */
905 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */
906 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */
907 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */
908 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */
909 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */
910 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */
911 {0x5418, "Chelsio T540-BT"}, /* 4 x 10GBaseT */
912 {0x5419, "Chelsio T540-LP-BT"}, /* 4 x 10GBaseT */
913 {0x541a, "Chelsio T540-SO-BT"}, /* 4 x 10GBaseT, nomem */
914 {0x541b, "Chelsio T540-SO-CR"}, /* 4 x 10G, nomem */
915
916 /* Custom */
917 {0x5483, "Custom T540-CR"},
918 {0x5484, "Custom T540-BT"},
919 }, t6_pciids[] = {
920 {0xc006, "Chelsio Terminator 6 FPGA"}, /* T6 PE10K6 FPGA (PF0) */
921 {0x6400, "Chelsio T6-DBG-25"}, /* 2 x 10/25G, debug */
922 {0x6401, "Chelsio T6225-CR"}, /* 2 x 10/25G */
923 {0x6402, "Chelsio T6225-SO-CR"}, /* 2 x 10/25G, nomem */
924 {0x6403, "Chelsio T6425-CR"}, /* 4 x 10/25G */
925 {0x6404, "Chelsio T6425-SO-CR"}, /* 4 x 10/25G, nomem */
926 {0x6405, "Chelsio T6225-OCP-SO"}, /* 2 x 10/25G, nomem */
927 {0x6406, "Chelsio T62100-OCP-SO"}, /* 2 x 40/50/100G, nomem */
928 {0x6407, "Chelsio T62100-LP-CR"}, /* 2 x 40/50/100G */
929 {0x6408, "Chelsio T62100-SO-CR"}, /* 2 x 40/50/100G, nomem */
930 {0x6409, "Chelsio T6210-BT"}, /* 2 x 10GBASE-T */
931 {0x640d, "Chelsio T62100-CR"}, /* 2 x 40/50/100G */
932 {0x6410, "Chelsio T6-DBG-100"}, /* 2 x 40/50/100G, debug */
933 {0x6411, "Chelsio T6225-LL-CR"}, /* 2 x 10/25G */
934 {0x6414, "Chelsio T61100-OCP-SO"}, /* 1 x 40/50/100G, nomem */
935 {0x6415, "Chelsio T6201-BT"}, /* 2 x 1000BASE-T */
936
937 /* Custom */
938 {0x6480, "Custom T6225-CR"},
939 {0x6481, "Custom T62100-CR"},
940 {0x6482, "Custom T6225-CR"},
941 {0x6483, "Custom T62100-CR"},
942 {0x6484, "Custom T64100-CR"},
943 {0x6485, "Custom T6240-SO"},
944 {0x6486, "Custom T6225-SO-CR"},
945 {0x6487, "Custom T6225-CR"},
946 };
947
948 #ifdef TCP_OFFLOAD
949 /*
950 * service_iq_fl() has an iq and needs the fl. Offset of fl from the iq should
951 * be exactly the same for both rxq and ofld_rxq.
952 */
953 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
954 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
955 #endif
956 CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE);
957
958 static int
t4_probe(device_t dev)959 t4_probe(device_t dev)
960 {
961 int i;
962 uint16_t v = pci_get_vendor(dev);
963 uint16_t d = pci_get_device(dev);
964 uint8_t f = pci_get_function(dev);
965
966 if (v != PCI_VENDOR_ID_CHELSIO)
967 return (ENXIO);
968
969 /* Attach only to PF0 of the FPGA */
970 if (d == 0xa000 && f != 0)
971 return (ENXIO);
972
973 for (i = 0; i < nitems(t4_pciids); i++) {
974 if (d == t4_pciids[i].device) {
975 device_set_desc(dev, t4_pciids[i].desc);
976 return (BUS_PROBE_DEFAULT);
977 }
978 }
979
980 return (ENXIO);
981 }
982
983 static int
t5_probe(device_t dev)984 t5_probe(device_t dev)
985 {
986 int i;
987 uint16_t v = pci_get_vendor(dev);
988 uint16_t d = pci_get_device(dev);
989 uint8_t f = pci_get_function(dev);
990
991 if (v != PCI_VENDOR_ID_CHELSIO)
992 return (ENXIO);
993
994 /* Attach only to PF0 of the FPGA */
995 if (d == 0xb000 && f != 0)
996 return (ENXIO);
997
998 for (i = 0; i < nitems(t5_pciids); i++) {
999 if (d == t5_pciids[i].device) {
1000 device_set_desc(dev, t5_pciids[i].desc);
1001 return (BUS_PROBE_DEFAULT);
1002 }
1003 }
1004
1005 return (ENXIO);
1006 }
1007
1008 static int
t6_probe(device_t dev)1009 t6_probe(device_t dev)
1010 {
1011 int i;
1012 uint16_t v = pci_get_vendor(dev);
1013 uint16_t d = pci_get_device(dev);
1014
1015 if (v != PCI_VENDOR_ID_CHELSIO)
1016 return (ENXIO);
1017
1018 for (i = 0; i < nitems(t6_pciids); i++) {
1019 if (d == t6_pciids[i].device) {
1020 device_set_desc(dev, t6_pciids[i].desc);
1021 return (BUS_PROBE_DEFAULT);
1022 }
1023 }
1024
1025 return (ENXIO);
1026 }
1027
1028 static void
t5_attribute_workaround(device_t dev)1029 t5_attribute_workaround(device_t dev)
1030 {
1031 device_t root_port;
1032 uint32_t v;
1033
1034 /*
1035 * The T5 chips do not properly echo the No Snoop and Relaxed
1036 * Ordering attributes when replying to a TLP from a Root
1037 * Port. As a workaround, find the parent Root Port and
1038 * disable No Snoop and Relaxed Ordering. Note that this
1039 * affects all devices under this root port.
1040 */
1041 root_port = pci_find_pcie_root_port(dev);
1042 if (root_port == NULL) {
1043 device_printf(dev, "Unable to find parent root port\n");
1044 return;
1045 }
1046
1047 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL,
1048 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2);
1049 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) !=
1050 0)
1051 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n",
1052 device_get_nameunit(root_port));
1053 }
1054
1055 static const struct devnames devnames[] = {
1056 {
1057 .nexus_name = "t4nex",
1058 .ifnet_name = "cxgbe",
1059 .vi_ifnet_name = "vcxgbe",
1060 .pf03_drv_name = "t4iov",
1061 .vf_nexus_name = "t4vf",
1062 .vf_ifnet_name = "cxgbev"
1063 }, {
1064 .nexus_name = "t5nex",
1065 .ifnet_name = "cxl",
1066 .vi_ifnet_name = "vcxl",
1067 .pf03_drv_name = "t5iov",
1068 .vf_nexus_name = "t5vf",
1069 .vf_ifnet_name = "cxlv"
1070 }, {
1071 .nexus_name = "t6nex",
1072 .ifnet_name = "cc",
1073 .vi_ifnet_name = "vcc",
1074 .pf03_drv_name = "t6iov",
1075 .vf_nexus_name = "t6vf",
1076 .vf_ifnet_name = "ccv"
1077 }
1078 };
1079
1080 void
t4_init_devnames(struct adapter * sc)1081 t4_init_devnames(struct adapter *sc)
1082 {
1083 int id;
1084
1085 id = chip_id(sc);
1086 if (id >= CHELSIO_T4 && id - CHELSIO_T4 < nitems(devnames))
1087 sc->names = &devnames[id - CHELSIO_T4];
1088 else {
1089 device_printf(sc->dev, "chip id %d is not supported.\n", id);
1090 sc->names = NULL;
1091 }
1092 }
1093
1094 static int
t4_ifnet_unit(struct adapter * sc,struct port_info * pi)1095 t4_ifnet_unit(struct adapter *sc, struct port_info *pi)
1096 {
1097 const char *parent, *name;
1098 long value;
1099 int line, unit;
1100
1101 line = 0;
1102 parent = device_get_nameunit(sc->dev);
1103 name = sc->names->ifnet_name;
1104 while (resource_find_dev(&line, name, &unit, "at", parent) == 0) {
1105 if (resource_long_value(name, unit, "port", &value) == 0 &&
1106 value == pi->port_id)
1107 return (unit);
1108 }
1109 return (-1);
1110 }
1111
1112 static void
t4_calibration(void * arg)1113 t4_calibration(void *arg)
1114 {
1115 struct adapter *sc;
1116 struct clock_sync *cur, *nex;
1117 uint64_t hw;
1118 sbintime_t sbt;
1119 int next_up;
1120
1121 sc = (struct adapter *)arg;
1122
1123 KASSERT((hw_off_limits(sc) == 0), ("hw_off_limits at t4_calibration"));
1124 hw = t4_read_reg64(sc, A_SGE_TIMESTAMP_LO);
1125 sbt = sbinuptime();
1126
1127 cur = &sc->cal_info[sc->cal_current];
1128 next_up = (sc->cal_current + 1) % CNT_CAL_INFO;
1129 nex = &sc->cal_info[next_up];
1130 if (__predict_false(sc->cal_count == 0)) {
1131 /* First time in, just get the values in */
1132 cur->hw_cur = hw;
1133 cur->sbt_cur = sbt;
1134 sc->cal_count++;
1135 goto done;
1136 }
1137
1138 if (cur->hw_cur == hw) {
1139 /* The clock is not advancing? */
1140 sc->cal_count = 0;
1141 atomic_store_rel_int(&cur->gen, 0);
1142 goto done;
1143 }
1144
1145 seqc_write_begin(&nex->gen);
1146 nex->hw_prev = cur->hw_cur;
1147 nex->sbt_prev = cur->sbt_cur;
1148 nex->hw_cur = hw;
1149 nex->sbt_cur = sbt;
1150 seqc_write_end(&nex->gen);
1151 sc->cal_current = next_up;
1152 done:
1153 callout_reset_sbt_curcpu(&sc->cal_callout, SBT_1S, 0, t4_calibration,
1154 sc, C_DIRECT_EXEC);
1155 }
1156
1157 static void
t4_calibration_start(struct adapter * sc)1158 t4_calibration_start(struct adapter *sc)
1159 {
1160 /*
1161 * Here if we have not done a calibration
1162 * then do so otherwise start the appropriate
1163 * timer.
1164 */
1165 int i;
1166
1167 for (i = 0; i < CNT_CAL_INFO; i++) {
1168 sc->cal_info[i].gen = 0;
1169 }
1170 sc->cal_current = 0;
1171 sc->cal_count = 0;
1172 sc->cal_gen = 0;
1173 t4_calibration(sc);
1174 }
1175
1176 static int
t4_attach(device_t dev)1177 t4_attach(device_t dev)
1178 {
1179 struct adapter *sc;
1180 int rc = 0, i, j, rqidx, tqidx, nports;
1181 struct make_dev_args mda;
1182 struct intrs_and_queues iaq;
1183 struct sge *s;
1184 uint32_t *buf;
1185 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1186 int ofld_tqidx;
1187 #endif
1188 #ifdef TCP_OFFLOAD
1189 int ofld_rqidx;
1190 #endif
1191 #ifdef DEV_NETMAP
1192 int nm_rqidx, nm_tqidx;
1193 #endif
1194 int num_vis;
1195
1196 sc = device_get_softc(dev);
1197 sc->dev = dev;
1198 sysctl_ctx_init(&sc->ctx);
1199 TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
1200
1201 if ((pci_get_device(dev) & 0xff00) == 0x5400)
1202 t5_attribute_workaround(dev);
1203 pci_enable_busmaster(dev);
1204 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
1205 uint32_t v;
1206
1207 pci_set_max_read_req(dev, 4096);
1208 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
1209 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5);
1210 if (pcie_relaxed_ordering == 0 &&
1211 (v & PCIEM_CTL_RELAXED_ORD_ENABLE) != 0) {
1212 v &= ~PCIEM_CTL_RELAXED_ORD_ENABLE;
1213 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
1214 } else if (pcie_relaxed_ordering == 1 &&
1215 (v & PCIEM_CTL_RELAXED_ORD_ENABLE) == 0) {
1216 v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
1217 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
1218 }
1219 }
1220
1221 sc->sge_gts_reg = MYPF_REG(A_SGE_PF_GTS);
1222 sc->sge_kdoorbell_reg = MYPF_REG(A_SGE_PF_KDOORBELL);
1223 sc->traceq = -1;
1224 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF);
1225 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer",
1226 device_get_nameunit(dev));
1227
1228 snprintf(sc->lockname, sizeof(sc->lockname), "%s",
1229 device_get_nameunit(dev));
1230 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
1231 t4_add_adapter(sc);
1232
1233 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
1234 TAILQ_INIT(&sc->sfl);
1235 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
1236
1237 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
1238
1239 sc->policy = NULL;
1240 rw_init(&sc->policy_lock, "connection offload policy");
1241
1242 callout_init(&sc->ktls_tick, 1);
1243
1244 callout_init(&sc->cal_callout, 1);
1245
1246 refcount_init(&sc->vxlan_refcount, 0);
1247
1248 TASK_INIT(&sc->reset_task, 0, reset_adapter_task, sc);
1249 TASK_INIT(&sc->fatal_error_task, 0, fatal_error_task, sc);
1250
1251 sc->ctrlq_oid = SYSCTL_ADD_NODE(&sc->ctx,
1252 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "ctrlq",
1253 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "control queues");
1254 sc->fwq_oid = SYSCTL_ADD_NODE(&sc->ctx,
1255 SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)), OID_AUTO, "fwq",
1256 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "firmware event queue");
1257
1258 rc = t4_map_bars_0_and_4(sc);
1259 if (rc != 0)
1260 goto done; /* error message displayed already */
1261
1262 memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
1263
1264 /* Prepare the adapter for operation. */
1265 buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK);
1266 rc = -t4_prep_adapter(sc, buf);
1267 free(buf, M_CXGBE);
1268 if (rc != 0) {
1269 device_printf(dev, "failed to prepare adapter: %d.\n", rc);
1270 goto done;
1271 }
1272
1273 /*
1274 * This is the real PF# to which we're attaching. Works from within PCI
1275 * passthrough environments too, where pci_get_function() could return a
1276 * different PF# depending on the passthrough configuration. We need to
1277 * use the real PF# in all our communication with the firmware.
1278 */
1279 j = t4_read_reg(sc, A_PL_WHOAMI);
1280 sc->pf = chip_id(sc) <= CHELSIO_T5 ? G_SOURCEPF(j) : G_T6_SOURCEPF(j);
1281 sc->mbox = sc->pf;
1282
1283 t4_init_devnames(sc);
1284 if (sc->names == NULL) {
1285 rc = ENOTSUP;
1286 goto done; /* error message displayed already */
1287 }
1288
1289 /*
1290 * Do this really early, with the memory windows set up even before the
1291 * character device. The userland tool's register i/o and mem read
1292 * will work even in "recovery mode".
1293 */
1294 setup_memwin(sc);
1295 if (t4_init_devlog_params(sc, 0) == 0)
1296 fixup_devlog_params(sc);
1297 make_dev_args_init(&mda);
1298 mda.mda_devsw = &t4_cdevsw;
1299 mda.mda_uid = UID_ROOT;
1300 mda.mda_gid = GID_WHEEL;
1301 mda.mda_mode = 0600;
1302 mda.mda_si_drv1 = sc;
1303 rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
1304 if (rc != 0)
1305 device_printf(dev, "failed to create nexus char device: %d.\n",
1306 rc);
1307
1308 /* Go no further if recovery mode has been requested. */
1309 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
1310 device_printf(dev, "recovery mode.\n");
1311 goto done;
1312 }
1313
1314 #if defined(__i386__)
1315 if ((cpu_feature & CPUID_CX8) == 0) {
1316 device_printf(dev, "64 bit atomics not available.\n");
1317 rc = ENOTSUP;
1318 goto done;
1319 }
1320 #endif
1321
1322 /* Contact the firmware and try to become the master driver. */
1323 rc = contact_firmware(sc);
1324 if (rc != 0)
1325 goto done; /* error message displayed already */
1326 MPASS(sc->flags & FW_OK);
1327
1328 rc = get_params__pre_init(sc);
1329 if (rc != 0)
1330 goto done; /* error message displayed already */
1331
1332 if (sc->flags & MASTER_PF) {
1333 rc = partition_resources(sc);
1334 if (rc != 0)
1335 goto done; /* error message displayed already */
1336 }
1337
1338 rc = get_params__post_init(sc);
1339 if (rc != 0)
1340 goto done; /* error message displayed already */
1341
1342 rc = set_params__post_init(sc);
1343 if (rc != 0)
1344 goto done; /* error message displayed already */
1345
1346 rc = t4_map_bar_2(sc);
1347 if (rc != 0)
1348 goto done; /* error message displayed already */
1349
1350 rc = t4_create_dma_tag(sc);
1351 if (rc != 0)
1352 goto done; /* error message displayed already */
1353
1354 /*
1355 * First pass over all the ports - allocate VIs and initialize some
1356 * basic parameters like mac address, port type, etc.
1357 */
1358 for_each_port(sc, i) {
1359 struct port_info *pi;
1360
1361 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
1362 sc->port[i] = pi;
1363
1364 /* These must be set before t4_port_init */
1365 pi->adapter = sc;
1366 pi->port_id = i;
1367 /*
1368 * XXX: vi[0] is special so we can't delay this allocation until
1369 * pi->nvi's final value is known.
1370 */
1371 pi->vi = malloc(sizeof(struct vi_info) * t4_num_vis, M_CXGBE,
1372 M_ZERO | M_WAITOK);
1373
1374 /*
1375 * Allocate the "main" VI and initialize parameters
1376 * like mac addr.
1377 */
1378 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
1379 if (rc != 0) {
1380 device_printf(dev, "unable to initialize port %d: %d\n",
1381 i, rc);
1382 free(pi->vi, M_CXGBE);
1383 free(pi, M_CXGBE);
1384 sc->port[i] = NULL;
1385 goto done;
1386 }
1387
1388 if (is_bt(pi->port_type))
1389 setbit(&sc->bt_map, pi->tx_chan);
1390 else
1391 MPASS(!isset(&sc->bt_map, pi->tx_chan));
1392
1393 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
1394 device_get_nameunit(dev), i);
1395 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
1396 sc->chan_map[pi->tx_chan] = i;
1397
1398 /*
1399 * The MPS counter for FCS errors doesn't work correctly on the
1400 * T6 so we use the MAC counter here. Which MAC is in use
1401 * depends on the link settings which will be known when the
1402 * link comes up.
1403 */
1404 if (is_t6(sc))
1405 pi->fcs_reg = -1;
1406 else {
1407 pi->fcs_reg = t4_port_reg(sc, pi->tx_chan,
1408 A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L);
1409 }
1410 pi->fcs_base = 0;
1411
1412 /* All VIs on this port share this media. */
1413 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
1414 cxgbe_media_status);
1415
1416 PORT_LOCK(pi);
1417 init_link_config(pi);
1418 fixup_link_config(pi);
1419 build_medialist(pi);
1420 if (fixed_ifmedia(pi))
1421 pi->flags |= FIXED_IFMEDIA;
1422 PORT_UNLOCK(pi);
1423
1424 pi->dev = device_add_child(dev, sc->names->ifnet_name,
1425 t4_ifnet_unit(sc, pi));
1426 if (pi->dev == NULL) {
1427 device_printf(dev,
1428 "failed to add device for port %d.\n", i);
1429 rc = ENXIO;
1430 goto done;
1431 }
1432 pi->vi[0].dev = pi->dev;
1433 device_set_softc(pi->dev, pi);
1434 }
1435
1436 /*
1437 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
1438 */
1439 nports = sc->params.nports;
1440 rc = cfg_itype_and_nqueues(sc, &iaq);
1441 if (rc != 0)
1442 goto done; /* error message displayed already */
1443
1444 num_vis = iaq.num_vis;
1445 sc->intr_type = iaq.intr_type;
1446 sc->intr_count = iaq.nirq;
1447
1448 s = &sc->sge;
1449 s->nrxq = nports * iaq.nrxq;
1450 s->ntxq = nports * iaq.ntxq;
1451 if (num_vis > 1) {
1452 s->nrxq += nports * (num_vis - 1) * iaq.nrxq_vi;
1453 s->ntxq += nports * (num_vis - 1) * iaq.ntxq_vi;
1454 }
1455 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
1456 s->neq += nports; /* ctrl queues: 1 per port */
1457 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
1458 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1459 if (is_offload(sc) || is_ethoffload(sc)) {
1460 s->nofldtxq = nports * iaq.nofldtxq;
1461 if (num_vis > 1)
1462 s->nofldtxq += nports * (num_vis - 1) * iaq.nofldtxq_vi;
1463 s->neq += s->nofldtxq;
1464
1465 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_ofld_txq),
1466 M_CXGBE, M_ZERO | M_WAITOK);
1467 }
1468 #endif
1469 #ifdef TCP_OFFLOAD
1470 if (is_offload(sc)) {
1471 s->nofldrxq = nports * iaq.nofldrxq;
1472 if (num_vis > 1)
1473 s->nofldrxq += nports * (num_vis - 1) * iaq.nofldrxq_vi;
1474 s->neq += s->nofldrxq; /* free list */
1475 s->niq += s->nofldrxq;
1476
1477 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
1478 M_CXGBE, M_ZERO | M_WAITOK);
1479 }
1480 #endif
1481 #ifdef DEV_NETMAP
1482 s->nnmrxq = 0;
1483 s->nnmtxq = 0;
1484 if (t4_native_netmap & NN_MAIN_VI) {
1485 s->nnmrxq += nports * iaq.nnmrxq;
1486 s->nnmtxq += nports * iaq.nnmtxq;
1487 }
1488 if (num_vis > 1 && t4_native_netmap & NN_EXTRA_VI) {
1489 s->nnmrxq += nports * (num_vis - 1) * iaq.nnmrxq_vi;
1490 s->nnmtxq += nports * (num_vis - 1) * iaq.nnmtxq_vi;
1491 }
1492 s->neq += s->nnmtxq + s->nnmrxq;
1493 s->niq += s->nnmrxq;
1494
1495 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq),
1496 M_CXGBE, M_ZERO | M_WAITOK);
1497 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq),
1498 M_CXGBE, M_ZERO | M_WAITOK);
1499 #endif
1500 MPASS(s->niq <= s->iqmap_sz);
1501 MPASS(s->neq <= s->eqmap_sz);
1502
1503 s->ctrlq = malloc(nports * sizeof(struct sge_wrq), M_CXGBE,
1504 M_ZERO | M_WAITOK);
1505 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
1506 M_ZERO | M_WAITOK);
1507 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
1508 M_ZERO | M_WAITOK);
1509 s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE,
1510 M_ZERO | M_WAITOK);
1511 s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE,
1512 M_ZERO | M_WAITOK);
1513
1514 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
1515 M_ZERO | M_WAITOK);
1516
1517 t4_init_l2t(sc, M_WAITOK);
1518 t4_init_smt(sc, M_WAITOK);
1519 t4_init_tx_sched(sc);
1520 t4_init_atid_table(sc);
1521 #ifdef RATELIMIT
1522 t4_init_etid_table(sc);
1523 #endif
1524 #ifdef INET6
1525 t4_init_clip_table(sc);
1526 #endif
1527 if (sc->vres.key.size != 0)
1528 sc->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
1529 sc->vres.key.size, 32, 0, M_FIRSTFIT | M_WAITOK);
1530
1531 /*
1532 * Second pass over the ports. This time we know the number of rx and
1533 * tx queues that each port should get.
1534 */
1535 rqidx = tqidx = 0;
1536 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1537 ofld_tqidx = 0;
1538 #endif
1539 #ifdef TCP_OFFLOAD
1540 ofld_rqidx = 0;
1541 #endif
1542 #ifdef DEV_NETMAP
1543 nm_rqidx = nm_tqidx = 0;
1544 #endif
1545 for_each_port(sc, i) {
1546 struct port_info *pi = sc->port[i];
1547 struct vi_info *vi;
1548
1549 if (pi == NULL)
1550 continue;
1551
1552 pi->nvi = num_vis;
1553 for_each_vi(pi, j, vi) {
1554 vi->pi = pi;
1555 vi->adapter = sc;
1556 vi->first_intr = -1;
1557 vi->qsize_rxq = t4_qsize_rxq;
1558 vi->qsize_txq = t4_qsize_txq;
1559
1560 vi->first_rxq = rqidx;
1561 vi->first_txq = tqidx;
1562 vi->tmr_idx = t4_tmr_idx;
1563 vi->pktc_idx = t4_pktc_idx;
1564 vi->nrxq = j == 0 ? iaq.nrxq : iaq.nrxq_vi;
1565 vi->ntxq = j == 0 ? iaq.ntxq : iaq.ntxq_vi;
1566
1567 rqidx += vi->nrxq;
1568 tqidx += vi->ntxq;
1569
1570 if (j == 0 && vi->ntxq > 1)
1571 vi->rsrv_noflowq = t4_rsrv_noflowq ? 1 : 0;
1572 else
1573 vi->rsrv_noflowq = 0;
1574
1575 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1576 vi->first_ofld_txq = ofld_tqidx;
1577 vi->nofldtxq = j == 0 ? iaq.nofldtxq : iaq.nofldtxq_vi;
1578 ofld_tqidx += vi->nofldtxq;
1579 #endif
1580 #ifdef TCP_OFFLOAD
1581 vi->ofld_tmr_idx = t4_tmr_idx_ofld;
1582 vi->ofld_pktc_idx = t4_pktc_idx_ofld;
1583 vi->first_ofld_rxq = ofld_rqidx;
1584 vi->nofldrxq = j == 0 ? iaq.nofldrxq : iaq.nofldrxq_vi;
1585
1586 ofld_rqidx += vi->nofldrxq;
1587 #endif
1588 #ifdef DEV_NETMAP
1589 vi->first_nm_rxq = nm_rqidx;
1590 vi->first_nm_txq = nm_tqidx;
1591 if (j == 0) {
1592 vi->nnmrxq = iaq.nnmrxq;
1593 vi->nnmtxq = iaq.nnmtxq;
1594 } else {
1595 vi->nnmrxq = iaq.nnmrxq_vi;
1596 vi->nnmtxq = iaq.nnmtxq_vi;
1597 }
1598 nm_rqidx += vi->nnmrxq;
1599 nm_tqidx += vi->nnmtxq;
1600 #endif
1601 }
1602 }
1603
1604 rc = t4_setup_intr_handlers(sc);
1605 if (rc != 0) {
1606 device_printf(dev,
1607 "failed to setup interrupt handlers: %d\n", rc);
1608 goto done;
1609 }
1610
1611 rc = bus_generic_probe(dev);
1612 if (rc != 0) {
1613 device_printf(dev, "failed to probe child drivers: %d\n", rc);
1614 goto done;
1615 }
1616
1617 /*
1618 * Ensure thread-safe mailbox access (in debug builds).
1619 *
1620 * So far this was the only thread accessing the mailbox but various
1621 * ifnets and sysctls are about to be created and their handlers/ioctls
1622 * will access the mailbox from different threads.
1623 */
1624 sc->flags |= CHK_MBOX_ACCESS;
1625
1626 rc = bus_generic_attach(dev);
1627 if (rc != 0) {
1628 device_printf(dev,
1629 "failed to attach all child ports: %d\n", rc);
1630 goto done;
1631 }
1632 t4_calibration_start(sc);
1633
1634 device_printf(dev,
1635 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
1636 sc->params.pci.speed, sc->params.pci.width, sc->params.nports,
1637 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
1638 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
1639 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
1640
1641 t4_set_desc(sc);
1642
1643 notify_siblings(dev, 0);
1644
1645 done:
1646 if (rc != 0 && sc->cdev) {
1647 /* cdev was created and so cxgbetool works; recover that way. */
1648 device_printf(dev,
1649 "error during attach, adapter is now in recovery mode.\n");
1650 rc = 0;
1651 }
1652
1653 if (rc != 0)
1654 t4_detach_common(dev);
1655 else
1656 t4_sysctls(sc);
1657
1658 return (rc);
1659 }
1660
1661 static int
t4_child_location(device_t bus,device_t dev,struct sbuf * sb)1662 t4_child_location(device_t bus, device_t dev, struct sbuf *sb)
1663 {
1664 struct adapter *sc;
1665 struct port_info *pi;
1666 int i;
1667
1668 sc = device_get_softc(bus);
1669 for_each_port(sc, i) {
1670 pi = sc->port[i];
1671 if (pi != NULL && pi->dev == dev) {
1672 sbuf_printf(sb, "port=%d", pi->port_id);
1673 break;
1674 }
1675 }
1676 return (0);
1677 }
1678
1679 static int
t4_ready(device_t dev)1680 t4_ready(device_t dev)
1681 {
1682 struct adapter *sc;
1683
1684 sc = device_get_softc(dev);
1685 if (sc->flags & FW_OK)
1686 return (0);
1687 return (ENXIO);
1688 }
1689
1690 static int
t4_read_port_device(device_t dev,int port,device_t * child)1691 t4_read_port_device(device_t dev, int port, device_t *child)
1692 {
1693 struct adapter *sc;
1694 struct port_info *pi;
1695
1696 sc = device_get_softc(dev);
1697 if (port < 0 || port >= MAX_NPORTS)
1698 return (EINVAL);
1699 pi = sc->port[port];
1700 if (pi == NULL || pi->dev == NULL)
1701 return (ENXIO);
1702 *child = pi->dev;
1703 return (0);
1704 }
1705
1706 static int
notify_siblings(device_t dev,int detaching)1707 notify_siblings(device_t dev, int detaching)
1708 {
1709 device_t sibling;
1710 int error, i;
1711
1712 error = 0;
1713 for (i = 0; i < PCI_FUNCMAX; i++) {
1714 if (i == pci_get_function(dev))
1715 continue;
1716 sibling = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
1717 pci_get_slot(dev), i);
1718 if (sibling == NULL || !device_is_attached(sibling))
1719 continue;
1720 if (detaching)
1721 error = T4_DETACH_CHILD(sibling);
1722 else
1723 (void)T4_ATTACH_CHILD(sibling);
1724 if (error)
1725 break;
1726 }
1727 return (error);
1728 }
1729
1730 /*
1731 * Idempotent
1732 */
1733 static int
t4_detach(device_t dev)1734 t4_detach(device_t dev)
1735 {
1736 int rc;
1737
1738 rc = notify_siblings(dev, 1);
1739 if (rc) {
1740 device_printf(dev,
1741 "failed to detach sibling devices: %d\n", rc);
1742 return (rc);
1743 }
1744
1745 return (t4_detach_common(dev));
1746 }
1747
1748 int
t4_detach_common(device_t dev)1749 t4_detach_common(device_t dev)
1750 {
1751 struct adapter *sc;
1752 struct port_info *pi;
1753 int i, rc;
1754
1755 sc = device_get_softc(dev);
1756
1757 #ifdef TCP_OFFLOAD
1758 rc = t4_deactivate_all_uld(sc);
1759 if (rc) {
1760 device_printf(dev,
1761 "failed to detach upper layer drivers: %d\n", rc);
1762 return (rc);
1763 }
1764 #endif
1765
1766 if (sc->cdev) {
1767 destroy_dev(sc->cdev);
1768 sc->cdev = NULL;
1769 }
1770
1771 sx_xlock(&t4_list_lock);
1772 SLIST_REMOVE(&t4_list, sc, adapter, link);
1773 sx_xunlock(&t4_list_lock);
1774
1775 sc->flags &= ~CHK_MBOX_ACCESS;
1776 if (sc->flags & FULL_INIT_DONE) {
1777 if (!(sc->flags & IS_VF))
1778 t4_intr_disable(sc);
1779 }
1780
1781 if (device_is_attached(dev)) {
1782 rc = bus_generic_detach(dev);
1783 if (rc) {
1784 device_printf(dev,
1785 "failed to detach child devices: %d\n", rc);
1786 return (rc);
1787 }
1788 }
1789
1790 for (i = 0; i < sc->intr_count; i++)
1791 t4_free_irq(sc, &sc->irq[i]);
1792
1793 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1794 t4_free_tx_sched(sc);
1795
1796 for (i = 0; i < MAX_NPORTS; i++) {
1797 pi = sc->port[i];
1798 if (pi) {
1799 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid);
1800 if (pi->dev)
1801 device_delete_child(dev, pi->dev);
1802
1803 mtx_destroy(&pi->pi_lock);
1804 free(pi->vi, M_CXGBE);
1805 free(pi, M_CXGBE);
1806 }
1807 }
1808 callout_stop(&sc->cal_callout);
1809 callout_drain(&sc->cal_callout);
1810 device_delete_children(dev);
1811 sysctl_ctx_free(&sc->ctx);
1812 adapter_full_uninit(sc);
1813
1814 if ((sc->flags & (IS_VF | FW_OK)) == FW_OK)
1815 t4_fw_bye(sc, sc->mbox);
1816
1817 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
1818 pci_release_msi(dev);
1819
1820 if (sc->regs_res)
1821 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
1822 sc->regs_res);
1823
1824 if (sc->udbs_res)
1825 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid,
1826 sc->udbs_res);
1827
1828 if (sc->msix_res)
1829 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
1830 sc->msix_res);
1831
1832 if (sc->l2t)
1833 t4_free_l2t(sc->l2t);
1834 if (sc->smt)
1835 t4_free_smt(sc->smt);
1836 t4_free_atid_table(sc);
1837 #ifdef RATELIMIT
1838 t4_free_etid_table(sc);
1839 #endif
1840 if (sc->key_map)
1841 vmem_destroy(sc->key_map);
1842 #ifdef INET6
1843 t4_destroy_clip_table(sc);
1844 #endif
1845
1846 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1847 free(sc->sge.ofld_txq, M_CXGBE);
1848 #endif
1849 #ifdef TCP_OFFLOAD
1850 free(sc->sge.ofld_rxq, M_CXGBE);
1851 #endif
1852 #ifdef DEV_NETMAP
1853 free(sc->sge.nm_rxq, M_CXGBE);
1854 free(sc->sge.nm_txq, M_CXGBE);
1855 #endif
1856 free(sc->irq, M_CXGBE);
1857 free(sc->sge.rxq, M_CXGBE);
1858 free(sc->sge.txq, M_CXGBE);
1859 free(sc->sge.ctrlq, M_CXGBE);
1860 free(sc->sge.iqmap, M_CXGBE);
1861 free(sc->sge.eqmap, M_CXGBE);
1862 free(sc->tids.ftid_tab, M_CXGBE);
1863 free(sc->tids.hpftid_tab, M_CXGBE);
1864 free_hftid_hash(&sc->tids);
1865 free(sc->tids.tid_tab, M_CXGBE);
1866 t4_destroy_dma_tag(sc);
1867
1868 callout_drain(&sc->ktls_tick);
1869 callout_drain(&sc->sfl_callout);
1870 if (mtx_initialized(&sc->tids.ftid_lock)) {
1871 mtx_destroy(&sc->tids.ftid_lock);
1872 cv_destroy(&sc->tids.ftid_cv);
1873 }
1874 if (mtx_initialized(&sc->tids.atid_lock))
1875 mtx_destroy(&sc->tids.atid_lock);
1876 if (mtx_initialized(&sc->ifp_lock))
1877 mtx_destroy(&sc->ifp_lock);
1878
1879 if (rw_initialized(&sc->policy_lock)) {
1880 rw_destroy(&sc->policy_lock);
1881 #ifdef TCP_OFFLOAD
1882 if (sc->policy != NULL)
1883 free_offload_policy(sc->policy);
1884 #endif
1885 }
1886
1887 for (i = 0; i < NUM_MEMWIN; i++) {
1888 struct memwin *mw = &sc->memwin[i];
1889
1890 if (rw_initialized(&mw->mw_lock))
1891 rw_destroy(&mw->mw_lock);
1892 }
1893
1894 mtx_destroy(&sc->sfl_lock);
1895 mtx_destroy(&sc->reg_lock);
1896 mtx_destroy(&sc->sc_lock);
1897
1898 bzero(sc, sizeof(*sc));
1899
1900 return (0);
1901 }
1902
1903 static inline bool
ok_to_reset(struct adapter * sc)1904 ok_to_reset(struct adapter *sc)
1905 {
1906 struct tid_info *t = &sc->tids;
1907 struct port_info *pi;
1908 struct vi_info *vi;
1909 int i, j;
1910 int caps = IFCAP_TOE | IFCAP_NETMAP | IFCAP_TXRTLMT;
1911
1912 if (is_t6(sc))
1913 caps |= IFCAP_TXTLS;
1914
1915 ASSERT_SYNCHRONIZED_OP(sc);
1916 MPASS(!(sc->flags & IS_VF));
1917
1918 for_each_port(sc, i) {
1919 pi = sc->port[i];
1920 for_each_vi(pi, j, vi) {
1921 if (if_getcapenable(vi->ifp) & caps)
1922 return (false);
1923 }
1924 }
1925
1926 if (atomic_load_int(&t->tids_in_use) > 0)
1927 return (false);
1928 if (atomic_load_int(&t->stids_in_use) > 0)
1929 return (false);
1930 if (atomic_load_int(&t->atids_in_use) > 0)
1931 return (false);
1932 if (atomic_load_int(&t->ftids_in_use) > 0)
1933 return (false);
1934 if (atomic_load_int(&t->hpftids_in_use) > 0)
1935 return (false);
1936 if (atomic_load_int(&t->etids_in_use) > 0)
1937 return (false);
1938
1939 return (true);
1940 }
1941
1942 static inline int
stop_adapter(struct adapter * sc)1943 stop_adapter(struct adapter *sc)
1944 {
1945 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_STOPPED)))
1946 return (1); /* Already stopped. */
1947 return (t4_shutdown_adapter(sc));
1948 }
1949
1950 static int
t4_suspend(device_t dev)1951 t4_suspend(device_t dev)
1952 {
1953 struct adapter *sc = device_get_softc(dev);
1954 struct port_info *pi;
1955 struct vi_info *vi;
1956 if_t ifp;
1957 struct sge_rxq *rxq;
1958 struct sge_txq *txq;
1959 struct sge_wrq *wrq;
1960 #ifdef TCP_OFFLOAD
1961 struct sge_ofld_rxq *ofld_rxq;
1962 #endif
1963 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
1964 struct sge_ofld_txq *ofld_txq;
1965 #endif
1966 int rc, i, j, k;
1967
1968 CH_ALERT(sc, "suspend requested\n");
1969
1970 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4sus");
1971 if (rc != 0)
1972 return (ENXIO);
1973
1974 /* XXX: Can the kernel call suspend repeatedly without resume? */
1975 MPASS(!hw_off_limits(sc));
1976
1977 if (!ok_to_reset(sc)) {
1978 /* XXX: should list what resource is preventing suspend. */
1979 CH_ERR(sc, "not safe to suspend.\n");
1980 rc = EBUSY;
1981 goto done;
1982 }
1983
1984 /* No more DMA or interrupts. */
1985 stop_adapter(sc);
1986
1987 /* Quiesce all activity. */
1988 for_each_port(sc, i) {
1989 pi = sc->port[i];
1990 pi->vxlan_tcam_entry = false;
1991
1992 PORT_LOCK(pi);
1993 if (pi->up_vis > 0) {
1994 /*
1995 * t4_shutdown_adapter has already shut down all the
1996 * PHYs but it also disables interrupts and DMA so there
1997 * won't be a link interrupt. So we update the state
1998 * manually and inform the kernel.
1999 */
2000 pi->link_cfg.link_ok = false;
2001 t4_os_link_changed(pi);
2002 }
2003 PORT_UNLOCK(pi);
2004
2005 for_each_vi(pi, j, vi) {
2006 vi->xact_addr_filt = -1;
2007 mtx_lock(&vi->tick_mtx);
2008 vi->flags |= VI_SKIP_STATS;
2009 mtx_unlock(&vi->tick_mtx);
2010 if (!(vi->flags & VI_INIT_DONE))
2011 continue;
2012
2013 ifp = vi->ifp;
2014 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
2015 mtx_lock(&vi->tick_mtx);
2016 callout_stop(&vi->tick);
2017 mtx_unlock(&vi->tick_mtx);
2018 callout_drain(&vi->tick);
2019 }
2020
2021 /*
2022 * Note that the HW is not available.
2023 */
2024 for_each_txq(vi, k, txq) {
2025 TXQ_LOCK(txq);
2026 txq->eq.flags &= ~(EQ_ENABLED | EQ_HW_ALLOCATED);
2027 TXQ_UNLOCK(txq);
2028 }
2029 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
2030 for_each_ofld_txq(vi, k, ofld_txq) {
2031 ofld_txq->wrq.eq.flags &= ~EQ_HW_ALLOCATED;
2032 }
2033 #endif
2034 for_each_rxq(vi, k, rxq) {
2035 rxq->iq.flags &= ~IQ_HW_ALLOCATED;
2036 }
2037 #if defined(TCP_OFFLOAD)
2038 for_each_ofld_rxq(vi, k, ofld_rxq) {
2039 ofld_rxq->iq.flags &= ~IQ_HW_ALLOCATED;
2040 }
2041 #endif
2042
2043 quiesce_vi(vi);
2044 }
2045
2046 if (sc->flags & FULL_INIT_DONE) {
2047 /* Control queue */
2048 wrq = &sc->sge.ctrlq[i];
2049 wrq->eq.flags &= ~EQ_HW_ALLOCATED;
2050 quiesce_wrq(wrq);
2051 }
2052 }
2053 if (sc->flags & FULL_INIT_DONE) {
2054 /* Firmware event queue */
2055 sc->sge.fwq.flags &= ~IQ_HW_ALLOCATED;
2056 quiesce_iq_fl(sc, &sc->sge.fwq, NULL);
2057 }
2058
2059 /* Stop calibration */
2060 callout_stop(&sc->cal_callout);
2061 callout_drain(&sc->cal_callout);
2062
2063 /* Mark the adapter totally off limits. */
2064 mtx_lock(&sc->reg_lock);
2065 atomic_set_int(&sc->error_flags, HW_OFF_LIMITS);
2066 sc->flags &= ~(FW_OK | MASTER_PF);
2067 sc->reset_thread = NULL;
2068 mtx_unlock(&sc->reg_lock);
2069
2070 if (t4_clock_gate_on_suspend) {
2071 t4_set_reg_field(sc, A_PMU_PART_CG_PWRMODE, F_MA_PART_CGEN |
2072 F_LE_PART_CGEN | F_EDC1_PART_CGEN | F_EDC0_PART_CGEN |
2073 F_TP_PART_CGEN | F_PDP_PART_CGEN | F_SGE_PART_CGEN, 0);
2074 }
2075
2076 CH_ALERT(sc, "suspend completed.\n");
2077 done:
2078 end_synchronized_op(sc, 0);
2079 return (rc);
2080 }
2081
2082 struct adapter_pre_reset_state {
2083 u_int flags;
2084 uint16_t nbmcaps;
2085 uint16_t linkcaps;
2086 uint16_t switchcaps;
2087 uint16_t niccaps;
2088 uint16_t toecaps;
2089 uint16_t rdmacaps;
2090 uint16_t cryptocaps;
2091 uint16_t iscsicaps;
2092 uint16_t fcoecaps;
2093
2094 u_int cfcsum;
2095 char cfg_file[32];
2096
2097 struct adapter_params params;
2098 struct t4_virt_res vres;
2099 struct tid_info tids;
2100 struct sge sge;
2101
2102 int rawf_base;
2103 int nrawf;
2104
2105 };
2106
2107 static void
save_caps_and_params(struct adapter * sc,struct adapter_pre_reset_state * o)2108 save_caps_and_params(struct adapter *sc, struct adapter_pre_reset_state *o)
2109 {
2110
2111 ASSERT_SYNCHRONIZED_OP(sc);
2112
2113 o->flags = sc->flags;
2114
2115 o->nbmcaps = sc->nbmcaps;
2116 o->linkcaps = sc->linkcaps;
2117 o->switchcaps = sc->switchcaps;
2118 o->niccaps = sc->niccaps;
2119 o->toecaps = sc->toecaps;
2120 o->rdmacaps = sc->rdmacaps;
2121 o->cryptocaps = sc->cryptocaps;
2122 o->iscsicaps = sc->iscsicaps;
2123 o->fcoecaps = sc->fcoecaps;
2124
2125 o->cfcsum = sc->cfcsum;
2126 MPASS(sizeof(o->cfg_file) == sizeof(sc->cfg_file));
2127 memcpy(o->cfg_file, sc->cfg_file, sizeof(o->cfg_file));
2128
2129 o->params = sc->params;
2130 o->vres = sc->vres;
2131 o->tids = sc->tids;
2132 o->sge = sc->sge;
2133
2134 o->rawf_base = sc->rawf_base;
2135 o->nrawf = sc->nrawf;
2136 }
2137
2138 static int
compare_caps_and_params(struct adapter * sc,struct adapter_pre_reset_state * o)2139 compare_caps_and_params(struct adapter *sc, struct adapter_pre_reset_state *o)
2140 {
2141 int rc = 0;
2142
2143 ASSERT_SYNCHRONIZED_OP(sc);
2144
2145 /* Capabilities */
2146 #define COMPARE_CAPS(c) do { \
2147 if (o->c##caps != sc->c##caps) { \
2148 CH_ERR(sc, "%scaps 0x%04x -> 0x%04x.\n", #c, o->c##caps, \
2149 sc->c##caps); \
2150 rc = EINVAL; \
2151 } \
2152 } while (0)
2153 COMPARE_CAPS(nbm);
2154 COMPARE_CAPS(link);
2155 COMPARE_CAPS(switch);
2156 COMPARE_CAPS(nic);
2157 COMPARE_CAPS(toe);
2158 COMPARE_CAPS(rdma);
2159 COMPARE_CAPS(crypto);
2160 COMPARE_CAPS(iscsi);
2161 COMPARE_CAPS(fcoe);
2162 #undef COMPARE_CAPS
2163
2164 /* Firmware config file */
2165 if (o->cfcsum != sc->cfcsum) {
2166 CH_ERR(sc, "config file %s (0x%x) -> %s (0x%x)\n", o->cfg_file,
2167 o->cfcsum, sc->cfg_file, sc->cfcsum);
2168 rc = EINVAL;
2169 }
2170
2171 #define COMPARE_PARAM(p, name) do { \
2172 if (o->p != sc->p) { \
2173 CH_ERR(sc, #name " %d -> %d\n", o->p, sc->p); \
2174 rc = EINVAL; \
2175 } \
2176 } while (0)
2177 COMPARE_PARAM(sge.iq_start, iq_start);
2178 COMPARE_PARAM(sge.eq_start, eq_start);
2179 COMPARE_PARAM(tids.ftid_base, ftid_base);
2180 COMPARE_PARAM(tids.ftid_end, ftid_end);
2181 COMPARE_PARAM(tids.nftids, nftids);
2182 COMPARE_PARAM(vres.l2t.start, l2t_start);
2183 COMPARE_PARAM(vres.l2t.size, l2t_size);
2184 COMPARE_PARAM(sge.iqmap_sz, iqmap_sz);
2185 COMPARE_PARAM(sge.eqmap_sz, eqmap_sz);
2186 COMPARE_PARAM(tids.tid_base, tid_base);
2187 COMPARE_PARAM(tids.hpftid_base, hpftid_base);
2188 COMPARE_PARAM(tids.hpftid_end, hpftid_end);
2189 COMPARE_PARAM(tids.nhpftids, nhpftids);
2190 COMPARE_PARAM(rawf_base, rawf_base);
2191 COMPARE_PARAM(nrawf, nrawf);
2192 COMPARE_PARAM(params.mps_bg_map, mps_bg_map);
2193 COMPARE_PARAM(params.filter2_wr_support, filter2_wr_support);
2194 COMPARE_PARAM(params.ulptx_memwrite_dsgl, ulptx_memwrite_dsgl);
2195 COMPARE_PARAM(params.fr_nsmr_tpte_wr_support, fr_nsmr_tpte_wr_support);
2196 COMPARE_PARAM(params.max_pkts_per_eth_tx_pkts_wr, max_pkts_per_eth_tx_pkts_wr);
2197 COMPARE_PARAM(tids.ntids, ntids);
2198 COMPARE_PARAM(tids.etid_base, etid_base);
2199 COMPARE_PARAM(tids.etid_end, etid_end);
2200 COMPARE_PARAM(tids.netids, netids);
2201 COMPARE_PARAM(params.eo_wr_cred, eo_wr_cred);
2202 COMPARE_PARAM(params.ethoffload, ethoffload);
2203 COMPARE_PARAM(tids.natids, natids);
2204 COMPARE_PARAM(tids.stid_base, stid_base);
2205 COMPARE_PARAM(vres.ddp.start, ddp_start);
2206 COMPARE_PARAM(vres.ddp.size, ddp_size);
2207 COMPARE_PARAM(params.ofldq_wr_cred, ofldq_wr_cred);
2208 COMPARE_PARAM(vres.stag.start, stag_start);
2209 COMPARE_PARAM(vres.stag.size, stag_size);
2210 COMPARE_PARAM(vres.rq.start, rq_start);
2211 COMPARE_PARAM(vres.rq.size, rq_size);
2212 COMPARE_PARAM(vres.pbl.start, pbl_start);
2213 COMPARE_PARAM(vres.pbl.size, pbl_size);
2214 COMPARE_PARAM(vres.qp.start, qp_start);
2215 COMPARE_PARAM(vres.qp.size, qp_size);
2216 COMPARE_PARAM(vres.cq.start, cq_start);
2217 COMPARE_PARAM(vres.cq.size, cq_size);
2218 COMPARE_PARAM(vres.ocq.start, ocq_start);
2219 COMPARE_PARAM(vres.ocq.size, ocq_size);
2220 COMPARE_PARAM(vres.srq.start, srq_start);
2221 COMPARE_PARAM(vres.srq.size, srq_size);
2222 COMPARE_PARAM(params.max_ordird_qp, max_ordird_qp);
2223 COMPARE_PARAM(params.max_ird_adapter, max_ird_adapter);
2224 COMPARE_PARAM(vres.iscsi.start, iscsi_start);
2225 COMPARE_PARAM(vres.iscsi.size, iscsi_size);
2226 COMPARE_PARAM(vres.key.start, key_start);
2227 COMPARE_PARAM(vres.key.size, key_size);
2228 #undef COMPARE_PARAM
2229
2230 return (rc);
2231 }
2232
2233 static int
t4_resume(device_t dev)2234 t4_resume(device_t dev)
2235 {
2236 struct adapter *sc = device_get_softc(dev);
2237 struct adapter_pre_reset_state *old_state = NULL;
2238 struct port_info *pi;
2239 struct vi_info *vi;
2240 if_t ifp;
2241 struct sge_txq *txq;
2242 int rc, i, j, k;
2243
2244 CH_ALERT(sc, "resume requested.\n");
2245
2246 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4res");
2247 if (rc != 0)
2248 return (ENXIO);
2249 MPASS(hw_off_limits(sc));
2250 MPASS((sc->flags & FW_OK) == 0);
2251 MPASS((sc->flags & MASTER_PF) == 0);
2252 MPASS(sc->reset_thread == NULL);
2253 sc->reset_thread = curthread;
2254
2255 /* Register access is expected to work by the time we're here. */
2256 if (t4_read_reg(sc, A_PL_WHOAMI) == 0xffffffff) {
2257 CH_ERR(sc, "%s: can't read device registers\n", __func__);
2258 rc = ENXIO;
2259 goto done;
2260 }
2261
2262 /* Note that HW_OFF_LIMITS is cleared a bit later. */
2263 atomic_clear_int(&sc->error_flags, ADAP_FATAL_ERR | ADAP_STOPPED);
2264
2265 /* Restore memory window. */
2266 setup_memwin(sc);
2267
2268 /* Go no further if recovery mode has been requested. */
2269 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
2270 CH_ALERT(sc, "recovery mode on resume.\n");
2271 rc = 0;
2272 mtx_lock(&sc->reg_lock);
2273 atomic_clear_int(&sc->error_flags, HW_OFF_LIMITS);
2274 mtx_unlock(&sc->reg_lock);
2275 goto done;
2276 }
2277
2278 old_state = malloc(sizeof(*old_state), M_CXGBE, M_ZERO | M_WAITOK);
2279 save_caps_and_params(sc, old_state);
2280
2281 /* Reestablish contact with firmware and become the primary PF. */
2282 rc = contact_firmware(sc);
2283 if (rc != 0)
2284 goto done; /* error message displayed already */
2285 MPASS(sc->flags & FW_OK);
2286
2287 if (sc->flags & MASTER_PF) {
2288 rc = partition_resources(sc);
2289 if (rc != 0)
2290 goto done; /* error message displayed already */
2291 }
2292
2293 rc = get_params__post_init(sc);
2294 if (rc != 0)
2295 goto done; /* error message displayed already */
2296
2297 rc = set_params__post_init(sc);
2298 if (rc != 0)
2299 goto done; /* error message displayed already */
2300
2301 rc = compare_caps_and_params(sc, old_state);
2302 if (rc != 0)
2303 goto done; /* error message displayed already */
2304
2305 for_each_port(sc, i) {
2306 pi = sc->port[i];
2307 MPASS(pi != NULL);
2308 MPASS(pi->vi != NULL);
2309 MPASS(pi->vi[0].dev == pi->dev);
2310
2311 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
2312 if (rc != 0) {
2313 CH_ERR(sc,
2314 "failed to re-initialize port %d: %d\n", i, rc);
2315 goto done;
2316 }
2317 MPASS(sc->chan_map[pi->tx_chan] == i);
2318
2319 PORT_LOCK(pi);
2320 fixup_link_config(pi);
2321 build_medialist(pi);
2322 PORT_UNLOCK(pi);
2323 for_each_vi(pi, j, vi) {
2324 if (IS_MAIN_VI(vi))
2325 continue;
2326 rc = alloc_extra_vi(sc, pi, vi);
2327 if (rc != 0) {
2328 CH_ERR(vi,
2329 "failed to re-allocate extra VI: %d\n", rc);
2330 goto done;
2331 }
2332 }
2333 }
2334
2335 /*
2336 * Interrupts and queues are about to be enabled and other threads will
2337 * want to access the hardware too. It is safe to do so. Note that
2338 * this thread is still in the middle of a synchronized_op.
2339 */
2340 mtx_lock(&sc->reg_lock);
2341 atomic_clear_int(&sc->error_flags, HW_OFF_LIMITS);
2342 mtx_unlock(&sc->reg_lock);
2343
2344 if (sc->flags & FULL_INIT_DONE) {
2345 rc = adapter_full_init(sc);
2346 if (rc != 0) {
2347 CH_ERR(sc, "failed to re-initialize adapter: %d\n", rc);
2348 goto done;
2349 }
2350
2351 if (sc->vxlan_refcount > 0)
2352 enable_vxlan_rx(sc);
2353
2354 for_each_port(sc, i) {
2355 pi = sc->port[i];
2356 for_each_vi(pi, j, vi) {
2357 mtx_lock(&vi->tick_mtx);
2358 vi->flags &= ~VI_SKIP_STATS;
2359 mtx_unlock(&vi->tick_mtx);
2360 if (!(vi->flags & VI_INIT_DONE))
2361 continue;
2362 rc = vi_full_init(vi);
2363 if (rc != 0) {
2364 CH_ERR(vi, "failed to re-initialize "
2365 "interface: %d\n", rc);
2366 goto done;
2367 }
2368
2369 ifp = vi->ifp;
2370 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
2371 continue;
2372 /*
2373 * Note that we do not setup multicast addresses
2374 * in the first pass. This ensures that the
2375 * unicast DMACs for all VIs on all ports get an
2376 * MPS TCAM entry.
2377 */
2378 rc = update_mac_settings(ifp, XGMAC_ALL &
2379 ~XGMAC_MCADDRS);
2380 if (rc != 0) {
2381 CH_ERR(vi, "failed to re-configure MAC: %d\n", rc);
2382 goto done;
2383 }
2384 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true,
2385 true);
2386 if (rc != 0) {
2387 CH_ERR(vi, "failed to re-enable VI: %d\n", rc);
2388 goto done;
2389 }
2390 for_each_txq(vi, k, txq) {
2391 TXQ_LOCK(txq);
2392 txq->eq.flags |= EQ_ENABLED;
2393 TXQ_UNLOCK(txq);
2394 }
2395 mtx_lock(&vi->tick_mtx);
2396 callout_schedule(&vi->tick, hz);
2397 mtx_unlock(&vi->tick_mtx);
2398 }
2399 PORT_LOCK(pi);
2400 if (pi->up_vis > 0) {
2401 t4_update_port_info(pi);
2402 fixup_link_config(pi);
2403 build_medialist(pi);
2404 apply_link_config(pi);
2405 if (pi->link_cfg.link_ok)
2406 t4_os_link_changed(pi);
2407 }
2408 PORT_UNLOCK(pi);
2409 }
2410
2411 /* Now reprogram the L2 multicast addresses. */
2412 for_each_port(sc, i) {
2413 pi = sc->port[i];
2414 for_each_vi(pi, j, vi) {
2415 if (!(vi->flags & VI_INIT_DONE))
2416 continue;
2417 ifp = vi->ifp;
2418 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING))
2419 continue;
2420 rc = update_mac_settings(ifp, XGMAC_MCADDRS);
2421 if (rc != 0) {
2422 CH_ERR(vi, "failed to re-configure MCAST MACs: %d\n", rc);
2423 rc = 0; /* carry on */
2424 }
2425 }
2426 }
2427 }
2428
2429 /* Reset all calibration */
2430 t4_calibration_start(sc);
2431
2432 done:
2433 if (rc == 0) {
2434 sc->incarnation++;
2435 CH_ALERT(sc, "resume completed.\n");
2436 }
2437 end_synchronized_op(sc, 0);
2438 free(old_state, M_CXGBE);
2439 return (rc);
2440 }
2441
2442 static int
t4_reset_prepare(device_t dev,device_t child)2443 t4_reset_prepare(device_t dev, device_t child)
2444 {
2445 struct adapter *sc = device_get_softc(dev);
2446
2447 CH_ALERT(sc, "reset_prepare.\n");
2448 return (0);
2449 }
2450
2451 static int
t4_reset_post(device_t dev,device_t child)2452 t4_reset_post(device_t dev, device_t child)
2453 {
2454 struct adapter *sc = device_get_softc(dev);
2455
2456 CH_ALERT(sc, "reset_post.\n");
2457 return (0);
2458 }
2459
2460 static int
reset_adapter(struct adapter * sc)2461 reset_adapter(struct adapter *sc)
2462 {
2463 int rc, oldinc, error_flags;
2464
2465 CH_ALERT(sc, "reset requested.\n");
2466
2467 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4rst1");
2468 if (rc != 0)
2469 return (EBUSY);
2470
2471 if (hw_off_limits(sc)) {
2472 CH_ERR(sc, "adapter is suspended, use resume (not reset).\n");
2473 rc = ENXIO;
2474 goto done;
2475 }
2476
2477 if (!ok_to_reset(sc)) {
2478 /* XXX: should list what resource is preventing reset. */
2479 CH_ERR(sc, "not safe to reset.\n");
2480 rc = EBUSY;
2481 goto done;
2482 }
2483
2484 done:
2485 oldinc = sc->incarnation;
2486 end_synchronized_op(sc, 0);
2487 if (rc != 0)
2488 return (rc); /* Error logged already. */
2489
2490 atomic_add_int(&sc->num_resets, 1);
2491 mtx_lock(&Giant);
2492 rc = BUS_RESET_CHILD(device_get_parent(sc->dev), sc->dev, 0);
2493 mtx_unlock(&Giant);
2494 if (rc != 0)
2495 CH_ERR(sc, "bus_reset_child failed: %d.\n", rc);
2496 else {
2497 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4rst2");
2498 if (rc != 0)
2499 return (EBUSY);
2500 error_flags = atomic_load_int(&sc->error_flags);
2501 if (sc->incarnation > oldinc && error_flags == 0) {
2502 CH_ALERT(sc, "bus_reset_child succeeded.\n");
2503 } else {
2504 CH_ERR(sc, "adapter did not reset properly, flags "
2505 "0x%08x, error_flags 0x%08x.\n", sc->flags,
2506 error_flags);
2507 rc = ENXIO;
2508 }
2509 end_synchronized_op(sc, 0);
2510 }
2511
2512 return (rc);
2513 }
2514
2515 static void
reset_adapter_task(void * arg,int pending)2516 reset_adapter_task(void *arg, int pending)
2517 {
2518 /* XXX: t4_async_event here? */
2519 reset_adapter(arg);
2520 }
2521
2522 static int
cxgbe_probe(device_t dev)2523 cxgbe_probe(device_t dev)
2524 {
2525 struct port_info *pi = device_get_softc(dev);
2526
2527 device_set_descf(dev, "port %d", pi->port_id);
2528
2529 return (BUS_PROBE_DEFAULT);
2530 }
2531
2532 #define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
2533 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
2534 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS | \
2535 IFCAP_HWRXTSTMP | IFCAP_MEXTPG)
2536 #define T4_CAP_ENABLE (T4_CAP)
2537
2538 static int
cxgbe_vi_attach(device_t dev,struct vi_info * vi)2539 cxgbe_vi_attach(device_t dev, struct vi_info *vi)
2540 {
2541 if_t ifp;
2542 struct sbuf *sb;
2543 struct sysctl_ctx_list *ctx = &vi->ctx;
2544 struct sysctl_oid_list *children;
2545 struct pfil_head_args pa;
2546 struct adapter *sc = vi->adapter;
2547
2548 sysctl_ctx_init(ctx);
2549 children = SYSCTL_CHILDREN(device_get_sysctl_tree(vi->dev));
2550 vi->rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rxq",
2551 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NIC rx queues");
2552 vi->txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "txq",
2553 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "NIC tx queues");
2554 #ifdef DEV_NETMAP
2555 vi->nm_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_rxq",
2556 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap rx queues");
2557 vi->nm_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "nm_txq",
2558 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "netmap tx queues");
2559 #endif
2560 #ifdef TCP_OFFLOAD
2561 vi->ofld_rxq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_rxq",
2562 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE rx queues");
2563 #endif
2564 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
2565 vi->ofld_txq_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "ofld_txq",
2566 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE/ETHOFLD tx queues");
2567 #endif
2568
2569 vi->xact_addr_filt = -1;
2570 mtx_init(&vi->tick_mtx, "vi tick", NULL, MTX_DEF);
2571 callout_init_mtx(&vi->tick, &vi->tick_mtx, 0);
2572 if (sc->flags & IS_VF || t4_tx_vm_wr != 0)
2573 vi->flags |= TX_USES_VM_WR;
2574
2575 /* Allocate an ifnet and set it up */
2576 ifp = if_alloc_dev(IFT_ETHER, dev);
2577 if (ifp == NULL) {
2578 device_printf(dev, "Cannot allocate ifnet\n");
2579 return (ENOMEM);
2580 }
2581 vi->ifp = ifp;
2582 if_setsoftc(ifp, vi);
2583
2584 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2585 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
2586
2587 if_setinitfn(ifp, cxgbe_init);
2588 if_setioctlfn(ifp, cxgbe_ioctl);
2589 if_settransmitfn(ifp, cxgbe_transmit);
2590 if_setqflushfn(ifp, cxgbe_qflush);
2591 if (vi->pi->nvi > 1 || sc->flags & IS_VF)
2592 if_setgetcounterfn(ifp, vi_get_counter);
2593 else
2594 if_setgetcounterfn(ifp, cxgbe_get_counter);
2595 #if defined(KERN_TLS) || defined(RATELIMIT)
2596 if_setsndtagallocfn(ifp, cxgbe_snd_tag_alloc);
2597 #endif
2598 #ifdef RATELIMIT
2599 if_setratelimitqueryfn(ifp, cxgbe_ratelimit_query);
2600 #endif
2601
2602 if_setcapabilities(ifp, T4_CAP);
2603 if_setcapenable(ifp, T4_CAP_ENABLE);
2604 if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
2605 CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
2606 if (chip_id(sc) >= CHELSIO_T6) {
2607 if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
2608 if_setcapenablebit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
2609 if_sethwassistbits(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP |
2610 CSUM_INNER_IP6_TSO | CSUM_INNER_IP | CSUM_INNER_IP_UDP |
2611 CSUM_INNER_IP_TCP | CSUM_INNER_IP_TSO | CSUM_ENCAP_VXLAN, 0);
2612 }
2613
2614 #ifdef TCP_OFFLOAD
2615 if (vi->nofldrxq != 0)
2616 if_setcapabilitiesbit(ifp, IFCAP_TOE, 0);
2617 #endif
2618 #ifdef RATELIMIT
2619 if (is_ethoffload(sc) && vi->nofldtxq != 0) {
2620 if_setcapabilitiesbit(ifp, IFCAP_TXRTLMT, 0);
2621 if_setcapenablebit(ifp, IFCAP_TXRTLMT, 0);
2622 }
2623 #endif
2624
2625 if_sethwtsomax(ifp, IP_MAXPACKET);
2626 if (vi->flags & TX_USES_VM_WR)
2627 if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_VM_TSO);
2628 else
2629 if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_TSO);
2630 #ifdef RATELIMIT
2631 if (is_ethoffload(sc) && vi->nofldtxq != 0)
2632 if_sethwtsomaxsegcount(ifp, TX_SGL_SEGS_EO_TSO);
2633 #endif
2634 if_sethwtsomaxsegsize(ifp, 65536);
2635 #ifdef KERN_TLS
2636 if (is_ktls(sc)) {
2637 if_setcapabilitiesbit(ifp, IFCAP_TXTLS, 0);
2638 if (sc->flags & KERN_TLS_ON || !is_t6(sc))
2639 if_setcapenablebit(ifp, IFCAP_TXTLS, 0);
2640 }
2641 #endif
2642
2643 ether_ifattach(ifp, vi->hw_addr);
2644 #ifdef DEV_NETMAP
2645 if (vi->nnmrxq != 0)
2646 cxgbe_nm_attach(vi);
2647 #endif
2648 sb = sbuf_new_auto();
2649 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq);
2650 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
2651 switch (if_getcapabilities(ifp) & (IFCAP_TOE | IFCAP_TXRTLMT)) {
2652 case IFCAP_TOE:
2653 sbuf_printf(sb, "; %d txq (TOE)", vi->nofldtxq);
2654 break;
2655 case IFCAP_TOE | IFCAP_TXRTLMT:
2656 sbuf_printf(sb, "; %d txq (TOE/ETHOFLD)", vi->nofldtxq);
2657 break;
2658 case IFCAP_TXRTLMT:
2659 sbuf_printf(sb, "; %d txq (ETHOFLD)", vi->nofldtxq);
2660 break;
2661 }
2662 #endif
2663 #ifdef TCP_OFFLOAD
2664 if (if_getcapabilities(ifp) & IFCAP_TOE)
2665 sbuf_printf(sb, ", %d rxq (TOE)", vi->nofldrxq);
2666 #endif
2667 #ifdef DEV_NETMAP
2668 if (if_getcapabilities(ifp) & IFCAP_NETMAP)
2669 sbuf_printf(sb, "; %d txq, %d rxq (netmap)",
2670 vi->nnmtxq, vi->nnmrxq);
2671 #endif
2672 sbuf_finish(sb);
2673 device_printf(dev, "%s\n", sbuf_data(sb));
2674 sbuf_delete(sb);
2675
2676 vi_sysctls(vi);
2677
2678 pa.pa_version = PFIL_VERSION;
2679 pa.pa_flags = PFIL_IN;
2680 pa.pa_type = PFIL_TYPE_ETHERNET;
2681 pa.pa_headname = if_name(ifp);
2682 vi->pfil = pfil_head_register(&pa);
2683
2684 return (0);
2685 }
2686
2687 static int
cxgbe_attach(device_t dev)2688 cxgbe_attach(device_t dev)
2689 {
2690 struct port_info *pi = device_get_softc(dev);
2691 struct adapter *sc = pi->adapter;
2692 struct vi_info *vi;
2693 int i, rc;
2694
2695 sysctl_ctx_init(&pi->ctx);
2696
2697 rc = cxgbe_vi_attach(dev, &pi->vi[0]);
2698 if (rc)
2699 return (rc);
2700
2701 for_each_vi(pi, i, vi) {
2702 if (i == 0)
2703 continue;
2704 vi->dev = device_add_child(dev, sc->names->vi_ifnet_name, -1);
2705 if (vi->dev == NULL) {
2706 device_printf(dev, "failed to add VI %d\n", i);
2707 continue;
2708 }
2709 device_set_softc(vi->dev, vi);
2710 }
2711
2712 cxgbe_sysctls(pi);
2713
2714 bus_generic_attach(dev);
2715
2716 return (0);
2717 }
2718
2719 static void
cxgbe_vi_detach(struct vi_info * vi)2720 cxgbe_vi_detach(struct vi_info *vi)
2721 {
2722 if_t ifp = vi->ifp;
2723
2724 if (vi->pfil != NULL) {
2725 pfil_head_unregister(vi->pfil);
2726 vi->pfil = NULL;
2727 }
2728
2729 ether_ifdetach(ifp);
2730
2731 /* Let detach proceed even if these fail. */
2732 #ifdef DEV_NETMAP
2733 if (if_getcapabilities(ifp) & IFCAP_NETMAP)
2734 cxgbe_nm_detach(vi);
2735 #endif
2736 cxgbe_uninit_synchronized(vi);
2737 callout_drain(&vi->tick);
2738 mtx_destroy(&vi->tick_mtx);
2739 sysctl_ctx_free(&vi->ctx);
2740 vi_full_uninit(vi);
2741
2742 if_free(vi->ifp);
2743 vi->ifp = NULL;
2744 }
2745
2746 static int
cxgbe_detach(device_t dev)2747 cxgbe_detach(device_t dev)
2748 {
2749 struct port_info *pi = device_get_softc(dev);
2750 struct adapter *sc = pi->adapter;
2751 int rc;
2752
2753 /* Detach the extra VIs first. */
2754 rc = bus_generic_detach(dev);
2755 if (rc)
2756 return (rc);
2757 device_delete_children(dev);
2758
2759 sysctl_ctx_free(&pi->ctx);
2760 begin_vi_detach(sc, &pi->vi[0]);
2761 if (pi->flags & HAS_TRACEQ) {
2762 sc->traceq = -1; /* cloner should not create ifnet */
2763 t4_tracer_port_detach(sc);
2764 }
2765 cxgbe_vi_detach(&pi->vi[0]);
2766 ifmedia_removeall(&pi->media);
2767 end_vi_detach(sc, &pi->vi[0]);
2768
2769 return (0);
2770 }
2771
2772 static void
cxgbe_init(void * arg)2773 cxgbe_init(void *arg)
2774 {
2775 struct vi_info *vi = arg;
2776 struct adapter *sc = vi->adapter;
2777
2778 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0)
2779 return;
2780 cxgbe_init_synchronized(vi);
2781 end_synchronized_op(sc, 0);
2782 }
2783
2784 static int
cxgbe_ioctl(if_t ifp,unsigned long cmd,caddr_t data)2785 cxgbe_ioctl(if_t ifp, unsigned long cmd, caddr_t data)
2786 {
2787 int rc = 0, mtu, flags;
2788 struct vi_info *vi = if_getsoftc(ifp);
2789 struct port_info *pi = vi->pi;
2790 struct adapter *sc = pi->adapter;
2791 struct ifreq *ifr = (struct ifreq *)data;
2792 uint32_t mask;
2793
2794 switch (cmd) {
2795 case SIOCSIFMTU:
2796 mtu = ifr->ifr_mtu;
2797 if (mtu < ETHERMIN || mtu > MAX_MTU)
2798 return (EINVAL);
2799
2800 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu");
2801 if (rc)
2802 return (rc);
2803 if_setmtu(ifp, mtu);
2804 if (vi->flags & VI_INIT_DONE) {
2805 t4_update_fl_bufsize(ifp);
2806 if (!hw_off_limits(sc) &&
2807 if_getdrvflags(ifp) & IFF_DRV_RUNNING)
2808 rc = update_mac_settings(ifp, XGMAC_MTU);
2809 }
2810 end_synchronized_op(sc, 0);
2811 break;
2812
2813 case SIOCSIFFLAGS:
2814 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4flg");
2815 if (rc)
2816 return (rc);
2817
2818 if (hw_off_limits(sc)) {
2819 rc = ENXIO;
2820 goto fail;
2821 }
2822
2823 if (if_getflags(ifp) & IFF_UP) {
2824 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
2825 flags = vi->if_flags;
2826 if ((if_getflags(ifp) ^ flags) &
2827 (IFF_PROMISC | IFF_ALLMULTI)) {
2828 rc = update_mac_settings(ifp,
2829 XGMAC_PROMISC | XGMAC_ALLMULTI);
2830 }
2831 } else {
2832 rc = cxgbe_init_synchronized(vi);
2833 }
2834 vi->if_flags = if_getflags(ifp);
2835 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
2836 rc = cxgbe_uninit_synchronized(vi);
2837 }
2838 end_synchronized_op(sc, 0);
2839 break;
2840
2841 case SIOCADDMULTI:
2842 case SIOCDELMULTI:
2843 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4multi");
2844 if (rc)
2845 return (rc);
2846 if (!hw_off_limits(sc) && if_getdrvflags(ifp) & IFF_DRV_RUNNING)
2847 rc = update_mac_settings(ifp, XGMAC_MCADDRS);
2848 end_synchronized_op(sc, 0);
2849 break;
2850
2851 case SIOCSIFCAP:
2852 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap");
2853 if (rc)
2854 return (rc);
2855
2856 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
2857 if (mask & IFCAP_TXCSUM) {
2858 if_togglecapenable(ifp, IFCAP_TXCSUM);
2859 if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP);
2860
2861 if (IFCAP_TSO4 & if_getcapenable(ifp) &&
2862 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
2863 mask &= ~IFCAP_TSO4;
2864 if_setcapenablebit(ifp, 0, IFCAP_TSO4);
2865 if_printf(ifp,
2866 "tso4 disabled due to -txcsum.\n");
2867 }
2868 }
2869 if (mask & IFCAP_TXCSUM_IPV6) {
2870 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
2871 if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
2872
2873 if (IFCAP_TSO6 & if_getcapenable(ifp) &&
2874 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
2875 mask &= ~IFCAP_TSO6;
2876 if_setcapenablebit(ifp, 0, IFCAP_TSO6);
2877 if_printf(ifp,
2878 "tso6 disabled due to -txcsum6.\n");
2879 }
2880 }
2881 if (mask & IFCAP_RXCSUM)
2882 if_togglecapenable(ifp, IFCAP_RXCSUM);
2883 if (mask & IFCAP_RXCSUM_IPV6)
2884 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
2885
2886 /*
2887 * Note that we leave CSUM_TSO alone (it is always set). The
2888 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
2889 * sending a TSO request our way, so it's sufficient to toggle
2890 * IFCAP_TSOx only.
2891 */
2892 if (mask & IFCAP_TSO4) {
2893 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) &&
2894 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
2895 if_printf(ifp, "enable txcsum first.\n");
2896 rc = EAGAIN;
2897 goto fail;
2898 }
2899 if_togglecapenable(ifp, IFCAP_TSO4);
2900 }
2901 if (mask & IFCAP_TSO6) {
2902 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) &&
2903 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
2904 if_printf(ifp, "enable txcsum6 first.\n");
2905 rc = EAGAIN;
2906 goto fail;
2907 }
2908 if_togglecapenable(ifp, IFCAP_TSO6);
2909 }
2910 if (mask & IFCAP_LRO) {
2911 #if defined(INET) || defined(INET6)
2912 int i;
2913 struct sge_rxq *rxq;
2914
2915 if_togglecapenable(ifp, IFCAP_LRO);
2916 for_each_rxq(vi, i, rxq) {
2917 if (if_getcapenable(ifp) & IFCAP_LRO)
2918 rxq->iq.flags |= IQ_LRO_ENABLED;
2919 else
2920 rxq->iq.flags &= ~IQ_LRO_ENABLED;
2921 }
2922 #endif
2923 }
2924 #ifdef TCP_OFFLOAD
2925 if (mask & IFCAP_TOE) {
2926 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE;
2927
2928 rc = toe_capability(vi, enable);
2929 if (rc != 0)
2930 goto fail;
2931
2932 if_togglecapenable(ifp, mask);
2933 }
2934 #endif
2935 if (mask & IFCAP_VLAN_HWTAGGING) {
2936 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
2937 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
2938 rc = update_mac_settings(ifp, XGMAC_VLANEX);
2939 }
2940 if (mask & IFCAP_VLAN_MTU) {
2941 if_togglecapenable(ifp, IFCAP_VLAN_MTU);
2942
2943 /* Need to find out how to disable auto-mtu-inflation */
2944 }
2945 if (mask & IFCAP_VLAN_HWTSO)
2946 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
2947 if (mask & IFCAP_VLAN_HWCSUM)
2948 if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM);
2949 #ifdef RATELIMIT
2950 if (mask & IFCAP_TXRTLMT)
2951 if_togglecapenable(ifp, IFCAP_TXRTLMT);
2952 #endif
2953 if (mask & IFCAP_HWRXTSTMP) {
2954 int i;
2955 struct sge_rxq *rxq;
2956
2957 if_togglecapenable(ifp, IFCAP_HWRXTSTMP);
2958 for_each_rxq(vi, i, rxq) {
2959 if (if_getcapenable(ifp) & IFCAP_HWRXTSTMP)
2960 rxq->iq.flags |= IQ_RX_TIMESTAMP;
2961 else
2962 rxq->iq.flags &= ~IQ_RX_TIMESTAMP;
2963 }
2964 }
2965 if (mask & IFCAP_MEXTPG)
2966 if_togglecapenable(ifp, IFCAP_MEXTPG);
2967
2968 #ifdef KERN_TLS
2969 if (mask & IFCAP_TXTLS) {
2970 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TXTLS;
2971
2972 rc = ktls_capability(sc, enable);
2973 if (rc != 0)
2974 goto fail;
2975
2976 if_togglecapenable(ifp, mask & IFCAP_TXTLS);
2977 }
2978 #endif
2979 if (mask & IFCAP_VXLAN_HWCSUM) {
2980 if_togglecapenable(ifp, IFCAP_VXLAN_HWCSUM);
2981 if_togglehwassist(ifp, CSUM_INNER_IP6_UDP |
2982 CSUM_INNER_IP6_TCP | CSUM_INNER_IP |
2983 CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP);
2984 }
2985 if (mask & IFCAP_VXLAN_HWTSO) {
2986 if_togglecapenable(ifp, IFCAP_VXLAN_HWTSO);
2987 if_togglehwassist(ifp, CSUM_INNER_IP6_TSO |
2988 CSUM_INNER_IP_TSO);
2989 }
2990
2991 #ifdef VLAN_CAPABILITIES
2992 VLAN_CAPABILITIES(ifp);
2993 #endif
2994 fail:
2995 end_synchronized_op(sc, 0);
2996 break;
2997
2998 case SIOCSIFMEDIA:
2999 case SIOCGIFMEDIA:
3000 case SIOCGIFXMEDIA:
3001 rc = ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
3002 break;
3003
3004 case SIOCGI2C: {
3005 struct ifi2creq i2c;
3006
3007 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3008 if (rc != 0)
3009 break;
3010 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
3011 rc = EPERM;
3012 break;
3013 }
3014 if (i2c.len > sizeof(i2c.data)) {
3015 rc = EINVAL;
3016 break;
3017 }
3018 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c");
3019 if (rc)
3020 return (rc);
3021 if (hw_off_limits(sc))
3022 rc = ENXIO;
3023 else
3024 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr,
3025 i2c.offset, i2c.len, &i2c.data[0]);
3026 end_synchronized_op(sc, 0);
3027 if (rc == 0)
3028 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3029 break;
3030 }
3031
3032 default:
3033 rc = ether_ioctl(ifp, cmd, data);
3034 }
3035
3036 return (rc);
3037 }
3038
3039 static int
cxgbe_transmit(if_t ifp,struct mbuf * m)3040 cxgbe_transmit(if_t ifp, struct mbuf *m)
3041 {
3042 struct vi_info *vi = if_getsoftc(ifp);
3043 struct port_info *pi = vi->pi;
3044 struct adapter *sc;
3045 struct sge_txq *txq;
3046 void *items[1];
3047 int rc;
3048
3049 M_ASSERTPKTHDR(m);
3050 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
3051 #if defined(KERN_TLS) || defined(RATELIMIT)
3052 if (m->m_pkthdr.csum_flags & CSUM_SND_TAG)
3053 MPASS(m->m_pkthdr.snd_tag->ifp == ifp);
3054 #endif
3055
3056 if (__predict_false(pi->link_cfg.link_ok == false)) {
3057 m_freem(m);
3058 return (ENETDOWN);
3059 }
3060
3061 rc = parse_pkt(&m, vi->flags & TX_USES_VM_WR);
3062 if (__predict_false(rc != 0)) {
3063 if (__predict_true(rc == EINPROGRESS)) {
3064 /* queued by parse_pkt */
3065 MPASS(m != NULL);
3066 return (0);
3067 }
3068
3069 MPASS(m == NULL); /* was freed already */
3070 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */
3071 return (rc);
3072 }
3073
3074 /* Select a txq. */
3075 sc = vi->adapter;
3076 txq = &sc->sge.txq[vi->first_txq];
3077 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
3078 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) +
3079 vi->rsrv_noflowq);
3080
3081 items[0] = m;
3082 rc = mp_ring_enqueue(txq->r, items, 1, 256);
3083 if (__predict_false(rc != 0))
3084 m_freem(m);
3085
3086 return (rc);
3087 }
3088
3089 static void
cxgbe_qflush(if_t ifp)3090 cxgbe_qflush(if_t ifp)
3091 {
3092 struct vi_info *vi = if_getsoftc(ifp);
3093 struct sge_txq *txq;
3094 int i;
3095
3096 /* queues do not exist if !VI_INIT_DONE. */
3097 if (vi->flags & VI_INIT_DONE) {
3098 for_each_txq(vi, i, txq) {
3099 TXQ_LOCK(txq);
3100 txq->eq.flags |= EQ_QFLUSH;
3101 TXQ_UNLOCK(txq);
3102 while (!mp_ring_is_idle(txq->r)) {
3103 mp_ring_check_drainage(txq->r, 4096);
3104 pause("qflush", 1);
3105 }
3106 TXQ_LOCK(txq);
3107 txq->eq.flags &= ~EQ_QFLUSH;
3108 TXQ_UNLOCK(txq);
3109 }
3110 }
3111 if_qflush(ifp);
3112 }
3113
3114 static uint64_t
vi_get_counter(if_t ifp,ift_counter c)3115 vi_get_counter(if_t ifp, ift_counter c)
3116 {
3117 struct vi_info *vi = if_getsoftc(ifp);
3118 struct fw_vi_stats_vf *s = &vi->stats;
3119
3120 mtx_lock(&vi->tick_mtx);
3121 vi_refresh_stats(vi);
3122 mtx_unlock(&vi->tick_mtx);
3123
3124 switch (c) {
3125 case IFCOUNTER_IPACKETS:
3126 return (s->rx_bcast_frames + s->rx_mcast_frames +
3127 s->rx_ucast_frames);
3128 case IFCOUNTER_IERRORS:
3129 return (s->rx_err_frames);
3130 case IFCOUNTER_OPACKETS:
3131 return (s->tx_bcast_frames + s->tx_mcast_frames +
3132 s->tx_ucast_frames + s->tx_offload_frames);
3133 case IFCOUNTER_OERRORS:
3134 return (s->tx_drop_frames);
3135 case IFCOUNTER_IBYTES:
3136 return (s->rx_bcast_bytes + s->rx_mcast_bytes +
3137 s->rx_ucast_bytes);
3138 case IFCOUNTER_OBYTES:
3139 return (s->tx_bcast_bytes + s->tx_mcast_bytes +
3140 s->tx_ucast_bytes + s->tx_offload_bytes);
3141 case IFCOUNTER_IMCASTS:
3142 return (s->rx_mcast_frames);
3143 case IFCOUNTER_OMCASTS:
3144 return (s->tx_mcast_frames);
3145 case IFCOUNTER_OQDROPS: {
3146 uint64_t drops;
3147
3148 drops = 0;
3149 if (vi->flags & VI_INIT_DONE) {
3150 int i;
3151 struct sge_txq *txq;
3152
3153 for_each_txq(vi, i, txq)
3154 drops += counter_u64_fetch(txq->r->dropped);
3155 }
3156
3157 return (drops);
3158
3159 }
3160
3161 default:
3162 return (if_get_counter_default(ifp, c));
3163 }
3164 }
3165
3166 static uint64_t
cxgbe_get_counter(if_t ifp,ift_counter c)3167 cxgbe_get_counter(if_t ifp, ift_counter c)
3168 {
3169 struct vi_info *vi = if_getsoftc(ifp);
3170 struct port_info *pi = vi->pi;
3171 struct port_stats *s = &pi->stats;
3172
3173 mtx_lock(&vi->tick_mtx);
3174 cxgbe_refresh_stats(vi);
3175 mtx_unlock(&vi->tick_mtx);
3176
3177 switch (c) {
3178 case IFCOUNTER_IPACKETS:
3179 return (s->rx_frames);
3180
3181 case IFCOUNTER_IERRORS:
3182 return (s->rx_jabber + s->rx_runt + s->rx_too_long +
3183 s->rx_fcs_err + s->rx_len_err);
3184
3185 case IFCOUNTER_OPACKETS:
3186 return (s->tx_frames);
3187
3188 case IFCOUNTER_OERRORS:
3189 return (s->tx_error_frames);
3190
3191 case IFCOUNTER_IBYTES:
3192 return (s->rx_octets);
3193
3194 case IFCOUNTER_OBYTES:
3195 return (s->tx_octets);
3196
3197 case IFCOUNTER_IMCASTS:
3198 return (s->rx_mcast_frames);
3199
3200 case IFCOUNTER_OMCASTS:
3201 return (s->tx_mcast_frames);
3202
3203 case IFCOUNTER_IQDROPS:
3204 return (s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
3205 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
3206 s->rx_trunc3 + pi->tnl_cong_drops);
3207
3208 case IFCOUNTER_OQDROPS: {
3209 uint64_t drops;
3210
3211 drops = s->tx_drop;
3212 if (vi->flags & VI_INIT_DONE) {
3213 int i;
3214 struct sge_txq *txq;
3215
3216 for_each_txq(vi, i, txq)
3217 drops += counter_u64_fetch(txq->r->dropped);
3218 }
3219
3220 return (drops);
3221
3222 }
3223
3224 default:
3225 return (if_get_counter_default(ifp, c));
3226 }
3227 }
3228
3229 #if defined(KERN_TLS) || defined(RATELIMIT)
3230 static int
cxgbe_snd_tag_alloc(if_t ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** pt)3231 cxgbe_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
3232 struct m_snd_tag **pt)
3233 {
3234 int error;
3235
3236 switch (params->hdr.type) {
3237 #ifdef RATELIMIT
3238 case IF_SND_TAG_TYPE_RATE_LIMIT:
3239 error = cxgbe_rate_tag_alloc(ifp, params, pt);
3240 break;
3241 #endif
3242 #ifdef KERN_TLS
3243 case IF_SND_TAG_TYPE_TLS:
3244 {
3245 struct vi_info *vi = if_getsoftc(ifp);
3246
3247 if (is_t6(vi->pi->adapter))
3248 error = t6_tls_tag_alloc(ifp, params, pt);
3249 else
3250 error = EOPNOTSUPP;
3251 break;
3252 }
3253 #endif
3254 default:
3255 error = EOPNOTSUPP;
3256 }
3257 return (error);
3258 }
3259 #endif
3260
3261 /*
3262 * The kernel picks a media from the list we had provided but we still validate
3263 * the requeste.
3264 */
3265 int
cxgbe_media_change(if_t ifp)3266 cxgbe_media_change(if_t ifp)
3267 {
3268 struct vi_info *vi = if_getsoftc(ifp);
3269 struct port_info *pi = vi->pi;
3270 struct ifmedia *ifm = &pi->media;
3271 struct link_config *lc = &pi->link_cfg;
3272 struct adapter *sc = pi->adapter;
3273 int rc;
3274
3275 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mec");
3276 if (rc != 0)
3277 return (rc);
3278 PORT_LOCK(pi);
3279 if (IFM_SUBTYPE(ifm->ifm_media) == IFM_AUTO) {
3280 /* ifconfig .. media autoselect */
3281 if (!(lc->pcaps & FW_PORT_CAP32_ANEG)) {
3282 rc = ENOTSUP; /* AN not supported by transceiver */
3283 goto done;
3284 }
3285 lc->requested_aneg = AUTONEG_ENABLE;
3286 lc->requested_speed = 0;
3287 lc->requested_fc |= PAUSE_AUTONEG;
3288 } else {
3289 lc->requested_aneg = AUTONEG_DISABLE;
3290 lc->requested_speed =
3291 ifmedia_baudrate(ifm->ifm_media) / 1000000;
3292 lc->requested_fc = 0;
3293 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_RXPAUSE)
3294 lc->requested_fc |= PAUSE_RX;
3295 if (IFM_OPTIONS(ifm->ifm_media) & IFM_ETH_TXPAUSE)
3296 lc->requested_fc |= PAUSE_TX;
3297 }
3298 if (pi->up_vis > 0 && !hw_off_limits(sc)) {
3299 fixup_link_config(pi);
3300 rc = apply_link_config(pi);
3301 }
3302 done:
3303 PORT_UNLOCK(pi);
3304 end_synchronized_op(sc, 0);
3305 return (rc);
3306 }
3307
3308 /*
3309 * Base media word (without ETHER, pause, link active, etc.) for the port at the
3310 * given speed.
3311 */
3312 static int
port_mword(struct port_info * pi,uint32_t speed)3313 port_mword(struct port_info *pi, uint32_t speed)
3314 {
3315
3316 MPASS(speed & M_FW_PORT_CAP32_SPEED);
3317 MPASS(powerof2(speed));
3318
3319 switch(pi->port_type) {
3320 case FW_PORT_TYPE_BT_SGMII:
3321 case FW_PORT_TYPE_BT_XFI:
3322 case FW_PORT_TYPE_BT_XAUI:
3323 /* BaseT */
3324 switch (speed) {
3325 case FW_PORT_CAP32_SPEED_100M:
3326 return (IFM_100_T);
3327 case FW_PORT_CAP32_SPEED_1G:
3328 return (IFM_1000_T);
3329 case FW_PORT_CAP32_SPEED_10G:
3330 return (IFM_10G_T);
3331 }
3332 break;
3333 case FW_PORT_TYPE_KX4:
3334 if (speed == FW_PORT_CAP32_SPEED_10G)
3335 return (IFM_10G_KX4);
3336 break;
3337 case FW_PORT_TYPE_CX4:
3338 if (speed == FW_PORT_CAP32_SPEED_10G)
3339 return (IFM_10G_CX4);
3340 break;
3341 case FW_PORT_TYPE_KX:
3342 if (speed == FW_PORT_CAP32_SPEED_1G)
3343 return (IFM_1000_KX);
3344 break;
3345 case FW_PORT_TYPE_KR:
3346 case FW_PORT_TYPE_BP_AP:
3347 case FW_PORT_TYPE_BP4_AP:
3348 case FW_PORT_TYPE_BP40_BA:
3349 case FW_PORT_TYPE_KR4_100G:
3350 case FW_PORT_TYPE_KR_SFP28:
3351 case FW_PORT_TYPE_KR_XLAUI:
3352 switch (speed) {
3353 case FW_PORT_CAP32_SPEED_1G:
3354 return (IFM_1000_KX);
3355 case FW_PORT_CAP32_SPEED_10G:
3356 return (IFM_10G_KR);
3357 case FW_PORT_CAP32_SPEED_25G:
3358 return (IFM_25G_KR);
3359 case FW_PORT_CAP32_SPEED_40G:
3360 return (IFM_40G_KR4);
3361 case FW_PORT_CAP32_SPEED_50G:
3362 return (IFM_50G_KR2);
3363 case FW_PORT_CAP32_SPEED_100G:
3364 return (IFM_100G_KR4);
3365 }
3366 break;
3367 case FW_PORT_TYPE_FIBER_XFI:
3368 case FW_PORT_TYPE_FIBER_XAUI:
3369 case FW_PORT_TYPE_SFP:
3370 case FW_PORT_TYPE_QSFP_10G:
3371 case FW_PORT_TYPE_QSA:
3372 case FW_PORT_TYPE_QSFP:
3373 case FW_PORT_TYPE_CR4_QSFP:
3374 case FW_PORT_TYPE_CR_QSFP:
3375 case FW_PORT_TYPE_CR2_QSFP:
3376 case FW_PORT_TYPE_SFP28:
3377 /* Pluggable transceiver */
3378 switch (pi->mod_type) {
3379 case FW_PORT_MOD_TYPE_LR:
3380 switch (speed) {
3381 case FW_PORT_CAP32_SPEED_1G:
3382 return (IFM_1000_LX);
3383 case FW_PORT_CAP32_SPEED_10G:
3384 return (IFM_10G_LR);
3385 case FW_PORT_CAP32_SPEED_25G:
3386 return (IFM_25G_LR);
3387 case FW_PORT_CAP32_SPEED_40G:
3388 return (IFM_40G_LR4);
3389 case FW_PORT_CAP32_SPEED_50G:
3390 return (IFM_50G_LR2);
3391 case FW_PORT_CAP32_SPEED_100G:
3392 return (IFM_100G_LR4);
3393 }
3394 break;
3395 case FW_PORT_MOD_TYPE_SR:
3396 switch (speed) {
3397 case FW_PORT_CAP32_SPEED_1G:
3398 return (IFM_1000_SX);
3399 case FW_PORT_CAP32_SPEED_10G:
3400 return (IFM_10G_SR);
3401 case FW_PORT_CAP32_SPEED_25G:
3402 return (IFM_25G_SR);
3403 case FW_PORT_CAP32_SPEED_40G:
3404 return (IFM_40G_SR4);
3405 case FW_PORT_CAP32_SPEED_50G:
3406 return (IFM_50G_SR2);
3407 case FW_PORT_CAP32_SPEED_100G:
3408 return (IFM_100G_SR4);
3409 }
3410 break;
3411 case FW_PORT_MOD_TYPE_ER:
3412 if (speed == FW_PORT_CAP32_SPEED_10G)
3413 return (IFM_10G_ER);
3414 break;
3415 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
3416 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
3417 switch (speed) {
3418 case FW_PORT_CAP32_SPEED_1G:
3419 return (IFM_1000_CX);
3420 case FW_PORT_CAP32_SPEED_10G:
3421 return (IFM_10G_TWINAX);
3422 case FW_PORT_CAP32_SPEED_25G:
3423 return (IFM_25G_CR);
3424 case FW_PORT_CAP32_SPEED_40G:
3425 return (IFM_40G_CR4);
3426 case FW_PORT_CAP32_SPEED_50G:
3427 return (IFM_50G_CR2);
3428 case FW_PORT_CAP32_SPEED_100G:
3429 return (IFM_100G_CR4);
3430 }
3431 break;
3432 case FW_PORT_MOD_TYPE_LRM:
3433 if (speed == FW_PORT_CAP32_SPEED_10G)
3434 return (IFM_10G_LRM);
3435 break;
3436 case FW_PORT_MOD_TYPE_NA:
3437 MPASS(0); /* Not pluggable? */
3438 /* fall throough */
3439 case FW_PORT_MOD_TYPE_ERROR:
3440 case FW_PORT_MOD_TYPE_UNKNOWN:
3441 case FW_PORT_MOD_TYPE_NOTSUPPORTED:
3442 break;
3443 case FW_PORT_MOD_TYPE_NONE:
3444 return (IFM_NONE);
3445 }
3446 break;
3447 case FW_PORT_TYPE_NONE:
3448 return (IFM_NONE);
3449 }
3450
3451 return (IFM_UNKNOWN);
3452 }
3453
3454 void
cxgbe_media_status(if_t ifp,struct ifmediareq * ifmr)3455 cxgbe_media_status(if_t ifp, struct ifmediareq *ifmr)
3456 {
3457 struct vi_info *vi = if_getsoftc(ifp);
3458 struct port_info *pi = vi->pi;
3459 struct adapter *sc = pi->adapter;
3460 struct link_config *lc = &pi->link_cfg;
3461
3462 if (begin_synchronized_op(sc, vi , SLEEP_OK | INTR_OK, "t4med") != 0)
3463 return;
3464 PORT_LOCK(pi);
3465
3466 if (pi->up_vis == 0 && !hw_off_limits(sc)) {
3467 /*
3468 * If all the interfaces are administratively down the firmware
3469 * does not report transceiver changes. Refresh port info here
3470 * so that ifconfig displays accurate ifmedia at all times.
3471 * This is the only reason we have a synchronized op in this
3472 * function. Just PORT_LOCK would have been enough otherwise.
3473 */
3474 t4_update_port_info(pi);
3475 build_medialist(pi);
3476 }
3477
3478 /* ifm_status */
3479 ifmr->ifm_status = IFM_AVALID;
3480 if (lc->link_ok == false)
3481 goto done;
3482 ifmr->ifm_status |= IFM_ACTIVE;
3483
3484 /* ifm_active */
3485 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
3486 ifmr->ifm_active &= ~(IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE);
3487 if (lc->fc & PAUSE_RX)
3488 ifmr->ifm_active |= IFM_ETH_RXPAUSE;
3489 if (lc->fc & PAUSE_TX)
3490 ifmr->ifm_active |= IFM_ETH_TXPAUSE;
3491 ifmr->ifm_active |= port_mword(pi, speed_to_fwcap(lc->speed));
3492 done:
3493 PORT_UNLOCK(pi);
3494 end_synchronized_op(sc, 0);
3495 }
3496
3497 static int
vcxgbe_probe(device_t dev)3498 vcxgbe_probe(device_t dev)
3499 {
3500 struct vi_info *vi = device_get_softc(dev);
3501
3502 device_set_descf(dev, "port %d vi %td", vi->pi->port_id,
3503 vi - vi->pi->vi);
3504
3505 return (BUS_PROBE_DEFAULT);
3506 }
3507
3508 static int
alloc_extra_vi(struct adapter * sc,struct port_info * pi,struct vi_info * vi)3509 alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi)
3510 {
3511 int func, index, rc;
3512 uint32_t param, val;
3513
3514 ASSERT_SYNCHRONIZED_OP(sc);
3515
3516 index = vi - pi->vi;
3517 MPASS(index > 0); /* This function deals with _extra_ VIs only */
3518 KASSERT(index < nitems(vi_mac_funcs),
3519 ("%s: VI %s doesn't have a MAC func", __func__,
3520 device_get_nameunit(vi->dev)));
3521 func = vi_mac_funcs[index];
3522 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1,
3523 vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0);
3524 if (rc < 0) {
3525 CH_ERR(vi, "failed to allocate virtual interface %d"
3526 "for port %d: %d\n", index, pi->port_id, -rc);
3527 return (-rc);
3528 }
3529 vi->viid = rc;
3530
3531 if (vi->rss_size == 1) {
3532 /*
3533 * This VI didn't get a slice of the RSS table. Reduce the
3534 * number of VIs being created (hw.cxgbe.num_vis) or modify the
3535 * configuration file (nvi, rssnvi for this PF) if this is a
3536 * problem.
3537 */
3538 device_printf(vi->dev, "RSS table not available.\n");
3539 vi->rss_base = 0xffff;
3540
3541 return (0);
3542 }
3543
3544 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
3545 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) |
3546 V_FW_PARAMS_PARAM_YZ(vi->viid);
3547 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
3548 if (rc)
3549 vi->rss_base = 0xffff;
3550 else {
3551 MPASS((val >> 16) == vi->rss_size);
3552 vi->rss_base = val & 0xffff;
3553 }
3554
3555 return (0);
3556 }
3557
3558 static int
vcxgbe_attach(device_t dev)3559 vcxgbe_attach(device_t dev)
3560 {
3561 struct vi_info *vi;
3562 struct port_info *pi;
3563 struct adapter *sc;
3564 int rc;
3565
3566 vi = device_get_softc(dev);
3567 pi = vi->pi;
3568 sc = pi->adapter;
3569
3570 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4via");
3571 if (rc)
3572 return (rc);
3573 rc = alloc_extra_vi(sc, pi, vi);
3574 end_synchronized_op(sc, 0);
3575 if (rc)
3576 return (rc);
3577
3578 rc = cxgbe_vi_attach(dev, vi);
3579 if (rc) {
3580 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
3581 return (rc);
3582 }
3583 return (0);
3584 }
3585
3586 static int
vcxgbe_detach(device_t dev)3587 vcxgbe_detach(device_t dev)
3588 {
3589 struct vi_info *vi;
3590 struct adapter *sc;
3591
3592 vi = device_get_softc(dev);
3593 sc = vi->adapter;
3594
3595 begin_vi_detach(sc, vi);
3596 cxgbe_vi_detach(vi);
3597 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid);
3598 end_vi_detach(sc, vi);
3599
3600 return (0);
3601 }
3602
3603 static struct callout fatal_callout;
3604 static struct taskqueue *reset_tq;
3605
3606 static void
delayed_panic(void * arg)3607 delayed_panic(void *arg)
3608 {
3609 struct adapter *sc = arg;
3610
3611 panic("%s: panic on fatal error", device_get_nameunit(sc->dev));
3612 }
3613
3614 static void
fatal_error_task(void * arg,int pending)3615 fatal_error_task(void *arg, int pending)
3616 {
3617 struct adapter *sc = arg;
3618 int rc;
3619
3620 #ifdef TCP_OFFLOAD
3621 t4_async_event(sc);
3622 #endif
3623 if (atomic_testandclear_int(&sc->error_flags, ilog2(ADAP_CIM_ERR))) {
3624 dump_cim_regs(sc);
3625 dump_cimla(sc);
3626 dump_devlog(sc);
3627 }
3628
3629 if (t4_reset_on_fatal_err) {
3630 CH_ALERT(sc, "resetting on fatal error.\n");
3631 rc = reset_adapter(sc);
3632 if (rc == 0 && t4_panic_on_fatal_err) {
3633 CH_ALERT(sc, "reset was successful, "
3634 "system will NOT panic.\n");
3635 return;
3636 }
3637 }
3638
3639 if (t4_panic_on_fatal_err) {
3640 CH_ALERT(sc, "panicking on fatal error (after 30s).\n");
3641 callout_reset(&fatal_callout, hz * 30, delayed_panic, sc);
3642 }
3643 }
3644
3645 void
t4_fatal_err(struct adapter * sc,bool fw_error)3646 t4_fatal_err(struct adapter *sc, bool fw_error)
3647 {
3648 const bool verbose = (sc->debug_flags & DF_VERBOSE_SLOWINTR) != 0;
3649
3650 stop_adapter(sc);
3651 if (atomic_testandset_int(&sc->error_flags, ilog2(ADAP_FATAL_ERR)))
3652 return;
3653 if (fw_error) {
3654 /*
3655 * We are here because of a firmware error/timeout and not
3656 * because of a hardware interrupt. It is possible (although
3657 * not very likely) that an error interrupt was also raised but
3658 * this thread ran first and inhibited t4_intr_err. We walk the
3659 * main INT_CAUSE registers here to make sure we haven't missed
3660 * anything interesting.
3661 */
3662 t4_slow_intr_handler(sc, verbose);
3663 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
3664 }
3665 t4_report_fw_error(sc);
3666 log(LOG_ALERT, "%s: encountered fatal error, adapter stopped (%d).\n",
3667 device_get_nameunit(sc->dev), fw_error);
3668 taskqueue_enqueue(reset_tq, &sc->fatal_error_task);
3669 }
3670
3671 void
t4_add_adapter(struct adapter * sc)3672 t4_add_adapter(struct adapter *sc)
3673 {
3674 sx_xlock(&t4_list_lock);
3675 SLIST_INSERT_HEAD(&t4_list, sc, link);
3676 sx_xunlock(&t4_list_lock);
3677 }
3678
3679 int
t4_map_bars_0_and_4(struct adapter * sc)3680 t4_map_bars_0_and_4(struct adapter *sc)
3681 {
3682 sc->regs_rid = PCIR_BAR(0);
3683 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3684 &sc->regs_rid, RF_ACTIVE);
3685 if (sc->regs_res == NULL) {
3686 device_printf(sc->dev, "cannot map registers.\n");
3687 return (ENXIO);
3688 }
3689 sc->bt = rman_get_bustag(sc->regs_res);
3690 sc->bh = rman_get_bushandle(sc->regs_res);
3691 sc->mmio_len = rman_get_size(sc->regs_res);
3692 setbit(&sc->doorbells, DOORBELL_KDB);
3693
3694 sc->msix_rid = PCIR_BAR(4);
3695 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3696 &sc->msix_rid, RF_ACTIVE);
3697 if (sc->msix_res == NULL) {
3698 device_printf(sc->dev, "cannot map MSI-X BAR.\n");
3699 return (ENXIO);
3700 }
3701
3702 return (0);
3703 }
3704
3705 int
t4_map_bar_2(struct adapter * sc)3706 t4_map_bar_2(struct adapter *sc)
3707 {
3708
3709 /*
3710 * T4: only iWARP driver uses the userspace doorbells. There is no need
3711 * to map it if RDMA is disabled.
3712 */
3713 if (is_t4(sc) && sc->rdmacaps == 0)
3714 return (0);
3715
3716 sc->udbs_rid = PCIR_BAR(2);
3717 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
3718 &sc->udbs_rid, RF_ACTIVE);
3719 if (sc->udbs_res == NULL) {
3720 device_printf(sc->dev, "cannot map doorbell BAR.\n");
3721 return (ENXIO);
3722 }
3723 sc->udbs_base = rman_get_virtual(sc->udbs_res);
3724
3725 if (chip_id(sc) >= CHELSIO_T5) {
3726 setbit(&sc->doorbells, DOORBELL_UDB);
3727 #if defined(__i386__) || defined(__amd64__)
3728 if (t5_write_combine) {
3729 int rc, mode;
3730
3731 /*
3732 * Enable write combining on BAR2. This is the
3733 * userspace doorbell BAR and is split into 128B
3734 * (UDBS_SEG_SIZE) doorbell regions, each associated
3735 * with an egress queue. The first 64B has the doorbell
3736 * and the second 64B can be used to submit a tx work
3737 * request with an implicit doorbell.
3738 */
3739
3740 rc = pmap_change_attr((vm_offset_t)sc->udbs_base,
3741 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING);
3742 if (rc == 0) {
3743 clrbit(&sc->doorbells, DOORBELL_UDB);
3744 setbit(&sc->doorbells, DOORBELL_WCWR);
3745 setbit(&sc->doorbells, DOORBELL_UDBWC);
3746 } else {
3747 device_printf(sc->dev,
3748 "couldn't enable write combining: %d\n",
3749 rc);
3750 }
3751
3752 mode = is_t5(sc) ? V_STATMODE(0) : V_T6_STATMODE(0);
3753 t4_write_reg(sc, A_SGE_STAT_CFG,
3754 V_STATSOURCE_T5(7) | mode);
3755 }
3756 #endif
3757 }
3758 sc->iwt.wc_en = isset(&sc->doorbells, DOORBELL_UDBWC) ? 1 : 0;
3759
3760 return (0);
3761 }
3762
3763 struct memwin_init {
3764 uint32_t base;
3765 uint32_t aperture;
3766 };
3767
3768 static const struct memwin_init t4_memwin[NUM_MEMWIN] = {
3769 { MEMWIN0_BASE, MEMWIN0_APERTURE },
3770 { MEMWIN1_BASE, MEMWIN1_APERTURE },
3771 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 }
3772 };
3773
3774 static const struct memwin_init t5_memwin[NUM_MEMWIN] = {
3775 { MEMWIN0_BASE, MEMWIN0_APERTURE },
3776 { MEMWIN1_BASE, MEMWIN1_APERTURE },
3777 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 },
3778 };
3779
3780 static void
setup_memwin(struct adapter * sc)3781 setup_memwin(struct adapter *sc)
3782 {
3783 const struct memwin_init *mw_init;
3784 struct memwin *mw;
3785 int i;
3786 uint32_t bar0;
3787
3788 if (is_t4(sc)) {
3789 /*
3790 * Read low 32b of bar0 indirectly via the hardware backdoor
3791 * mechanism. Works from within PCI passthrough environments
3792 * too, where rman_get_start() can return a different value. We
3793 * need to program the T4 memory window decoders with the actual
3794 * addresses that will be coming across the PCIe link.
3795 */
3796 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
3797 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
3798
3799 mw_init = &t4_memwin[0];
3800 } else {
3801 /* T5+ use the relative offset inside the PCIe BAR */
3802 bar0 = 0;
3803
3804 mw_init = &t5_memwin[0];
3805 }
3806
3807 for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) {
3808 if (!rw_initialized(&mw->mw_lock)) {
3809 rw_init(&mw->mw_lock, "memory window access");
3810 mw->mw_base = mw_init->base;
3811 mw->mw_aperture = mw_init->aperture;
3812 mw->mw_curpos = 0;
3813 }
3814 t4_write_reg(sc,
3815 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i),
3816 (mw->mw_base + bar0) | V_BIR(0) |
3817 V_WINDOW(ilog2(mw->mw_aperture) - 10));
3818 rw_wlock(&mw->mw_lock);
3819 position_memwin(sc, i, mw->mw_curpos);
3820 rw_wunlock(&mw->mw_lock);
3821 }
3822
3823 /* flush */
3824 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
3825 }
3826
3827 /*
3828 * Positions the memory window at the given address in the card's address space.
3829 * There are some alignment requirements and the actual position may be at an
3830 * address prior to the requested address. mw->mw_curpos always has the actual
3831 * position of the window.
3832 */
3833 static void
position_memwin(struct adapter * sc,int idx,uint32_t addr)3834 position_memwin(struct adapter *sc, int idx, uint32_t addr)
3835 {
3836 struct memwin *mw;
3837 uint32_t pf;
3838 uint32_t reg;
3839
3840 MPASS(idx >= 0 && idx < NUM_MEMWIN);
3841 mw = &sc->memwin[idx];
3842 rw_assert(&mw->mw_lock, RA_WLOCKED);
3843
3844 if (is_t4(sc)) {
3845 pf = 0;
3846 mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */
3847 } else {
3848 pf = V_PFNUM(sc->pf);
3849 mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */
3850 }
3851 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx);
3852 t4_write_reg(sc, reg, mw->mw_curpos | pf);
3853 t4_read_reg(sc, reg); /* flush */
3854 }
3855
3856 int
rw_via_memwin(struct adapter * sc,int idx,uint32_t addr,uint32_t * val,int len,int rw)3857 rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
3858 int len, int rw)
3859 {
3860 struct memwin *mw;
3861 uint32_t mw_end, v;
3862
3863 MPASS(idx >= 0 && idx < NUM_MEMWIN);
3864
3865 /* Memory can only be accessed in naturally aligned 4 byte units */
3866 if (addr & 3 || len & 3 || len <= 0)
3867 return (EINVAL);
3868
3869 mw = &sc->memwin[idx];
3870 while (len > 0) {
3871 rw_rlock(&mw->mw_lock);
3872 mw_end = mw->mw_curpos + mw->mw_aperture;
3873 if (addr >= mw_end || addr < mw->mw_curpos) {
3874 /* Will need to reposition the window */
3875 if (!rw_try_upgrade(&mw->mw_lock)) {
3876 rw_runlock(&mw->mw_lock);
3877 rw_wlock(&mw->mw_lock);
3878 }
3879 rw_assert(&mw->mw_lock, RA_WLOCKED);
3880 position_memwin(sc, idx, addr);
3881 rw_downgrade(&mw->mw_lock);
3882 mw_end = mw->mw_curpos + mw->mw_aperture;
3883 }
3884 rw_assert(&mw->mw_lock, RA_RLOCKED);
3885 while (addr < mw_end && len > 0) {
3886 if (rw == 0) {
3887 v = t4_read_reg(sc, mw->mw_base + addr -
3888 mw->mw_curpos);
3889 *val++ = le32toh(v);
3890 } else {
3891 v = *val++;
3892 t4_write_reg(sc, mw->mw_base + addr -
3893 mw->mw_curpos, htole32(v));
3894 }
3895 addr += 4;
3896 len -= 4;
3897 }
3898 rw_runlock(&mw->mw_lock);
3899 }
3900
3901 return (0);
3902 }
3903
3904 CTASSERT(M_TID_COOKIE == M_COOKIE);
3905 CTASSERT(MAX_ATIDS <= (M_TID_TID + 1));
3906
3907 static void
t4_init_atid_table(struct adapter * sc)3908 t4_init_atid_table(struct adapter *sc)
3909 {
3910 struct tid_info *t;
3911 int i;
3912
3913 t = &sc->tids;
3914 if (t->natids == 0)
3915 return;
3916
3917 MPASS(t->atid_tab == NULL);
3918
3919 t->atid_tab = malloc(t->natids * sizeof(*t->atid_tab), M_CXGBE,
3920 M_ZERO | M_WAITOK);
3921 mtx_init(&t->atid_lock, "atid lock", NULL, MTX_DEF);
3922 t->afree = t->atid_tab;
3923 t->atids_in_use = 0;
3924 for (i = 1; i < t->natids; i++)
3925 t->atid_tab[i - 1].next = &t->atid_tab[i];
3926 t->atid_tab[t->natids - 1].next = NULL;
3927 }
3928
3929 static void
t4_free_atid_table(struct adapter * sc)3930 t4_free_atid_table(struct adapter *sc)
3931 {
3932 struct tid_info *t;
3933
3934 t = &sc->tids;
3935
3936 KASSERT(t->atids_in_use == 0,
3937 ("%s: %d atids still in use.", __func__, t->atids_in_use));
3938
3939 if (mtx_initialized(&t->atid_lock))
3940 mtx_destroy(&t->atid_lock);
3941 free(t->atid_tab, M_CXGBE);
3942 t->atid_tab = NULL;
3943 }
3944
3945 int
alloc_atid(struct adapter * sc,void * ctx)3946 alloc_atid(struct adapter *sc, void *ctx)
3947 {
3948 struct tid_info *t = &sc->tids;
3949 int atid = -1;
3950
3951 mtx_lock(&t->atid_lock);
3952 if (t->afree) {
3953 union aopen_entry *p = t->afree;
3954
3955 atid = p - t->atid_tab;
3956 MPASS(atid <= M_TID_TID);
3957 t->afree = p->next;
3958 p->data = ctx;
3959 t->atids_in_use++;
3960 }
3961 mtx_unlock(&t->atid_lock);
3962 return (atid);
3963 }
3964
3965 void *
lookup_atid(struct adapter * sc,int atid)3966 lookup_atid(struct adapter *sc, int atid)
3967 {
3968 struct tid_info *t = &sc->tids;
3969
3970 return (t->atid_tab[atid].data);
3971 }
3972
3973 void
free_atid(struct adapter * sc,int atid)3974 free_atid(struct adapter *sc, int atid)
3975 {
3976 struct tid_info *t = &sc->tids;
3977 union aopen_entry *p = &t->atid_tab[atid];
3978
3979 mtx_lock(&t->atid_lock);
3980 p->next = t->afree;
3981 t->afree = p;
3982 t->atids_in_use--;
3983 mtx_unlock(&t->atid_lock);
3984 }
3985
3986 static void
queue_tid_release(struct adapter * sc,int tid)3987 queue_tid_release(struct adapter *sc, int tid)
3988 {
3989
3990 CXGBE_UNIMPLEMENTED("deferred tid release");
3991 }
3992
3993 void
release_tid(struct adapter * sc,int tid,struct sge_wrq * ctrlq)3994 release_tid(struct adapter *sc, int tid, struct sge_wrq *ctrlq)
3995 {
3996 struct wrqe *wr;
3997 struct cpl_tid_release *req;
3998
3999 wr = alloc_wrqe(sizeof(*req), ctrlq);
4000 if (wr == NULL) {
4001 queue_tid_release(sc, tid); /* defer */
4002 return;
4003 }
4004 req = wrtod(wr);
4005
4006 INIT_TP_WR_MIT_CPL(req, CPL_TID_RELEASE, tid);
4007
4008 t4_wrq_tx(sc, wr);
4009 }
4010
4011 static int
t4_range_cmp(const void * a,const void * b)4012 t4_range_cmp(const void *a, const void *b)
4013 {
4014 return ((const struct t4_range *)a)->start -
4015 ((const struct t4_range *)b)->start;
4016 }
4017
4018 /*
4019 * Verify that the memory range specified by the addr/len pair is valid within
4020 * the card's address space.
4021 */
4022 static int
validate_mem_range(struct adapter * sc,uint32_t addr,uint32_t len)4023 validate_mem_range(struct adapter *sc, uint32_t addr, uint32_t len)
4024 {
4025 struct t4_range mem_ranges[4], *r, *next;
4026 uint32_t em, addr_len;
4027 int i, n, remaining;
4028
4029 /* Memory can only be accessed in naturally aligned 4 byte units */
4030 if (addr & 3 || len & 3 || len == 0)
4031 return (EINVAL);
4032
4033 /* Enabled memories */
4034 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4035
4036 r = &mem_ranges[0];
4037 n = 0;
4038 bzero(r, sizeof(mem_ranges));
4039 if (em & F_EDRAM0_ENABLE) {
4040 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4041 r->size = G_EDRAM0_SIZE(addr_len) << 20;
4042 if (r->size > 0) {
4043 r->start = G_EDRAM0_BASE(addr_len) << 20;
4044 if (addr >= r->start &&
4045 addr + len <= r->start + r->size)
4046 return (0);
4047 r++;
4048 n++;
4049 }
4050 }
4051 if (em & F_EDRAM1_ENABLE) {
4052 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4053 r->size = G_EDRAM1_SIZE(addr_len) << 20;
4054 if (r->size > 0) {
4055 r->start = G_EDRAM1_BASE(addr_len) << 20;
4056 if (addr >= r->start &&
4057 addr + len <= r->start + r->size)
4058 return (0);
4059 r++;
4060 n++;
4061 }
4062 }
4063 if (em & F_EXT_MEM_ENABLE) {
4064 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4065 r->size = G_EXT_MEM_SIZE(addr_len) << 20;
4066 if (r->size > 0) {
4067 r->start = G_EXT_MEM_BASE(addr_len) << 20;
4068 if (addr >= r->start &&
4069 addr + len <= r->start + r->size)
4070 return (0);
4071 r++;
4072 n++;
4073 }
4074 }
4075 if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) {
4076 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
4077 r->size = G_EXT_MEM1_SIZE(addr_len) << 20;
4078 if (r->size > 0) {
4079 r->start = G_EXT_MEM1_BASE(addr_len) << 20;
4080 if (addr >= r->start &&
4081 addr + len <= r->start + r->size)
4082 return (0);
4083 r++;
4084 n++;
4085 }
4086 }
4087 MPASS(n <= nitems(mem_ranges));
4088
4089 if (n > 1) {
4090 /* Sort and merge the ranges. */
4091 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp);
4092
4093 /* Start from index 0 and examine the next n - 1 entries. */
4094 r = &mem_ranges[0];
4095 for (remaining = n - 1; remaining > 0; remaining--, r++) {
4096
4097 MPASS(r->size > 0); /* r is a valid entry. */
4098 next = r + 1;
4099 MPASS(next->size > 0); /* and so is the next one. */
4100
4101 while (r->start + r->size >= next->start) {
4102 /* Merge the next one into the current entry. */
4103 r->size = max(r->start + r->size,
4104 next->start + next->size) - r->start;
4105 n--; /* One fewer entry in total. */
4106 if (--remaining == 0)
4107 goto done; /* short circuit */
4108 next++;
4109 }
4110 if (next != r + 1) {
4111 /*
4112 * Some entries were merged into r and next
4113 * points to the first valid entry that couldn't
4114 * be merged.
4115 */
4116 MPASS(next->size > 0); /* must be valid */
4117 memcpy(r + 1, next, remaining * sizeof(*r));
4118 #ifdef INVARIANTS
4119 /*
4120 * This so that the foo->size assertion in the
4121 * next iteration of the loop do the right
4122 * thing for entries that were pulled up and are
4123 * no longer valid.
4124 */
4125 MPASS(n < nitems(mem_ranges));
4126 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) *
4127 sizeof(struct t4_range));
4128 #endif
4129 }
4130 }
4131 done:
4132 /* Done merging the ranges. */
4133 MPASS(n > 0);
4134 r = &mem_ranges[0];
4135 for (i = 0; i < n; i++, r++) {
4136 if (addr >= r->start &&
4137 addr + len <= r->start + r->size)
4138 return (0);
4139 }
4140 }
4141
4142 return (EFAULT);
4143 }
4144
4145 static int
fwmtype_to_hwmtype(int mtype)4146 fwmtype_to_hwmtype(int mtype)
4147 {
4148
4149 switch (mtype) {
4150 case FW_MEMTYPE_EDC0:
4151 return (MEM_EDC0);
4152 case FW_MEMTYPE_EDC1:
4153 return (MEM_EDC1);
4154 case FW_MEMTYPE_EXTMEM:
4155 return (MEM_MC0);
4156 case FW_MEMTYPE_EXTMEM1:
4157 return (MEM_MC1);
4158 default:
4159 panic("%s: cannot translate fw mtype %d.", __func__, mtype);
4160 }
4161 }
4162
4163 /*
4164 * Verify that the memory range specified by the memtype/offset/len pair is
4165 * valid and lies entirely within the memtype specified. The global address of
4166 * the start of the range is returned in addr.
4167 */
4168 static int
validate_mt_off_len(struct adapter * sc,int mtype,uint32_t off,uint32_t len,uint32_t * addr)4169 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, uint32_t len,
4170 uint32_t *addr)
4171 {
4172 uint32_t em, addr_len, maddr;
4173
4174 /* Memory can only be accessed in naturally aligned 4 byte units */
4175 if (off & 3 || len & 3 || len == 0)
4176 return (EINVAL);
4177
4178 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4179 switch (fwmtype_to_hwmtype(mtype)) {
4180 case MEM_EDC0:
4181 if (!(em & F_EDRAM0_ENABLE))
4182 return (EINVAL);
4183 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4184 maddr = G_EDRAM0_BASE(addr_len) << 20;
4185 break;
4186 case MEM_EDC1:
4187 if (!(em & F_EDRAM1_ENABLE))
4188 return (EINVAL);
4189 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4190 maddr = G_EDRAM1_BASE(addr_len) << 20;
4191 break;
4192 case MEM_MC:
4193 if (!(em & F_EXT_MEM_ENABLE))
4194 return (EINVAL);
4195 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4196 maddr = G_EXT_MEM_BASE(addr_len) << 20;
4197 break;
4198 case MEM_MC1:
4199 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE))
4200 return (EINVAL);
4201 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
4202 maddr = G_EXT_MEM1_BASE(addr_len) << 20;
4203 break;
4204 default:
4205 return (EINVAL);
4206 }
4207
4208 *addr = maddr + off; /* global address */
4209 return (validate_mem_range(sc, *addr, len));
4210 }
4211
4212 static int
fixup_devlog_params(struct adapter * sc)4213 fixup_devlog_params(struct adapter *sc)
4214 {
4215 struct devlog_params *dparams = &sc->params.devlog;
4216 int rc;
4217
4218 rc = validate_mt_off_len(sc, dparams->memtype, dparams->start,
4219 dparams->size, &dparams->addr);
4220
4221 return (rc);
4222 }
4223
4224 static void
update_nirq(struct intrs_and_queues * iaq,int nports)4225 update_nirq(struct intrs_and_queues *iaq, int nports)
4226 {
4227
4228 iaq->nirq = T4_EXTRA_INTR;
4229 iaq->nirq += nports * max(iaq->nrxq, iaq->nnmrxq);
4230 iaq->nirq += nports * iaq->nofldrxq;
4231 iaq->nirq += nports * (iaq->num_vis - 1) *
4232 max(iaq->nrxq_vi, iaq->nnmrxq_vi);
4233 iaq->nirq += nports * (iaq->num_vis - 1) * iaq->nofldrxq_vi;
4234 }
4235
4236 /*
4237 * Adjust requirements to fit the number of interrupts available.
4238 */
4239 static void
calculate_iaq(struct adapter * sc,struct intrs_and_queues * iaq,int itype,int navail)4240 calculate_iaq(struct adapter *sc, struct intrs_and_queues *iaq, int itype,
4241 int navail)
4242 {
4243 int old_nirq;
4244 const int nports = sc->params.nports;
4245
4246 MPASS(nports > 0);
4247 MPASS(navail > 0);
4248
4249 bzero(iaq, sizeof(*iaq));
4250 iaq->intr_type = itype;
4251 iaq->num_vis = t4_num_vis;
4252 iaq->ntxq = t4_ntxq;
4253 iaq->ntxq_vi = t4_ntxq_vi;
4254 iaq->nrxq = t4_nrxq;
4255 iaq->nrxq_vi = t4_nrxq_vi;
4256 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
4257 if (is_offload(sc) || is_ethoffload(sc)) {
4258 iaq->nofldtxq = t4_nofldtxq;
4259 iaq->nofldtxq_vi = t4_nofldtxq_vi;
4260 }
4261 #endif
4262 #ifdef TCP_OFFLOAD
4263 if (is_offload(sc)) {
4264 iaq->nofldrxq = t4_nofldrxq;
4265 iaq->nofldrxq_vi = t4_nofldrxq_vi;
4266 }
4267 #endif
4268 #ifdef DEV_NETMAP
4269 if (t4_native_netmap & NN_MAIN_VI) {
4270 iaq->nnmtxq = t4_nnmtxq;
4271 iaq->nnmrxq = t4_nnmrxq;
4272 }
4273 if (t4_native_netmap & NN_EXTRA_VI) {
4274 iaq->nnmtxq_vi = t4_nnmtxq_vi;
4275 iaq->nnmrxq_vi = t4_nnmrxq_vi;
4276 }
4277 #endif
4278
4279 update_nirq(iaq, nports);
4280 if (iaq->nirq <= navail &&
4281 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4282 /*
4283 * This is the normal case -- there are enough interrupts for
4284 * everything.
4285 */
4286 goto done;
4287 }
4288
4289 /*
4290 * If extra VIs have been configured try reducing their count and see if
4291 * that works.
4292 */
4293 while (iaq->num_vis > 1) {
4294 iaq->num_vis--;
4295 update_nirq(iaq, nports);
4296 if (iaq->nirq <= navail &&
4297 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4298 device_printf(sc->dev, "virtual interfaces per port "
4299 "reduced to %d from %d. nrxq=%u, nofldrxq=%u, "
4300 "nrxq_vi=%u nofldrxq_vi=%u, nnmrxq_vi=%u. "
4301 "itype %d, navail %u, nirq %d.\n",
4302 iaq->num_vis, t4_num_vis, iaq->nrxq, iaq->nofldrxq,
4303 iaq->nrxq_vi, iaq->nofldrxq_vi, iaq->nnmrxq_vi,
4304 itype, navail, iaq->nirq);
4305 goto done;
4306 }
4307 }
4308
4309 /*
4310 * Extra VIs will not be created. Log a message if they were requested.
4311 */
4312 MPASS(iaq->num_vis == 1);
4313 iaq->ntxq_vi = iaq->nrxq_vi = 0;
4314 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0;
4315 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0;
4316 if (iaq->num_vis != t4_num_vis) {
4317 device_printf(sc->dev, "extra virtual interfaces disabled. "
4318 "nrxq=%u, nofldrxq=%u, nrxq_vi=%u nofldrxq_vi=%u, "
4319 "nnmrxq_vi=%u. itype %d, navail %u, nirq %d.\n",
4320 iaq->nrxq, iaq->nofldrxq, iaq->nrxq_vi, iaq->nofldrxq_vi,
4321 iaq->nnmrxq_vi, itype, navail, iaq->nirq);
4322 }
4323
4324 /*
4325 * Keep reducing the number of NIC rx queues to the next lower power of
4326 * 2 (for even RSS distribution) and halving the TOE rx queues and see
4327 * if that works.
4328 */
4329 do {
4330 if (iaq->nrxq > 1) {
4331 do {
4332 iaq->nrxq--;
4333 } while (!powerof2(iaq->nrxq));
4334 if (iaq->nnmrxq > iaq->nrxq)
4335 iaq->nnmrxq = iaq->nrxq;
4336 }
4337 if (iaq->nofldrxq > 1)
4338 iaq->nofldrxq >>= 1;
4339
4340 old_nirq = iaq->nirq;
4341 update_nirq(iaq, nports);
4342 if (iaq->nirq <= navail &&
4343 (itype != INTR_MSI || powerof2(iaq->nirq))) {
4344 device_printf(sc->dev, "running with reduced number of "
4345 "rx queues because of shortage of interrupts. "
4346 "nrxq=%u, nofldrxq=%u. "
4347 "itype %d, navail %u, nirq %d.\n", iaq->nrxq,
4348 iaq->nofldrxq, itype, navail, iaq->nirq);
4349 goto done;
4350 }
4351 } while (old_nirq != iaq->nirq);
4352
4353 /* One interrupt for everything. Ugh. */
4354 device_printf(sc->dev, "running with minimal number of queues. "
4355 "itype %d, navail %u.\n", itype, navail);
4356 iaq->nirq = 1;
4357 iaq->nrxq = 1;
4358 iaq->ntxq = 1;
4359 if (iaq->nofldrxq > 0) {
4360 iaq->nofldrxq = 1;
4361 iaq->nofldtxq = 1;
4362 }
4363 iaq->nnmtxq = 0;
4364 iaq->nnmrxq = 0;
4365 done:
4366 MPASS(iaq->num_vis > 0);
4367 if (iaq->num_vis > 1) {
4368 MPASS(iaq->nrxq_vi > 0);
4369 MPASS(iaq->ntxq_vi > 0);
4370 }
4371 MPASS(iaq->nirq > 0);
4372 MPASS(iaq->nrxq > 0);
4373 MPASS(iaq->ntxq > 0);
4374 if (itype == INTR_MSI) {
4375 MPASS(powerof2(iaq->nirq));
4376 }
4377 }
4378
4379 static int
cfg_itype_and_nqueues(struct adapter * sc,struct intrs_and_queues * iaq)4380 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq)
4381 {
4382 int rc, itype, navail, nalloc;
4383
4384 for (itype = INTR_MSIX; itype; itype >>= 1) {
4385
4386 if ((itype & t4_intr_types) == 0)
4387 continue; /* not allowed */
4388
4389 if (itype == INTR_MSIX)
4390 navail = pci_msix_count(sc->dev);
4391 else if (itype == INTR_MSI)
4392 navail = pci_msi_count(sc->dev);
4393 else
4394 navail = 1;
4395 restart:
4396 if (navail == 0)
4397 continue;
4398
4399 calculate_iaq(sc, iaq, itype, navail);
4400 nalloc = iaq->nirq;
4401 rc = 0;
4402 if (itype == INTR_MSIX)
4403 rc = pci_alloc_msix(sc->dev, &nalloc);
4404 else if (itype == INTR_MSI)
4405 rc = pci_alloc_msi(sc->dev, &nalloc);
4406
4407 if (rc == 0 && nalloc > 0) {
4408 if (nalloc == iaq->nirq)
4409 return (0);
4410
4411 /*
4412 * Didn't get the number requested. Use whatever number
4413 * the kernel is willing to allocate.
4414 */
4415 device_printf(sc->dev, "fewer vectors than requested, "
4416 "type=%d, req=%d, rcvd=%d; will downshift req.\n",
4417 itype, iaq->nirq, nalloc);
4418 pci_release_msi(sc->dev);
4419 navail = nalloc;
4420 goto restart;
4421 }
4422
4423 device_printf(sc->dev,
4424 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
4425 itype, rc, iaq->nirq, nalloc);
4426 }
4427
4428 device_printf(sc->dev,
4429 "failed to find a usable interrupt type. "
4430 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
4431 pci_msix_count(sc->dev), pci_msi_count(sc->dev));
4432
4433 return (ENXIO);
4434 }
4435
4436 #define FW_VERSION(chip) ( \
4437 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \
4438 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \
4439 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \
4440 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD))
4441 #define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf)
4442
4443 /* Just enough of fw_hdr to cover all version info. */
4444 struct fw_h {
4445 __u8 ver;
4446 __u8 chip;
4447 __be16 len512;
4448 __be32 fw_ver;
4449 __be32 tp_microcode_ver;
4450 __u8 intfver_nic;
4451 __u8 intfver_vnic;
4452 __u8 intfver_ofld;
4453 __u8 intfver_ri;
4454 __u8 intfver_iscsipdu;
4455 __u8 intfver_iscsi;
4456 __u8 intfver_fcoepdu;
4457 __u8 intfver_fcoe;
4458 };
4459 /* Spot check a couple of fields. */
4460 CTASSERT(offsetof(struct fw_h, fw_ver) == offsetof(struct fw_hdr, fw_ver));
4461 CTASSERT(offsetof(struct fw_h, intfver_nic) == offsetof(struct fw_hdr, intfver_nic));
4462 CTASSERT(offsetof(struct fw_h, intfver_fcoe) == offsetof(struct fw_hdr, intfver_fcoe));
4463
4464 struct fw_info {
4465 uint8_t chip;
4466 char *kld_name;
4467 char *fw_mod_name;
4468 struct fw_h fw_h;
4469 } fw_info[] = {
4470 {
4471 .chip = CHELSIO_T4,
4472 .kld_name = "t4fw_cfg",
4473 .fw_mod_name = "t4fw",
4474 .fw_h = {
4475 .chip = FW_HDR_CHIP_T4,
4476 .fw_ver = htobe32(FW_VERSION(T4)),
4477 .intfver_nic = FW_INTFVER(T4, NIC),
4478 .intfver_vnic = FW_INTFVER(T4, VNIC),
4479 .intfver_ofld = FW_INTFVER(T4, OFLD),
4480 .intfver_ri = FW_INTFVER(T4, RI),
4481 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU),
4482 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
4483 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU),
4484 .intfver_fcoe = FW_INTFVER(T4, FCOE),
4485 },
4486 }, {
4487 .chip = CHELSIO_T5,
4488 .kld_name = "t5fw_cfg",
4489 .fw_mod_name = "t5fw",
4490 .fw_h = {
4491 .chip = FW_HDR_CHIP_T5,
4492 .fw_ver = htobe32(FW_VERSION(T5)),
4493 .intfver_nic = FW_INTFVER(T5, NIC),
4494 .intfver_vnic = FW_INTFVER(T5, VNIC),
4495 .intfver_ofld = FW_INTFVER(T5, OFLD),
4496 .intfver_ri = FW_INTFVER(T5, RI),
4497 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU),
4498 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
4499 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU),
4500 .intfver_fcoe = FW_INTFVER(T5, FCOE),
4501 },
4502 }, {
4503 .chip = CHELSIO_T6,
4504 .kld_name = "t6fw_cfg",
4505 .fw_mod_name = "t6fw",
4506 .fw_h = {
4507 .chip = FW_HDR_CHIP_T6,
4508 .fw_ver = htobe32(FW_VERSION(T6)),
4509 .intfver_nic = FW_INTFVER(T6, NIC),
4510 .intfver_vnic = FW_INTFVER(T6, VNIC),
4511 .intfver_ofld = FW_INTFVER(T6, OFLD),
4512 .intfver_ri = FW_INTFVER(T6, RI),
4513 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU),
4514 .intfver_iscsi = FW_INTFVER(T6, ISCSI),
4515 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU),
4516 .intfver_fcoe = FW_INTFVER(T6, FCOE),
4517 },
4518 }
4519 };
4520
4521 static struct fw_info *
find_fw_info(int chip)4522 find_fw_info(int chip)
4523 {
4524 int i;
4525
4526 for (i = 0; i < nitems(fw_info); i++) {
4527 if (fw_info[i].chip == chip)
4528 return (&fw_info[i]);
4529 }
4530 return (NULL);
4531 }
4532
4533 /*
4534 * Is the given firmware API compatible with the one the driver was compiled
4535 * with?
4536 */
4537 static int
fw_compatible(const struct fw_h * hdr1,const struct fw_h * hdr2)4538 fw_compatible(const struct fw_h *hdr1, const struct fw_h *hdr2)
4539 {
4540
4541 /* short circuit if it's the exact same firmware version */
4542 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver)
4543 return (1);
4544
4545 /*
4546 * XXX: Is this too conservative? Perhaps I should limit this to the
4547 * features that are supported in the driver.
4548 */
4549 #define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x)
4550 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) &&
4551 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) &&
4552 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe))
4553 return (1);
4554 #undef SAME_INTF
4555
4556 return (0);
4557 }
4558
4559 static int
load_fw_module(struct adapter * sc,const struct firmware ** dcfg,const struct firmware ** fw)4560 load_fw_module(struct adapter *sc, const struct firmware **dcfg,
4561 const struct firmware **fw)
4562 {
4563 struct fw_info *fw_info;
4564
4565 *dcfg = NULL;
4566 if (fw != NULL)
4567 *fw = NULL;
4568
4569 fw_info = find_fw_info(chip_id(sc));
4570 if (fw_info == NULL) {
4571 device_printf(sc->dev,
4572 "unable to look up firmware information for chip %d.\n",
4573 chip_id(sc));
4574 return (EINVAL);
4575 }
4576
4577 *dcfg = firmware_get(fw_info->kld_name);
4578 if (*dcfg != NULL) {
4579 if (fw != NULL)
4580 *fw = firmware_get(fw_info->fw_mod_name);
4581 return (0);
4582 }
4583
4584 return (ENOENT);
4585 }
4586
4587 static void
unload_fw_module(struct adapter * sc,const struct firmware * dcfg,const struct firmware * fw)4588 unload_fw_module(struct adapter *sc, const struct firmware *dcfg,
4589 const struct firmware *fw)
4590 {
4591
4592 if (fw != NULL)
4593 firmware_put(fw, FIRMWARE_UNLOAD);
4594 if (dcfg != NULL)
4595 firmware_put(dcfg, FIRMWARE_UNLOAD);
4596 }
4597
4598 /*
4599 * Return values:
4600 * 0 means no firmware install attempted.
4601 * ERESTART means a firmware install was attempted and was successful.
4602 * +ve errno means a firmware install was attempted but failed.
4603 */
4604 static int
install_kld_firmware(struct adapter * sc,struct fw_h * card_fw,const struct fw_h * drv_fw,const char * reason,int * already)4605 install_kld_firmware(struct adapter *sc, struct fw_h *card_fw,
4606 const struct fw_h *drv_fw, const char *reason, int *already)
4607 {
4608 const struct firmware *cfg, *fw;
4609 const uint32_t c = be32toh(card_fw->fw_ver);
4610 uint32_t d, k;
4611 int rc, fw_install;
4612 struct fw_h bundled_fw;
4613 bool load_attempted;
4614
4615 cfg = fw = NULL;
4616 load_attempted = false;
4617 fw_install = t4_fw_install < 0 ? -t4_fw_install : t4_fw_install;
4618
4619 memcpy(&bundled_fw, drv_fw, sizeof(bundled_fw));
4620 if (t4_fw_install < 0) {
4621 rc = load_fw_module(sc, &cfg, &fw);
4622 if (rc != 0 || fw == NULL) {
4623 device_printf(sc->dev,
4624 "failed to load firmware module: %d. cfg %p, fw %p;"
4625 " will use compiled-in firmware version for"
4626 "hw.cxgbe.fw_install checks.\n",
4627 rc, cfg, fw);
4628 } else {
4629 memcpy(&bundled_fw, fw->data, sizeof(bundled_fw));
4630 }
4631 load_attempted = true;
4632 }
4633 d = be32toh(bundled_fw.fw_ver);
4634
4635 if (reason != NULL)
4636 goto install;
4637
4638 if ((sc->flags & FW_OK) == 0) {
4639
4640 if (c == 0xffffffff) {
4641 reason = "missing";
4642 goto install;
4643 }
4644
4645 rc = 0;
4646 goto done;
4647 }
4648
4649 if (!fw_compatible(card_fw, &bundled_fw)) {
4650 reason = "incompatible or unusable";
4651 goto install;
4652 }
4653
4654 if (d > c) {
4655 reason = "older than the version bundled with this driver";
4656 goto install;
4657 }
4658
4659 if (fw_install == 2 && d != c) {
4660 reason = "different than the version bundled with this driver";
4661 goto install;
4662 }
4663
4664 /* No reason to do anything to the firmware already on the card. */
4665 rc = 0;
4666 goto done;
4667
4668 install:
4669 rc = 0;
4670 if ((*already)++)
4671 goto done;
4672
4673 if (fw_install == 0) {
4674 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
4675 "but the driver is prohibited from installing a firmware "
4676 "on the card.\n",
4677 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
4678 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
4679
4680 goto done;
4681 }
4682
4683 /*
4684 * We'll attempt to install a firmware. Load the module first (if it
4685 * hasn't been loaded already).
4686 */
4687 if (!load_attempted) {
4688 rc = load_fw_module(sc, &cfg, &fw);
4689 if (rc != 0 || fw == NULL) {
4690 device_printf(sc->dev,
4691 "failed to load firmware module: %d. cfg %p, fw %p\n",
4692 rc, cfg, fw);
4693 /* carry on */
4694 }
4695 }
4696 if (fw == NULL) {
4697 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
4698 "but the driver cannot take corrective action because it "
4699 "is unable to load the firmware module.\n",
4700 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
4701 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason);
4702 rc = sc->flags & FW_OK ? 0 : ENOENT;
4703 goto done;
4704 }
4705 k = be32toh(((const struct fw_hdr *)fw->data)->fw_ver);
4706 if (k != d) {
4707 MPASS(t4_fw_install > 0);
4708 device_printf(sc->dev,
4709 "firmware in KLD (%u.%u.%u.%u) is not what the driver was "
4710 "expecting (%u.%u.%u.%u) and will not be used.\n",
4711 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k),
4712 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k),
4713 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
4714 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));
4715 rc = sc->flags & FW_OK ? 0 : EINVAL;
4716 goto done;
4717 }
4718
4719 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, "
4720 "installing firmware %u.%u.%u.%u on card.\n",
4721 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c),
4722 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason,
4723 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d),
4724 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d));
4725
4726 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0);
4727 if (rc != 0) {
4728 device_printf(sc->dev, "failed to install firmware: %d\n", rc);
4729 } else {
4730 /* Installed successfully, update the cached header too. */
4731 rc = ERESTART;
4732 memcpy(card_fw, fw->data, sizeof(*card_fw));
4733 }
4734 done:
4735 unload_fw_module(sc, cfg, fw);
4736
4737 return (rc);
4738 }
4739
4740 /*
4741 * Establish contact with the firmware and attempt to become the master driver.
4742 *
4743 * A firmware will be installed to the card if needed (if the driver is allowed
4744 * to do so).
4745 */
4746 static int
contact_firmware(struct adapter * sc)4747 contact_firmware(struct adapter *sc)
4748 {
4749 int rc, already = 0;
4750 enum dev_state state;
4751 struct fw_info *fw_info;
4752 struct fw_hdr *card_fw; /* fw on the card */
4753 const struct fw_h *drv_fw;
4754
4755 fw_info = find_fw_info(chip_id(sc));
4756 if (fw_info == NULL) {
4757 device_printf(sc->dev,
4758 "unable to look up firmware information for chip %d.\n",
4759 chip_id(sc));
4760 return (EINVAL);
4761 }
4762 drv_fw = &fw_info->fw_h;
4763
4764 /* Read the header of the firmware on the card */
4765 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK);
4766 restart:
4767 rc = -t4_get_fw_hdr(sc, card_fw);
4768 if (rc != 0) {
4769 device_printf(sc->dev,
4770 "unable to read firmware header from card's flash: %d\n",
4771 rc);
4772 goto done;
4773 }
4774
4775 rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw, NULL,
4776 &already);
4777 if (rc == ERESTART)
4778 goto restart;
4779 if (rc != 0)
4780 goto done;
4781
4782 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
4783 if (rc < 0 || state == DEV_STATE_ERR) {
4784 rc = -rc;
4785 device_printf(sc->dev,
4786 "failed to connect to the firmware: %d, %d. "
4787 "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
4788 #if 0
4789 if (install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw,
4790 "not responding properly to HELLO", &already) == ERESTART)
4791 goto restart;
4792 #endif
4793 goto done;
4794 }
4795 MPASS(be32toh(card_fw->flags) & FW_HDR_FLAGS_RESET_HALT);
4796 sc->flags |= FW_OK; /* The firmware responded to the FW_HELLO. */
4797
4798 if (rc == sc->pf) {
4799 sc->flags |= MASTER_PF;
4800 rc = install_kld_firmware(sc, (struct fw_h *)card_fw, drv_fw,
4801 NULL, &already);
4802 if (rc == ERESTART)
4803 rc = 0;
4804 else if (rc != 0)
4805 goto done;
4806 } else if (state == DEV_STATE_UNINIT) {
4807 /*
4808 * We didn't get to be the master so we definitely won't be
4809 * configuring the chip. It's a bug if someone else hasn't
4810 * configured it already.
4811 */
4812 device_printf(sc->dev, "couldn't be master(%d), "
4813 "device not already initialized either(%d). "
4814 "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
4815 rc = EPROTO;
4816 goto done;
4817 } else {
4818 /*
4819 * Some other PF is the master and has configured the chip.
4820 * This is allowed but untested.
4821 */
4822 device_printf(sc->dev, "PF%d is master, device state %d. "
4823 "PCIE_FW 0x%08x\n", rc, state, t4_read_reg(sc, A_PCIE_FW));
4824 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", rc);
4825 sc->cfcsum = 0;
4826 rc = 0;
4827 }
4828 done:
4829 if (rc != 0 && sc->flags & FW_OK) {
4830 t4_fw_bye(sc, sc->mbox);
4831 sc->flags &= ~FW_OK;
4832 }
4833 free(card_fw, M_CXGBE);
4834 return (rc);
4835 }
4836
4837 static int
copy_cfg_file_to_card(struct adapter * sc,char * cfg_file,uint32_t mtype,uint32_t moff)4838 copy_cfg_file_to_card(struct adapter *sc, char *cfg_file,
4839 uint32_t mtype, uint32_t moff)
4840 {
4841 struct fw_info *fw_info;
4842 const struct firmware *dcfg, *rcfg = NULL;
4843 const uint32_t *cfdata;
4844 uint32_t cflen, addr;
4845 int rc;
4846
4847 load_fw_module(sc, &dcfg, NULL);
4848
4849 /* Card specific interpretation of "default". */
4850 if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
4851 if (pci_get_device(sc->dev) == 0x440a)
4852 snprintf(cfg_file, sizeof(t4_cfg_file), UWIRE_CF);
4853 if (is_fpga(sc))
4854 snprintf(cfg_file, sizeof(t4_cfg_file), FPGA_CF);
4855 }
4856
4857 if (strncmp(cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) {
4858 if (dcfg == NULL) {
4859 device_printf(sc->dev,
4860 "KLD with default config is not available.\n");
4861 rc = ENOENT;
4862 goto done;
4863 }
4864 cfdata = dcfg->data;
4865 cflen = dcfg->datasize & ~3;
4866 } else {
4867 char s[32];
4868
4869 fw_info = find_fw_info(chip_id(sc));
4870 if (fw_info == NULL) {
4871 device_printf(sc->dev,
4872 "unable to look up firmware information for chip %d.\n",
4873 chip_id(sc));
4874 rc = EINVAL;
4875 goto done;
4876 }
4877 snprintf(s, sizeof(s), "%s_%s", fw_info->kld_name, cfg_file);
4878
4879 rcfg = firmware_get(s);
4880 if (rcfg == NULL) {
4881 device_printf(sc->dev,
4882 "unable to load module \"%s\" for configuration "
4883 "profile \"%s\".\n", s, cfg_file);
4884 rc = ENOENT;
4885 goto done;
4886 }
4887 cfdata = rcfg->data;
4888 cflen = rcfg->datasize & ~3;
4889 }
4890
4891 if (cflen > FLASH_CFG_MAX_SIZE) {
4892 device_printf(sc->dev,
4893 "config file too long (%d, max allowed is %d).\n",
4894 cflen, FLASH_CFG_MAX_SIZE);
4895 rc = EINVAL;
4896 goto done;
4897 }
4898
4899 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr);
4900 if (rc != 0) {
4901 device_printf(sc->dev,
4902 "%s: addr (%d/0x%x) or len %d is not valid: %d.\n",
4903 __func__, mtype, moff, cflen, rc);
4904 rc = EINVAL;
4905 goto done;
4906 }
4907 write_via_memwin(sc, 2, addr, cfdata, cflen);
4908 done:
4909 if (rcfg != NULL)
4910 firmware_put(rcfg, FIRMWARE_UNLOAD);
4911 unload_fw_module(sc, dcfg, NULL);
4912 return (rc);
4913 }
4914
4915 struct caps_allowed {
4916 uint16_t nbmcaps;
4917 uint16_t linkcaps;
4918 uint16_t switchcaps;
4919 uint16_t niccaps;
4920 uint16_t toecaps;
4921 uint16_t rdmacaps;
4922 uint16_t cryptocaps;
4923 uint16_t iscsicaps;
4924 uint16_t fcoecaps;
4925 };
4926
4927 #define FW_PARAM_DEV(param) \
4928 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
4929 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
4930 #define FW_PARAM_PFVF(param) \
4931 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
4932 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
4933
4934 /*
4935 * Provide a configuration profile to the firmware and have it initialize the
4936 * chip accordingly. This may involve uploading a configuration file to the
4937 * card.
4938 */
4939 static int
apply_cfg_and_initialize(struct adapter * sc,char * cfg_file,const struct caps_allowed * caps_allowed)4940 apply_cfg_and_initialize(struct adapter *sc, char *cfg_file,
4941 const struct caps_allowed *caps_allowed)
4942 {
4943 int rc;
4944 struct fw_caps_config_cmd caps;
4945 uint32_t mtype, moff, finicsum, cfcsum, param, val;
4946
4947 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
4948 if (rc != 0) {
4949 device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
4950 return (rc);
4951 }
4952
4953 bzero(&caps, sizeof(caps));
4954 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4955 F_FW_CMD_REQUEST | F_FW_CMD_READ);
4956 if (strncmp(cfg_file, BUILTIN_CF, sizeof(t4_cfg_file)) == 0) {
4957 mtype = 0;
4958 moff = 0;
4959 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
4960 } else if (strncmp(cfg_file, FLASH_CF, sizeof(t4_cfg_file)) == 0) {
4961 mtype = FW_MEMTYPE_FLASH;
4962 moff = t4_flash_cfg_addr(sc);
4963 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
4964 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4965 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) |
4966 FW_LEN16(caps));
4967 } else {
4968 /*
4969 * Ask the firmware where it wants us to upload the config file.
4970 */
4971 param = FW_PARAM_DEV(CF);
4972 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
4973 if (rc != 0) {
4974 /* No support for config file? Shouldn't happen. */
4975 device_printf(sc->dev,
4976 "failed to query config file location: %d.\n", rc);
4977 goto done;
4978 }
4979 mtype = G_FW_PARAMS_PARAM_Y(val);
4980 moff = G_FW_PARAMS_PARAM_Z(val) << 16;
4981 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
4982 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4983 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) |
4984 FW_LEN16(caps));
4985
4986 rc = copy_cfg_file_to_card(sc, cfg_file, mtype, moff);
4987 if (rc != 0) {
4988 device_printf(sc->dev,
4989 "failed to upload config file to card: %d.\n", rc);
4990 goto done;
4991 }
4992 }
4993 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
4994 if (rc != 0) {
4995 device_printf(sc->dev, "failed to pre-process config file: %d "
4996 "(mtype %d, moff 0x%x).\n", rc, mtype, moff);
4997 goto done;
4998 }
4999
5000 finicsum = be32toh(caps.finicsum);
5001 cfcsum = be32toh(caps.cfcsum); /* actual */
5002 if (finicsum != cfcsum) {
5003 device_printf(sc->dev,
5004 "WARNING: config file checksum mismatch: %08x %08x\n",
5005 finicsum, cfcsum);
5006 }
5007 sc->cfcsum = cfcsum;
5008 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", cfg_file);
5009
5010 /*
5011 * Let the firmware know what features will (not) be used so it can tune
5012 * things accordingly.
5013 */
5014 #define LIMIT_CAPS(x) do { \
5015 caps.x##caps &= htobe16(caps_allowed->x##caps); \
5016 } while (0)
5017 LIMIT_CAPS(nbm);
5018 LIMIT_CAPS(link);
5019 LIMIT_CAPS(switch);
5020 LIMIT_CAPS(nic);
5021 LIMIT_CAPS(toe);
5022 LIMIT_CAPS(rdma);
5023 LIMIT_CAPS(crypto);
5024 LIMIT_CAPS(iscsi);
5025 LIMIT_CAPS(fcoe);
5026 #undef LIMIT_CAPS
5027 if (caps.niccaps & htobe16(FW_CAPS_CONFIG_NIC_HASHFILTER)) {
5028 /*
5029 * TOE and hashfilters are mutually exclusive. It is a config
5030 * file or firmware bug if both are reported as available. Try
5031 * to cope with the situation in non-debug builds by disabling
5032 * TOE.
5033 */
5034 MPASS(caps.toecaps == 0);
5035
5036 caps.toecaps = 0;
5037 caps.rdmacaps = 0;
5038 caps.iscsicaps = 0;
5039 }
5040
5041 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5042 F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
5043 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
5044 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
5045 if (rc != 0) {
5046 device_printf(sc->dev,
5047 "failed to process config file: %d.\n", rc);
5048 goto done;
5049 }
5050
5051 t4_tweak_chip_settings(sc);
5052 set_params__pre_init(sc);
5053
5054 /* get basic stuff going */
5055 rc = -t4_fw_initialize(sc, sc->mbox);
5056 if (rc != 0) {
5057 device_printf(sc->dev, "fw_initialize failed: %d.\n", rc);
5058 goto done;
5059 }
5060 done:
5061 return (rc);
5062 }
5063
5064 /*
5065 * Partition chip resources for use between various PFs, VFs, etc.
5066 */
5067 static int
partition_resources(struct adapter * sc)5068 partition_resources(struct adapter *sc)
5069 {
5070 char cfg_file[sizeof(t4_cfg_file)];
5071 struct caps_allowed caps_allowed;
5072 int rc;
5073 bool fallback;
5074
5075 /* Only the master driver gets to configure the chip resources. */
5076 MPASS(sc->flags & MASTER_PF);
5077
5078 #define COPY_CAPS(x) do { \
5079 caps_allowed.x##caps = t4_##x##caps_allowed; \
5080 } while (0)
5081 bzero(&caps_allowed, sizeof(caps_allowed));
5082 COPY_CAPS(nbm);
5083 COPY_CAPS(link);
5084 COPY_CAPS(switch);
5085 COPY_CAPS(nic);
5086 COPY_CAPS(toe);
5087 COPY_CAPS(rdma);
5088 COPY_CAPS(crypto);
5089 COPY_CAPS(iscsi);
5090 COPY_CAPS(fcoe);
5091 fallback = sc->debug_flags & DF_DISABLE_CFG_RETRY ? false : true;
5092 snprintf(cfg_file, sizeof(cfg_file), "%s", t4_cfg_file);
5093 retry:
5094 rc = apply_cfg_and_initialize(sc, cfg_file, &caps_allowed);
5095 if (rc != 0 && fallback) {
5096 dump_devlog(sc);
5097 device_printf(sc->dev,
5098 "failed (%d) to configure card with \"%s\" profile, "
5099 "will fall back to a basic configuration and retry.\n",
5100 rc, cfg_file);
5101 snprintf(cfg_file, sizeof(cfg_file), "%s", BUILTIN_CF);
5102 bzero(&caps_allowed, sizeof(caps_allowed));
5103 COPY_CAPS(switch);
5104 caps_allowed.niccaps = FW_CAPS_CONFIG_NIC;
5105 fallback = false;
5106 goto retry;
5107 }
5108 #undef COPY_CAPS
5109 return (rc);
5110 }
5111
5112 /*
5113 * Retrieve parameters that are needed (or nice to have) very early.
5114 */
5115 static int
get_params__pre_init(struct adapter * sc)5116 get_params__pre_init(struct adapter *sc)
5117 {
5118 int rc;
5119 uint32_t param[2], val[2];
5120
5121 t4_get_version_info(sc);
5122
5123 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
5124 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
5125 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
5126 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
5127 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
5128
5129 snprintf(sc->bs_version, sizeof(sc->bs_version), "%u.%u.%u.%u",
5130 G_FW_HDR_FW_VER_MAJOR(sc->params.bs_vers),
5131 G_FW_HDR_FW_VER_MINOR(sc->params.bs_vers),
5132 G_FW_HDR_FW_VER_MICRO(sc->params.bs_vers),
5133 G_FW_HDR_FW_VER_BUILD(sc->params.bs_vers));
5134
5135 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
5136 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
5137 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
5138 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
5139 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
5140
5141 snprintf(sc->er_version, sizeof(sc->er_version), "%u.%u.%u.%u",
5142 G_FW_HDR_FW_VER_MAJOR(sc->params.er_vers),
5143 G_FW_HDR_FW_VER_MINOR(sc->params.er_vers),
5144 G_FW_HDR_FW_VER_MICRO(sc->params.er_vers),
5145 G_FW_HDR_FW_VER_BUILD(sc->params.er_vers));
5146
5147 param[0] = FW_PARAM_DEV(PORTVEC);
5148 param[1] = FW_PARAM_DEV(CCLK);
5149 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5150 if (rc != 0) {
5151 device_printf(sc->dev,
5152 "failed to query parameters (pre_init): %d.\n", rc);
5153 return (rc);
5154 }
5155
5156 sc->params.portvec = val[0];
5157 sc->params.nports = bitcount32(val[0]);
5158 sc->params.vpd.cclk = val[1];
5159
5160 /* Read device log parameters. */
5161 rc = -t4_init_devlog_params(sc, 1);
5162 if (rc == 0)
5163 fixup_devlog_params(sc);
5164 else {
5165 device_printf(sc->dev,
5166 "failed to get devlog parameters: %d.\n", rc);
5167 rc = 0; /* devlog isn't critical for device operation */
5168 }
5169
5170 return (rc);
5171 }
5172
5173 /*
5174 * Any params that need to be set before FW_INITIALIZE.
5175 */
5176 static int
set_params__pre_init(struct adapter * sc)5177 set_params__pre_init(struct adapter *sc)
5178 {
5179 int rc = 0;
5180 uint32_t param, val;
5181
5182 if (chip_id(sc) >= CHELSIO_T6) {
5183 param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT);
5184 val = 1;
5185 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5186 /* firmwares < 1.20.1.0 do not have this param. */
5187 if (rc == FW_EINVAL &&
5188 sc->params.fw_vers < FW_VERSION32(1, 20, 1, 0)) {
5189 rc = 0;
5190 }
5191 if (rc != 0) {
5192 device_printf(sc->dev,
5193 "failed to enable high priority filters :%d.\n",
5194 rc);
5195 }
5196
5197 param = FW_PARAM_DEV(PPOD_EDRAM);
5198 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5199 if (rc == 0 && val == 1) {
5200 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m,
5201 &val);
5202 if (rc != 0) {
5203 device_printf(sc->dev,
5204 "failed to set PPOD_EDRAM: %d.\n", rc);
5205 }
5206 }
5207 }
5208
5209 /* Enable opaque VIIDs with firmwares that support it. */
5210 param = FW_PARAM_DEV(OPAQUE_VIID_SMT_EXTN);
5211 val = 1;
5212 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5213 if (rc == 0 && val == 1)
5214 sc->params.viid_smt_extn_support = true;
5215 else
5216 sc->params.viid_smt_extn_support = false;
5217
5218 return (rc);
5219 }
5220
5221 /*
5222 * Retrieve various parameters that are of interest to the driver. The device
5223 * has been initialized by the firmware at this point.
5224 */
5225 static int
get_params__post_init(struct adapter * sc)5226 get_params__post_init(struct adapter *sc)
5227 {
5228 int rc;
5229 uint32_t param[7], val[7];
5230 struct fw_caps_config_cmd caps;
5231
5232 param[0] = FW_PARAM_PFVF(IQFLINT_START);
5233 param[1] = FW_PARAM_PFVF(EQ_START);
5234 param[2] = FW_PARAM_PFVF(FILTER_START);
5235 param[3] = FW_PARAM_PFVF(FILTER_END);
5236 param[4] = FW_PARAM_PFVF(L2T_START);
5237 param[5] = FW_PARAM_PFVF(L2T_END);
5238 param[6] = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5239 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
5240 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
5241 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 7, param, val);
5242 if (rc != 0) {
5243 device_printf(sc->dev,
5244 "failed to query parameters (post_init): %d.\n", rc);
5245 return (rc);
5246 }
5247
5248 sc->sge.iq_start = val[0];
5249 sc->sge.eq_start = val[1];
5250 if ((int)val[3] > (int)val[2]) {
5251 sc->tids.ftid_base = val[2];
5252 sc->tids.ftid_end = val[3];
5253 sc->tids.nftids = val[3] - val[2] + 1;
5254 }
5255 sc->vres.l2t.start = val[4];
5256 sc->vres.l2t.size = val[5] - val[4] + 1;
5257 KASSERT(sc->vres.l2t.size <= L2T_SIZE,
5258 ("%s: L2 table size (%u) larger than expected (%u)",
5259 __func__, sc->vres.l2t.size, L2T_SIZE));
5260 sc->params.core_vdd = val[6];
5261
5262 param[0] = FW_PARAM_PFVF(IQFLINT_END);
5263 param[1] = FW_PARAM_PFVF(EQ_END);
5264 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5265 if (rc != 0) {
5266 device_printf(sc->dev,
5267 "failed to query parameters (post_init2): %d.\n", rc);
5268 return (rc);
5269 }
5270 MPASS((int)val[0] >= sc->sge.iq_start);
5271 sc->sge.iqmap_sz = val[0] - sc->sge.iq_start + 1;
5272 MPASS((int)val[1] >= sc->sge.eq_start);
5273 sc->sge.eqmap_sz = val[1] - sc->sge.eq_start + 1;
5274
5275 if (chip_id(sc) >= CHELSIO_T6) {
5276
5277 sc->tids.tid_base = t4_read_reg(sc,
5278 A_LE_DB_ACTIVE_TABLE_START_INDEX);
5279
5280 param[0] = FW_PARAM_PFVF(HPFILTER_START);
5281 param[1] = FW_PARAM_PFVF(HPFILTER_END);
5282 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5283 if (rc != 0) {
5284 device_printf(sc->dev,
5285 "failed to query hpfilter parameters: %d.\n", rc);
5286 return (rc);
5287 }
5288 if ((int)val[1] > (int)val[0]) {
5289 sc->tids.hpftid_base = val[0];
5290 sc->tids.hpftid_end = val[1];
5291 sc->tids.nhpftids = val[1] - val[0] + 1;
5292
5293 /*
5294 * These should go off if the layout changes and the
5295 * driver needs to catch up.
5296 */
5297 MPASS(sc->tids.hpftid_base == 0);
5298 MPASS(sc->tids.tid_base == sc->tids.nhpftids);
5299 }
5300
5301 param[0] = FW_PARAM_PFVF(RAWF_START);
5302 param[1] = FW_PARAM_PFVF(RAWF_END);
5303 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5304 if (rc != 0) {
5305 device_printf(sc->dev,
5306 "failed to query rawf parameters: %d.\n", rc);
5307 return (rc);
5308 }
5309 if ((int)val[1] > (int)val[0]) {
5310 sc->rawf_base = val[0];
5311 sc->nrawf = val[1] - val[0] + 1;
5312 }
5313 }
5314
5315 /*
5316 * The parameters that follow may not be available on all firmwares. We
5317 * query them individually rather than in a compound query because old
5318 * firmwares fail the entire query if an unknown parameter is queried.
5319 */
5320
5321 /*
5322 * MPS buffer group configuration.
5323 */
5324 param[0] = FW_PARAM_DEV(MPSBGMAP);
5325 val[0] = 0;
5326 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5327 if (rc == 0)
5328 sc->params.mps_bg_map = val[0];
5329 else
5330 sc->params.mps_bg_map = UINT32_MAX; /* Not a legal value. */
5331
5332 param[0] = FW_PARAM_DEV(TPCHMAP);
5333 val[0] = 0;
5334 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5335 if (rc == 0)
5336 sc->params.tp_ch_map = val[0];
5337 else
5338 sc->params.tp_ch_map = UINT32_MAX; /* Not a legal value. */
5339
5340 /*
5341 * Determine whether the firmware supports the filter2 work request.
5342 */
5343 param[0] = FW_PARAM_DEV(FILTER2_WR);
5344 val[0] = 0;
5345 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5346 if (rc == 0)
5347 sc->params.filter2_wr_support = val[0] != 0;
5348 else
5349 sc->params.filter2_wr_support = 0;
5350
5351 /*
5352 * Find out whether we're allowed to use the ULPTX MEMWRITE DSGL.
5353 */
5354 param[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5355 val[0] = 0;
5356 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5357 if (rc == 0)
5358 sc->params.ulptx_memwrite_dsgl = val[0] != 0;
5359 else
5360 sc->params.ulptx_memwrite_dsgl = false;
5361
5362 /* FW_RI_FR_NSMR_TPTE_WR support */
5363 param[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR);
5364 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5365 if (rc == 0)
5366 sc->params.fr_nsmr_tpte_wr_support = val[0] != 0;
5367 else
5368 sc->params.fr_nsmr_tpte_wr_support = false;
5369
5370 /* Support for 512 SGL entries per FR MR. */
5371 param[0] = FW_PARAM_DEV(DEV_512SGL_MR);
5372 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5373 if (rc == 0)
5374 sc->params.dev_512sgl_mr = val[0] != 0;
5375 else
5376 sc->params.dev_512sgl_mr = false;
5377
5378 param[0] = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR);
5379 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5380 if (rc == 0)
5381 sc->params.max_pkts_per_eth_tx_pkts_wr = val[0];
5382 else
5383 sc->params.max_pkts_per_eth_tx_pkts_wr = 15;
5384
5385 param[0] = FW_PARAM_DEV(NUM_TM_CLASS);
5386 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5387 if (rc == 0) {
5388 MPASS(val[0] > 0 && val[0] < 256); /* nsched_cls is 8b */
5389 sc->params.nsched_cls = val[0];
5390 } else
5391 sc->params.nsched_cls = sc->chip_params->nsched_cls;
5392
5393 /* get capabilites */
5394 bzero(&caps, sizeof(caps));
5395 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5396 F_FW_CMD_REQUEST | F_FW_CMD_READ);
5397 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
5398 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
5399 if (rc != 0) {
5400 device_printf(sc->dev,
5401 "failed to get card capabilities: %d.\n", rc);
5402 return (rc);
5403 }
5404
5405 #define READ_CAPS(x) do { \
5406 sc->x = htobe16(caps.x); \
5407 } while (0)
5408 READ_CAPS(nbmcaps);
5409 READ_CAPS(linkcaps);
5410 READ_CAPS(switchcaps);
5411 READ_CAPS(niccaps);
5412 READ_CAPS(toecaps);
5413 READ_CAPS(rdmacaps);
5414 READ_CAPS(cryptocaps);
5415 READ_CAPS(iscsicaps);
5416 READ_CAPS(fcoecaps);
5417
5418 if (sc->niccaps & FW_CAPS_CONFIG_NIC_HASHFILTER) {
5419 MPASS(chip_id(sc) > CHELSIO_T4);
5420 MPASS(sc->toecaps == 0);
5421 sc->toecaps = 0;
5422
5423 param[0] = FW_PARAM_DEV(NTID);
5424 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, param, val);
5425 if (rc != 0) {
5426 device_printf(sc->dev,
5427 "failed to query HASHFILTER parameters: %d.\n", rc);
5428 return (rc);
5429 }
5430 sc->tids.ntids = val[0];
5431 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
5432 MPASS(sc->tids.ntids >= sc->tids.nhpftids);
5433 sc->tids.ntids -= sc->tids.nhpftids;
5434 }
5435 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
5436 sc->params.hash_filter = 1;
5437 }
5438 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) {
5439 param[0] = FW_PARAM_PFVF(ETHOFLD_START);
5440 param[1] = FW_PARAM_PFVF(ETHOFLD_END);
5441 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5442 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val);
5443 if (rc != 0) {
5444 device_printf(sc->dev,
5445 "failed to query NIC parameters: %d.\n", rc);
5446 return (rc);
5447 }
5448 if ((int)val[1] > (int)val[0]) {
5449 sc->tids.etid_base = val[0];
5450 sc->tids.etid_end = val[1];
5451 sc->tids.netids = val[1] - val[0] + 1;
5452 sc->params.eo_wr_cred = val[2];
5453 sc->params.ethoffload = 1;
5454 }
5455 }
5456 if (sc->toecaps) {
5457 /* query offload-related parameters */
5458 param[0] = FW_PARAM_DEV(NTID);
5459 param[1] = FW_PARAM_PFVF(SERVER_START);
5460 param[2] = FW_PARAM_PFVF(SERVER_END);
5461 param[3] = FW_PARAM_PFVF(TDDP_START);
5462 param[4] = FW_PARAM_PFVF(TDDP_END);
5463 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5464 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5465 if (rc != 0) {
5466 device_printf(sc->dev,
5467 "failed to query TOE parameters: %d.\n", rc);
5468 return (rc);
5469 }
5470 sc->tids.ntids = val[0];
5471 if (sc->params.fw_vers < FW_VERSION32(1, 20, 5, 0)) {
5472 MPASS(sc->tids.ntids >= sc->tids.nhpftids);
5473 sc->tids.ntids -= sc->tids.nhpftids;
5474 }
5475 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
5476 if ((int)val[2] > (int)val[1]) {
5477 sc->tids.stid_base = val[1];
5478 sc->tids.nstids = val[2] - val[1] + 1;
5479 }
5480 sc->vres.ddp.start = val[3];
5481 sc->vres.ddp.size = val[4] - val[3] + 1;
5482 sc->params.ofldq_wr_cred = val[5];
5483 sc->params.offload = 1;
5484 } else {
5485 /*
5486 * The firmware attempts memfree TOE configuration for -SO cards
5487 * and will report toecaps=0 if it runs out of resources (this
5488 * depends on the config file). It may not report 0 for other
5489 * capabilities dependent on the TOE in this case. Set them to
5490 * 0 here so that the driver doesn't bother tracking resources
5491 * that will never be used.
5492 */
5493 sc->iscsicaps = 0;
5494 sc->rdmacaps = 0;
5495 }
5496 if (sc->rdmacaps) {
5497 param[0] = FW_PARAM_PFVF(STAG_START);
5498 param[1] = FW_PARAM_PFVF(STAG_END);
5499 param[2] = FW_PARAM_PFVF(RQ_START);
5500 param[3] = FW_PARAM_PFVF(RQ_END);
5501 param[4] = FW_PARAM_PFVF(PBL_START);
5502 param[5] = FW_PARAM_PFVF(PBL_END);
5503 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5504 if (rc != 0) {
5505 device_printf(sc->dev,
5506 "failed to query RDMA parameters(1): %d.\n", rc);
5507 return (rc);
5508 }
5509 sc->vres.stag.start = val[0];
5510 sc->vres.stag.size = val[1] - val[0] + 1;
5511 sc->vres.rq.start = val[2];
5512 sc->vres.rq.size = val[3] - val[2] + 1;
5513 sc->vres.pbl.start = val[4];
5514 sc->vres.pbl.size = val[5] - val[4] + 1;
5515
5516 param[0] = FW_PARAM_PFVF(SQRQ_START);
5517 param[1] = FW_PARAM_PFVF(SQRQ_END);
5518 param[2] = FW_PARAM_PFVF(CQ_START);
5519 param[3] = FW_PARAM_PFVF(CQ_END);
5520 param[4] = FW_PARAM_PFVF(OCQ_START);
5521 param[5] = FW_PARAM_PFVF(OCQ_END);
5522 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
5523 if (rc != 0) {
5524 device_printf(sc->dev,
5525 "failed to query RDMA parameters(2): %d.\n", rc);
5526 return (rc);
5527 }
5528 sc->vres.qp.start = val[0];
5529 sc->vres.qp.size = val[1] - val[0] + 1;
5530 sc->vres.cq.start = val[2];
5531 sc->vres.cq.size = val[3] - val[2] + 1;
5532 sc->vres.ocq.start = val[4];
5533 sc->vres.ocq.size = val[5] - val[4] + 1;
5534
5535 param[0] = FW_PARAM_PFVF(SRQ_START);
5536 param[1] = FW_PARAM_PFVF(SRQ_END);
5537 param[2] = FW_PARAM_DEV(MAXORDIRD_QP);
5538 param[3] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5539 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
5540 if (rc != 0) {
5541 device_printf(sc->dev,
5542 "failed to query RDMA parameters(3): %d.\n", rc);
5543 return (rc);
5544 }
5545 sc->vres.srq.start = val[0];
5546 sc->vres.srq.size = val[1] - val[0] + 1;
5547 sc->params.max_ordird_qp = val[2];
5548 sc->params.max_ird_adapter = val[3];
5549 }
5550 if (sc->iscsicaps) {
5551 param[0] = FW_PARAM_PFVF(ISCSI_START);
5552 param[1] = FW_PARAM_PFVF(ISCSI_END);
5553 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5554 if (rc != 0) {
5555 device_printf(sc->dev,
5556 "failed to query iSCSI parameters: %d.\n", rc);
5557 return (rc);
5558 }
5559 sc->vres.iscsi.start = val[0];
5560 sc->vres.iscsi.size = val[1] - val[0] + 1;
5561 }
5562 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
5563 param[0] = FW_PARAM_PFVF(TLS_START);
5564 param[1] = FW_PARAM_PFVF(TLS_END);
5565 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
5566 if (rc != 0) {
5567 device_printf(sc->dev,
5568 "failed to query TLS parameters: %d.\n", rc);
5569 return (rc);
5570 }
5571 sc->vres.key.start = val[0];
5572 sc->vres.key.size = val[1] - val[0] + 1;
5573 }
5574
5575 /*
5576 * We've got the params we wanted to query directly from the firmware.
5577 * Grab some others via other means.
5578 */
5579 t4_init_sge_params(sc);
5580 t4_init_tp_params(sc);
5581 t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
5582 t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd);
5583
5584 rc = t4_verify_chip_settings(sc);
5585 if (rc != 0)
5586 return (rc);
5587 t4_init_rx_buf_info(sc);
5588
5589 return (rc);
5590 }
5591
5592 #ifdef KERN_TLS
5593 static void
ktls_tick(void * arg)5594 ktls_tick(void *arg)
5595 {
5596 struct adapter *sc;
5597 uint32_t tstamp;
5598
5599 sc = arg;
5600 tstamp = tcp_ts_getticks();
5601 t4_write_reg(sc, A_TP_SYNC_TIME_HI, tstamp >> 1);
5602 t4_write_reg(sc, A_TP_SYNC_TIME_LO, tstamp << 31);
5603 callout_schedule_sbt(&sc->ktls_tick, SBT_1MS, 0, C_HARDCLOCK);
5604 }
5605
5606 static int
t6_config_kern_tls(struct adapter * sc,bool enable)5607 t6_config_kern_tls(struct adapter *sc, bool enable)
5608 {
5609 int rc;
5610 uint32_t param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5611 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_KTLS_HW) |
5612 V_FW_PARAMS_PARAM_Y(enable ? 1 : 0) |
5613 V_FW_PARAMS_PARAM_Z(FW_PARAMS_PARAM_DEV_KTLS_HW_USER_ENABLE);
5614
5615 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, ¶m);
5616 if (rc != 0) {
5617 CH_ERR(sc, "failed to %s NIC TLS: %d\n",
5618 enable ? "enable" : "disable", rc);
5619 return (rc);
5620 }
5621
5622 if (enable) {
5623 sc->flags |= KERN_TLS_ON;
5624 callout_reset_sbt(&sc->ktls_tick, SBT_1MS, 0, ktls_tick, sc,
5625 C_HARDCLOCK);
5626 } else {
5627 sc->flags &= ~KERN_TLS_ON;
5628 callout_stop(&sc->ktls_tick);
5629 }
5630
5631 return (rc);
5632 }
5633 #endif
5634
5635 static int
set_params__post_init(struct adapter * sc)5636 set_params__post_init(struct adapter *sc)
5637 {
5638 uint32_t mask, param, val;
5639 #ifdef TCP_OFFLOAD
5640 int i, v, shift;
5641 #endif
5642
5643 /* ask for encapsulated CPLs */
5644 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5645 val = 1;
5646 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
5647
5648 /* Enable 32b port caps if the firmware supports it. */
5649 param = FW_PARAM_PFVF(PORT_CAPS32);
5650 val = 1;
5651 if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val) == 0)
5652 sc->params.port_caps32 = 1;
5653
5654 /* Let filter + maskhash steer to a part of the VI's RSS region. */
5655 val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
5656 t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER),
5657 V_MASKFILTER(val - 1));
5658
5659 mask = F_DROPERRORANY | F_DROPERRORMAC | F_DROPERRORIPVER |
5660 F_DROPERRORFRAG | F_DROPERRORATTACK | F_DROPERRORETHHDRLEN |
5661 F_DROPERRORIPHDRLEN | F_DROPERRORTCPHDRLEN | F_DROPERRORPKTLEN |
5662 F_DROPERRORTCPOPT | F_DROPERRORCSUMIP | F_DROPERRORCSUM;
5663 val = 0;
5664 if (chip_id(sc) < CHELSIO_T6 && t4_attack_filter != 0) {
5665 t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG, F_ATTACKFILTERENABLE,
5666 F_ATTACKFILTERENABLE);
5667 val |= F_DROPERRORATTACK;
5668 }
5669 if (t4_drop_ip_fragments != 0) {
5670 t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG, F_FRAGMENTDROP,
5671 F_FRAGMENTDROP);
5672 val |= F_DROPERRORFRAG;
5673 }
5674 if (t4_drop_pkts_with_l2_errors != 0)
5675 val |= F_DROPERRORMAC | F_DROPERRORETHHDRLEN;
5676 if (t4_drop_pkts_with_l3_errors != 0) {
5677 val |= F_DROPERRORIPVER | F_DROPERRORIPHDRLEN |
5678 F_DROPERRORCSUMIP;
5679 }
5680 if (t4_drop_pkts_with_l4_errors != 0) {
5681 val |= F_DROPERRORTCPHDRLEN | F_DROPERRORPKTLEN |
5682 F_DROPERRORTCPOPT | F_DROPERRORCSUM;
5683 }
5684 t4_set_reg_field(sc, A_TP_ERR_CONFIG, mask, val);
5685
5686 #ifdef TCP_OFFLOAD
5687 /*
5688 * Override the TOE timers with user provided tunables. This is not the
5689 * recommended way to change the timers (the firmware config file is) so
5690 * these tunables are not documented.
5691 *
5692 * All the timer tunables are in microseconds.
5693 */
5694 if (t4_toe_keepalive_idle != 0) {
5695 v = us_to_tcp_ticks(sc, t4_toe_keepalive_idle);
5696 v &= M_KEEPALIVEIDLE;
5697 t4_set_reg_field(sc, A_TP_KEEP_IDLE,
5698 V_KEEPALIVEIDLE(M_KEEPALIVEIDLE), V_KEEPALIVEIDLE(v));
5699 }
5700 if (t4_toe_keepalive_interval != 0) {
5701 v = us_to_tcp_ticks(sc, t4_toe_keepalive_interval);
5702 v &= M_KEEPALIVEINTVL;
5703 t4_set_reg_field(sc, A_TP_KEEP_INTVL,
5704 V_KEEPALIVEINTVL(M_KEEPALIVEINTVL), V_KEEPALIVEINTVL(v));
5705 }
5706 if (t4_toe_keepalive_count != 0) {
5707 v = t4_toe_keepalive_count & M_KEEPALIVEMAXR2;
5708 t4_set_reg_field(sc, A_TP_SHIFT_CNT,
5709 V_KEEPALIVEMAXR1(M_KEEPALIVEMAXR1) |
5710 V_KEEPALIVEMAXR2(M_KEEPALIVEMAXR2),
5711 V_KEEPALIVEMAXR1(1) | V_KEEPALIVEMAXR2(v));
5712 }
5713 if (t4_toe_rexmt_min != 0) {
5714 v = us_to_tcp_ticks(sc, t4_toe_rexmt_min);
5715 v &= M_RXTMIN;
5716 t4_set_reg_field(sc, A_TP_RXT_MIN,
5717 V_RXTMIN(M_RXTMIN), V_RXTMIN(v));
5718 }
5719 if (t4_toe_rexmt_max != 0) {
5720 v = us_to_tcp_ticks(sc, t4_toe_rexmt_max);
5721 v &= M_RXTMAX;
5722 t4_set_reg_field(sc, A_TP_RXT_MAX,
5723 V_RXTMAX(M_RXTMAX), V_RXTMAX(v));
5724 }
5725 if (t4_toe_rexmt_count != 0) {
5726 v = t4_toe_rexmt_count & M_RXTSHIFTMAXR2;
5727 t4_set_reg_field(sc, A_TP_SHIFT_CNT,
5728 V_RXTSHIFTMAXR1(M_RXTSHIFTMAXR1) |
5729 V_RXTSHIFTMAXR2(M_RXTSHIFTMAXR2),
5730 V_RXTSHIFTMAXR1(1) | V_RXTSHIFTMAXR2(v));
5731 }
5732 for (i = 0; i < nitems(t4_toe_rexmt_backoff); i++) {
5733 if (t4_toe_rexmt_backoff[i] != -1) {
5734 v = t4_toe_rexmt_backoff[i] & M_TIMERBACKOFFINDEX0;
5735 shift = (i & 3) << 3;
5736 t4_set_reg_field(sc, A_TP_TCP_BACKOFF_REG0 + (i & ~3),
5737 M_TIMERBACKOFFINDEX0 << shift, v << shift);
5738 }
5739 }
5740 #endif
5741
5742 /*
5743 * Limit TOE connections to 2 reassembly "islands". This is
5744 * required to permit migrating TOE connections to either
5745 * ULP_MODE_TCPDDP or UPL_MODE_TLS.
5746 */
5747 t4_tp_wr_bits_indirect(sc, A_TP_FRAG_CONFIG, V_PASSMODE(M_PASSMODE),
5748 V_PASSMODE(2));
5749
5750 #ifdef KERN_TLS
5751 if (is_ktls(sc)) {
5752 sc->tlst.inline_keys = t4_tls_inline_keys;
5753 sc->tlst.combo_wrs = t4_tls_combo_wrs;
5754 if (t4_kern_tls != 0 && is_t6(sc))
5755 t6_config_kern_tls(sc, true);
5756 }
5757 #endif
5758 return (0);
5759 }
5760
5761 #undef FW_PARAM_PFVF
5762 #undef FW_PARAM_DEV
5763
5764 static void
t4_set_desc(struct adapter * sc)5765 t4_set_desc(struct adapter *sc)
5766 {
5767 struct adapter_params *p = &sc->params;
5768
5769 device_set_descf(sc->dev, "Chelsio %s", p->vpd.id);
5770 }
5771
5772 static inline void
ifmedia_add4(struct ifmedia * ifm,int m)5773 ifmedia_add4(struct ifmedia *ifm, int m)
5774 {
5775
5776 ifmedia_add(ifm, m, 0, NULL);
5777 ifmedia_add(ifm, m | IFM_ETH_TXPAUSE, 0, NULL);
5778 ifmedia_add(ifm, m | IFM_ETH_RXPAUSE, 0, NULL);
5779 ifmedia_add(ifm, m | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE, 0, NULL);
5780 }
5781
5782 /*
5783 * This is the selected media, which is not quite the same as the active media.
5784 * The media line in ifconfig is "media: Ethernet selected (active)" if selected
5785 * and active are not the same, and "media: Ethernet selected" otherwise.
5786 */
5787 static void
set_current_media(struct port_info * pi)5788 set_current_media(struct port_info *pi)
5789 {
5790 struct link_config *lc;
5791 struct ifmedia *ifm;
5792 int mword;
5793 u_int speed;
5794
5795 PORT_LOCK_ASSERT_OWNED(pi);
5796
5797 /* Leave current media alone if it's already set to IFM_NONE. */
5798 ifm = &pi->media;
5799 if (ifm->ifm_cur != NULL &&
5800 IFM_SUBTYPE(ifm->ifm_cur->ifm_media) == IFM_NONE)
5801 return;
5802
5803 lc = &pi->link_cfg;
5804 if (lc->requested_aneg != AUTONEG_DISABLE &&
5805 lc->pcaps & FW_PORT_CAP32_ANEG) {
5806 ifmedia_set(ifm, IFM_ETHER | IFM_AUTO);
5807 return;
5808 }
5809 mword = IFM_ETHER | IFM_FDX;
5810 if (lc->requested_fc & PAUSE_TX)
5811 mword |= IFM_ETH_TXPAUSE;
5812 if (lc->requested_fc & PAUSE_RX)
5813 mword |= IFM_ETH_RXPAUSE;
5814 if (lc->requested_speed == 0)
5815 speed = port_top_speed(pi) * 1000; /* Gbps -> Mbps */
5816 else
5817 speed = lc->requested_speed;
5818 mword |= port_mword(pi, speed_to_fwcap(speed));
5819 ifmedia_set(ifm, mword);
5820 }
5821
5822 /*
5823 * Returns true if the ifmedia list for the port cannot change.
5824 */
5825 static bool
fixed_ifmedia(struct port_info * pi)5826 fixed_ifmedia(struct port_info *pi)
5827 {
5828
5829 return (pi->port_type == FW_PORT_TYPE_BT_SGMII ||
5830 pi->port_type == FW_PORT_TYPE_BT_XFI ||
5831 pi->port_type == FW_PORT_TYPE_BT_XAUI ||
5832 pi->port_type == FW_PORT_TYPE_KX4 ||
5833 pi->port_type == FW_PORT_TYPE_KX ||
5834 pi->port_type == FW_PORT_TYPE_KR ||
5835 pi->port_type == FW_PORT_TYPE_BP_AP ||
5836 pi->port_type == FW_PORT_TYPE_BP4_AP ||
5837 pi->port_type == FW_PORT_TYPE_BP40_BA ||
5838 pi->port_type == FW_PORT_TYPE_KR4_100G ||
5839 pi->port_type == FW_PORT_TYPE_KR_SFP28 ||
5840 pi->port_type == FW_PORT_TYPE_KR_XLAUI);
5841 }
5842
5843 static void
build_medialist(struct port_info * pi)5844 build_medialist(struct port_info *pi)
5845 {
5846 uint32_t ss, speed;
5847 int unknown, mword, bit;
5848 struct link_config *lc;
5849 struct ifmedia *ifm;
5850
5851 PORT_LOCK_ASSERT_OWNED(pi);
5852
5853 if (pi->flags & FIXED_IFMEDIA)
5854 return;
5855
5856 /*
5857 * Rebuild the ifmedia list.
5858 */
5859 ifm = &pi->media;
5860 ifmedia_removeall(ifm);
5861 lc = &pi->link_cfg;
5862 ss = G_FW_PORT_CAP32_SPEED(lc->pcaps); /* Supported Speeds */
5863 if (__predict_false(ss == 0)) { /* not supposed to happen. */
5864 MPASS(ss != 0);
5865 no_media:
5866 MPASS(LIST_EMPTY(&ifm->ifm_list));
5867 ifmedia_add(ifm, IFM_ETHER | IFM_NONE, 0, NULL);
5868 ifmedia_set(ifm, IFM_ETHER | IFM_NONE);
5869 return;
5870 }
5871
5872 unknown = 0;
5873 for (bit = S_FW_PORT_CAP32_SPEED; bit < fls(ss); bit++) {
5874 speed = 1 << bit;
5875 MPASS(speed & M_FW_PORT_CAP32_SPEED);
5876 if (ss & speed) {
5877 mword = port_mword(pi, speed);
5878 if (mword == IFM_NONE) {
5879 goto no_media;
5880 } else if (mword == IFM_UNKNOWN)
5881 unknown++;
5882 else
5883 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | mword);
5884 }
5885 }
5886 if (unknown > 0) /* Add one unknown for all unknown media types. */
5887 ifmedia_add4(ifm, IFM_ETHER | IFM_FDX | IFM_UNKNOWN);
5888 if (lc->pcaps & FW_PORT_CAP32_ANEG)
5889 ifmedia_add(ifm, IFM_ETHER | IFM_AUTO, 0, NULL);
5890
5891 set_current_media(pi);
5892 }
5893
5894 /*
5895 * Initialize the requested fields in the link config based on driver tunables.
5896 */
5897 static void
init_link_config(struct port_info * pi)5898 init_link_config(struct port_info *pi)
5899 {
5900 struct link_config *lc = &pi->link_cfg;
5901
5902 PORT_LOCK_ASSERT_OWNED(pi);
5903
5904 lc->requested_caps = 0;
5905 lc->requested_speed = 0;
5906
5907 if (t4_autoneg == 0)
5908 lc->requested_aneg = AUTONEG_DISABLE;
5909 else if (t4_autoneg == 1)
5910 lc->requested_aneg = AUTONEG_ENABLE;
5911 else
5912 lc->requested_aneg = AUTONEG_AUTO;
5913
5914 lc->requested_fc = t4_pause_settings & (PAUSE_TX | PAUSE_RX |
5915 PAUSE_AUTONEG);
5916
5917 if (t4_fec & FEC_AUTO)
5918 lc->requested_fec = FEC_AUTO;
5919 else if (t4_fec == 0)
5920 lc->requested_fec = FEC_NONE;
5921 else {
5922 /* -1 is handled by the FEC_AUTO block above and not here. */
5923 lc->requested_fec = t4_fec &
5924 (FEC_RS | FEC_BASER_RS | FEC_NONE | FEC_MODULE);
5925 if (lc->requested_fec == 0)
5926 lc->requested_fec = FEC_AUTO;
5927 }
5928 if (t4_force_fec < 0)
5929 lc->force_fec = -1;
5930 else if (t4_force_fec > 0)
5931 lc->force_fec = 1;
5932 else
5933 lc->force_fec = 0;
5934 }
5935
5936 /*
5937 * Makes sure that all requested settings comply with what's supported by the
5938 * port. Returns the number of settings that were invalid and had to be fixed.
5939 */
5940 static int
fixup_link_config(struct port_info * pi)5941 fixup_link_config(struct port_info *pi)
5942 {
5943 int n = 0;
5944 struct link_config *lc = &pi->link_cfg;
5945 uint32_t fwspeed;
5946
5947 PORT_LOCK_ASSERT_OWNED(pi);
5948
5949 /* Speed (when not autonegotiating) */
5950 if (lc->requested_speed != 0) {
5951 fwspeed = speed_to_fwcap(lc->requested_speed);
5952 if ((fwspeed & lc->pcaps) == 0) {
5953 n++;
5954 lc->requested_speed = 0;
5955 }
5956 }
5957
5958 /* Link autonegotiation */
5959 MPASS(lc->requested_aneg == AUTONEG_ENABLE ||
5960 lc->requested_aneg == AUTONEG_DISABLE ||
5961 lc->requested_aneg == AUTONEG_AUTO);
5962 if (lc->requested_aneg == AUTONEG_ENABLE &&
5963 !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
5964 n++;
5965 lc->requested_aneg = AUTONEG_AUTO;
5966 }
5967
5968 /* Flow control */
5969 MPASS((lc->requested_fc & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG)) == 0);
5970 if (lc->requested_fc & PAUSE_TX &&
5971 !(lc->pcaps & FW_PORT_CAP32_FC_TX)) {
5972 n++;
5973 lc->requested_fc &= ~PAUSE_TX;
5974 }
5975 if (lc->requested_fc & PAUSE_RX &&
5976 !(lc->pcaps & FW_PORT_CAP32_FC_RX)) {
5977 n++;
5978 lc->requested_fc &= ~PAUSE_RX;
5979 }
5980 if (!(lc->requested_fc & PAUSE_AUTONEG) &&
5981 !(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE)) {
5982 n++;
5983 lc->requested_fc |= PAUSE_AUTONEG;
5984 }
5985
5986 /* FEC */
5987 if ((lc->requested_fec & FEC_RS &&
5988 !(lc->pcaps & FW_PORT_CAP32_FEC_RS)) ||
5989 (lc->requested_fec & FEC_BASER_RS &&
5990 !(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS))) {
5991 n++;
5992 lc->requested_fec = FEC_AUTO;
5993 }
5994
5995 return (n);
5996 }
5997
5998 /*
5999 * Apply the requested L1 settings, which are expected to be valid, to the
6000 * hardware.
6001 */
6002 static int
apply_link_config(struct port_info * pi)6003 apply_link_config(struct port_info *pi)
6004 {
6005 struct adapter *sc = pi->adapter;
6006 struct link_config *lc = &pi->link_cfg;
6007 int rc;
6008
6009 #ifdef INVARIANTS
6010 ASSERT_SYNCHRONIZED_OP(sc);
6011 PORT_LOCK_ASSERT_OWNED(pi);
6012
6013 if (lc->requested_aneg == AUTONEG_ENABLE)
6014 MPASS(lc->pcaps & FW_PORT_CAP32_ANEG);
6015 if (!(lc->requested_fc & PAUSE_AUTONEG))
6016 MPASS(lc->pcaps & FW_PORT_CAP32_FORCE_PAUSE);
6017 if (lc->requested_fc & PAUSE_TX)
6018 MPASS(lc->pcaps & FW_PORT_CAP32_FC_TX);
6019 if (lc->requested_fc & PAUSE_RX)
6020 MPASS(lc->pcaps & FW_PORT_CAP32_FC_RX);
6021 if (lc->requested_fec & FEC_RS)
6022 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_RS);
6023 if (lc->requested_fec & FEC_BASER_RS)
6024 MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS);
6025 #endif
6026 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc);
6027 if (rc != 0) {
6028 /* Don't complain if the VF driver gets back an EPERM. */
6029 if (!(sc->flags & IS_VF) || rc != FW_EPERM)
6030 device_printf(pi->dev, "l1cfg failed: %d\n", rc);
6031 } else {
6032 /*
6033 * An L1_CFG will almost always result in a link-change event if
6034 * the link is up, and the driver will refresh the actual
6035 * fec/fc/etc. when the notification is processed. If the link
6036 * is down then the actual settings are meaningless.
6037 *
6038 * This takes care of the case where a change in the L1 settings
6039 * may not result in a notification.
6040 */
6041 if (lc->link_ok && !(lc->requested_fc & PAUSE_AUTONEG))
6042 lc->fc = lc->requested_fc & (PAUSE_TX | PAUSE_RX);
6043 }
6044 return (rc);
6045 }
6046
6047 #define FW_MAC_EXACT_CHUNK 7
6048 struct mcaddr_ctx {
6049 if_t ifp;
6050 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
6051 uint64_t hash;
6052 int i;
6053 int del;
6054 int rc;
6055 };
6056
6057 static u_int
add_maddr(void * arg,struct sockaddr_dl * sdl,u_int cnt)6058 add_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt)
6059 {
6060 struct mcaddr_ctx *ctx = arg;
6061 struct vi_info *vi = if_getsoftc(ctx->ifp);
6062 struct port_info *pi = vi->pi;
6063 struct adapter *sc = pi->adapter;
6064
6065 if (ctx->rc < 0)
6066 return (0);
6067
6068 ctx->mcaddr[ctx->i] = LLADDR(sdl);
6069 MPASS(ETHER_IS_MULTICAST(ctx->mcaddr[ctx->i]));
6070 ctx->i++;
6071
6072 if (ctx->i == FW_MAC_EXACT_CHUNK) {
6073 ctx->rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, ctx->del,
6074 ctx->i, ctx->mcaddr, NULL, &ctx->hash, 0);
6075 if (ctx->rc < 0) {
6076 int j;
6077
6078 for (j = 0; j < ctx->i; j++) {
6079 if_printf(ctx->ifp,
6080 "failed to add mc address"
6081 " %02x:%02x:%02x:"
6082 "%02x:%02x:%02x rc=%d\n",
6083 ctx->mcaddr[j][0], ctx->mcaddr[j][1],
6084 ctx->mcaddr[j][2], ctx->mcaddr[j][3],
6085 ctx->mcaddr[j][4], ctx->mcaddr[j][5],
6086 -ctx->rc);
6087 }
6088 return (0);
6089 }
6090 ctx->del = 0;
6091 ctx->i = 0;
6092 }
6093
6094 return (1);
6095 }
6096
6097 /*
6098 * Program the port's XGMAC based on parameters in ifnet. The caller also
6099 * indicates which parameters should be programmed (the rest are left alone).
6100 */
6101 int
update_mac_settings(if_t ifp,int flags)6102 update_mac_settings(if_t ifp, int flags)
6103 {
6104 int rc = 0;
6105 struct vi_info *vi = if_getsoftc(ifp);
6106 struct port_info *pi = vi->pi;
6107 struct adapter *sc = pi->adapter;
6108 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
6109 uint8_t match_all_mac[ETHER_ADDR_LEN] = {0};
6110
6111 ASSERT_SYNCHRONIZED_OP(sc);
6112 KASSERT(flags, ("%s: not told what to update.", __func__));
6113
6114 if (flags & XGMAC_MTU)
6115 mtu = if_getmtu(ifp);
6116
6117 if (flags & XGMAC_PROMISC)
6118 promisc = if_getflags(ifp) & IFF_PROMISC ? 1 : 0;
6119
6120 if (flags & XGMAC_ALLMULTI)
6121 allmulti = if_getflags(ifp) & IFF_ALLMULTI ? 1 : 0;
6122
6123 if (flags & XGMAC_VLANEX)
6124 vlanex = if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING ? 1 : 0;
6125
6126 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) {
6127 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc,
6128 allmulti, 1, vlanex, false);
6129 if (rc) {
6130 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags,
6131 rc);
6132 return (rc);
6133 }
6134 }
6135
6136 if (flags & XGMAC_UCADDR) {
6137 uint8_t ucaddr[ETHER_ADDR_LEN];
6138
6139 bcopy(if_getlladdr(ifp), ucaddr, sizeof(ucaddr));
6140 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt,
6141 ucaddr, true, &vi->smt_idx);
6142 if (rc < 0) {
6143 rc = -rc;
6144 if_printf(ifp, "change_mac failed: %d\n", rc);
6145 return (rc);
6146 } else {
6147 vi->xact_addr_filt = rc;
6148 rc = 0;
6149 }
6150 }
6151
6152 if (flags & XGMAC_MCADDRS) {
6153 struct epoch_tracker et;
6154 struct mcaddr_ctx ctx;
6155 int j;
6156
6157 ctx.ifp = ifp;
6158 ctx.hash = 0;
6159 ctx.i = 0;
6160 ctx.del = 1;
6161 ctx.rc = 0;
6162 /*
6163 * Unlike other drivers, we accumulate list of pointers into
6164 * interface address lists and we need to keep it safe even
6165 * after if_foreach_llmaddr() returns, thus we must enter the
6166 * network epoch.
6167 */
6168 NET_EPOCH_ENTER(et);
6169 if_foreach_llmaddr(ifp, add_maddr, &ctx);
6170 if (ctx.rc < 0) {
6171 NET_EPOCH_EXIT(et);
6172 rc = -ctx.rc;
6173 return (rc);
6174 }
6175 if (ctx.i > 0) {
6176 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid,
6177 ctx.del, ctx.i, ctx.mcaddr, NULL, &ctx.hash, 0);
6178 NET_EPOCH_EXIT(et);
6179 if (rc < 0) {
6180 rc = -rc;
6181 for (j = 0; j < ctx.i; j++) {
6182 if_printf(ifp,
6183 "failed to add mcast address"
6184 " %02x:%02x:%02x:"
6185 "%02x:%02x:%02x rc=%d\n",
6186 ctx.mcaddr[j][0], ctx.mcaddr[j][1],
6187 ctx.mcaddr[j][2], ctx.mcaddr[j][3],
6188 ctx.mcaddr[j][4], ctx.mcaddr[j][5],
6189 rc);
6190 }
6191 return (rc);
6192 }
6193 ctx.del = 0;
6194 } else
6195 NET_EPOCH_EXIT(et);
6196
6197 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, ctx.hash, 0);
6198 if (rc != 0)
6199 if_printf(ifp, "failed to set mcast address hash: %d\n",
6200 rc);
6201 if (ctx.del == 0) {
6202 /* We clobbered the VXLAN entry if there was one. */
6203 pi->vxlan_tcam_entry = false;
6204 }
6205 }
6206
6207 if (IS_MAIN_VI(vi) && sc->vxlan_refcount > 0 &&
6208 pi->vxlan_tcam_entry == false) {
6209 rc = t4_alloc_raw_mac_filt(sc, vi->viid, match_all_mac,
6210 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id,
6211 true);
6212 if (rc < 0) {
6213 rc = -rc;
6214 if_printf(ifp, "failed to add VXLAN TCAM entry: %d.\n",
6215 rc);
6216 } else {
6217 MPASS(rc == sc->rawf_base + pi->port_id);
6218 rc = 0;
6219 pi->vxlan_tcam_entry = true;
6220 }
6221 }
6222
6223 return (rc);
6224 }
6225
6226 /*
6227 * {begin|end}_synchronized_op must be called from the same thread.
6228 */
6229 int
begin_synchronized_op(struct adapter * sc,struct vi_info * vi,int flags,char * wmesg)6230 begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags,
6231 char *wmesg)
6232 {
6233 int rc, pri;
6234
6235 #ifdef WITNESS
6236 /* the caller thinks it's ok to sleep, but is it really? */
6237 if (flags & SLEEP_OK)
6238 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
6239 "begin_synchronized_op");
6240 #endif
6241
6242 if (INTR_OK)
6243 pri = PCATCH;
6244 else
6245 pri = 0;
6246
6247 ADAPTER_LOCK(sc);
6248 for (;;) {
6249
6250 if (vi && IS_DETACHING(vi)) {
6251 rc = ENXIO;
6252 goto done;
6253 }
6254
6255 if (!IS_BUSY(sc)) {
6256 rc = 0;
6257 break;
6258 }
6259
6260 if (!(flags & SLEEP_OK)) {
6261 rc = EBUSY;
6262 goto done;
6263 }
6264
6265 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
6266 rc = EINTR;
6267 goto done;
6268 }
6269 }
6270
6271 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
6272 SET_BUSY(sc);
6273 #ifdef INVARIANTS
6274 sc->last_op = wmesg;
6275 sc->last_op_thr = curthread;
6276 sc->last_op_flags = flags;
6277 #endif
6278
6279 done:
6280 if (!(flags & HOLD_LOCK) || rc)
6281 ADAPTER_UNLOCK(sc);
6282
6283 return (rc);
6284 }
6285
6286 /*
6287 * Tell if_ioctl and if_init that the VI is going away. This is
6288 * special variant of begin_synchronized_op and must be paired with a
6289 * call to end_vi_detach.
6290 */
6291 void
begin_vi_detach(struct adapter * sc,struct vi_info * vi)6292 begin_vi_detach(struct adapter *sc, struct vi_info *vi)
6293 {
6294 ADAPTER_LOCK(sc);
6295 SET_DETACHING(vi);
6296 wakeup(&sc->flags);
6297 while (IS_BUSY(sc))
6298 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
6299 SET_BUSY(sc);
6300 #ifdef INVARIANTS
6301 sc->last_op = "t4detach";
6302 sc->last_op_thr = curthread;
6303 sc->last_op_flags = 0;
6304 #endif
6305 ADAPTER_UNLOCK(sc);
6306 }
6307
6308 void
end_vi_detach(struct adapter * sc,struct vi_info * vi)6309 end_vi_detach(struct adapter *sc, struct vi_info *vi)
6310 {
6311 ADAPTER_LOCK(sc);
6312 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
6313 CLR_BUSY(sc);
6314 CLR_DETACHING(vi);
6315 wakeup(&sc->flags);
6316 ADAPTER_UNLOCK(sc);
6317 }
6318
6319 /*
6320 * {begin|end}_synchronized_op must be called from the same thread.
6321 */
6322 void
end_synchronized_op(struct adapter * sc,int flags)6323 end_synchronized_op(struct adapter *sc, int flags)
6324 {
6325
6326 if (flags & LOCK_HELD)
6327 ADAPTER_LOCK_ASSERT_OWNED(sc);
6328 else
6329 ADAPTER_LOCK(sc);
6330
6331 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
6332 CLR_BUSY(sc);
6333 wakeup(&sc->flags);
6334 ADAPTER_UNLOCK(sc);
6335 }
6336
6337 static int
cxgbe_init_synchronized(struct vi_info * vi)6338 cxgbe_init_synchronized(struct vi_info *vi)
6339 {
6340 struct port_info *pi = vi->pi;
6341 struct adapter *sc = pi->adapter;
6342 if_t ifp = vi->ifp;
6343 int rc = 0, i;
6344 struct sge_txq *txq;
6345
6346 ASSERT_SYNCHRONIZED_OP(sc);
6347
6348 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
6349 return (0); /* already running */
6350
6351 if (!(sc->flags & FULL_INIT_DONE) && ((rc = adapter_init(sc)) != 0))
6352 return (rc); /* error message displayed already */
6353
6354 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0))
6355 return (rc); /* error message displayed already */
6356
6357 rc = update_mac_settings(ifp, XGMAC_ALL);
6358 if (rc)
6359 goto done; /* error message displayed already */
6360
6361 PORT_LOCK(pi);
6362 if (pi->up_vis == 0) {
6363 t4_update_port_info(pi);
6364 fixup_link_config(pi);
6365 build_medialist(pi);
6366 apply_link_config(pi);
6367 }
6368
6369 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true);
6370 if (rc != 0) {
6371 if_printf(ifp, "enable_vi failed: %d\n", rc);
6372 PORT_UNLOCK(pi);
6373 goto done;
6374 }
6375
6376 /*
6377 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized
6378 * if this changes.
6379 */
6380
6381 for_each_txq(vi, i, txq) {
6382 TXQ_LOCK(txq);
6383 txq->eq.flags |= EQ_ENABLED;
6384 TXQ_UNLOCK(txq);
6385 }
6386
6387 /*
6388 * The first iq of the first port to come up is used for tracing.
6389 */
6390 if (sc->traceq < 0 && IS_MAIN_VI(vi)) {
6391 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id;
6392 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL :
6393 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) |
6394 V_QUEUENUMBER(sc->traceq));
6395 pi->flags |= HAS_TRACEQ;
6396 }
6397
6398 /* all ok */
6399 pi->up_vis++;
6400 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
6401 if (pi->link_cfg.link_ok)
6402 t4_os_link_changed(pi);
6403 PORT_UNLOCK(pi);
6404
6405 mtx_lock(&vi->tick_mtx);
6406 if (vi->pi->nvi > 1 || sc->flags & IS_VF)
6407 callout_reset(&vi->tick, hz, vi_tick, vi);
6408 else
6409 callout_reset(&vi->tick, hz, cxgbe_tick, vi);
6410 mtx_unlock(&vi->tick_mtx);
6411 done:
6412 if (rc != 0)
6413 cxgbe_uninit_synchronized(vi);
6414
6415 return (rc);
6416 }
6417
6418 /*
6419 * Idempotent.
6420 */
6421 static int
cxgbe_uninit_synchronized(struct vi_info * vi)6422 cxgbe_uninit_synchronized(struct vi_info *vi)
6423 {
6424 struct port_info *pi = vi->pi;
6425 struct adapter *sc = pi->adapter;
6426 if_t ifp = vi->ifp;
6427 int rc, i;
6428 struct sge_txq *txq;
6429
6430 ASSERT_SYNCHRONIZED_OP(sc);
6431
6432 if (!(vi->flags & VI_INIT_DONE)) {
6433 if (__predict_false(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
6434 KASSERT(0, ("uninited VI is running"));
6435 if_printf(ifp, "uninited VI with running ifnet. "
6436 "vi->flags 0x%016lx, if_flags 0x%08x, "
6437 "if_drv_flags 0x%08x\n", vi->flags, if_getflags(ifp),
6438 if_getdrvflags(ifp));
6439 }
6440 return (0);
6441 }
6442
6443 /*
6444 * Disable the VI so that all its data in either direction is discarded
6445 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz
6446 * tick) intact as the TP can deliver negative advice or data that it's
6447 * holding in its RAM (for an offloaded connection) even after the VI is
6448 * disabled.
6449 */
6450 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false);
6451 if (rc) {
6452 if_printf(ifp, "disable_vi failed: %d\n", rc);
6453 return (rc);
6454 }
6455
6456 for_each_txq(vi, i, txq) {
6457 TXQ_LOCK(txq);
6458 txq->eq.flags &= ~EQ_ENABLED;
6459 TXQ_UNLOCK(txq);
6460 }
6461
6462 mtx_lock(&vi->tick_mtx);
6463 callout_stop(&vi->tick);
6464 mtx_unlock(&vi->tick_mtx);
6465
6466 PORT_LOCK(pi);
6467 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
6468 PORT_UNLOCK(pi);
6469 return (0);
6470 }
6471 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
6472 pi->up_vis--;
6473 if (pi->up_vis > 0) {
6474 PORT_UNLOCK(pi);
6475 return (0);
6476 }
6477
6478 pi->link_cfg.link_ok = false;
6479 pi->link_cfg.speed = 0;
6480 pi->link_cfg.link_down_rc = 255;
6481 t4_os_link_changed(pi);
6482 PORT_UNLOCK(pi);
6483
6484 return (0);
6485 }
6486
6487 /*
6488 * It is ok for this function to fail midway and return right away. t4_detach
6489 * will walk the entire sc->irq list and clean up whatever is valid.
6490 */
6491 int
t4_setup_intr_handlers(struct adapter * sc)6492 t4_setup_intr_handlers(struct adapter *sc)
6493 {
6494 int rc, rid, p, q, v;
6495 char s[8];
6496 struct irq *irq;
6497 struct port_info *pi;
6498 struct vi_info *vi;
6499 struct sge *sge = &sc->sge;
6500 struct sge_rxq *rxq;
6501 #ifdef TCP_OFFLOAD
6502 struct sge_ofld_rxq *ofld_rxq;
6503 #endif
6504 #ifdef DEV_NETMAP
6505 struct sge_nm_rxq *nm_rxq;
6506 #endif
6507 #ifdef RSS
6508 int nbuckets = rss_getnumbuckets();
6509 #endif
6510
6511 /*
6512 * Setup interrupts.
6513 */
6514 irq = &sc->irq[0];
6515 rid = sc->intr_type == INTR_INTX ? 0 : 1;
6516 if (forwarding_intr_to_fwq(sc))
6517 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"));
6518
6519 /* Multiple interrupts. */
6520 if (sc->flags & IS_VF)
6521 KASSERT(sc->intr_count >= T4VF_EXTRA_INTR + sc->params.nports,
6522 ("%s: too few intr.", __func__));
6523 else
6524 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
6525 ("%s: too few intr.", __func__));
6526
6527 /* The first one is always error intr on PFs */
6528 if (!(sc->flags & IS_VF)) {
6529 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
6530 if (rc != 0)
6531 return (rc);
6532 irq++;
6533 rid++;
6534 }
6535
6536 /* The second one is always the firmware event queue (first on VFs) */
6537 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt");
6538 if (rc != 0)
6539 return (rc);
6540 irq++;
6541 rid++;
6542
6543 for_each_port(sc, p) {
6544 pi = sc->port[p];
6545 for_each_vi(pi, v, vi) {
6546 vi->first_intr = rid - 1;
6547
6548 if (vi->nnmrxq > 0) {
6549 int n = max(vi->nrxq, vi->nnmrxq);
6550
6551 rxq = &sge->rxq[vi->first_rxq];
6552 #ifdef DEV_NETMAP
6553 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq];
6554 #endif
6555 for (q = 0; q < n; q++) {
6556 snprintf(s, sizeof(s), "%x%c%x", p,
6557 'a' + v, q);
6558 if (q < vi->nrxq)
6559 irq->rxq = rxq++;
6560 #ifdef DEV_NETMAP
6561 if (q < vi->nnmrxq)
6562 irq->nm_rxq = nm_rxq++;
6563
6564 if (irq->nm_rxq != NULL &&
6565 irq->rxq == NULL) {
6566 /* Netmap rx only */
6567 rc = t4_alloc_irq(sc, irq, rid,
6568 t4_nm_intr, irq->nm_rxq, s);
6569 }
6570 if (irq->nm_rxq != NULL &&
6571 irq->rxq != NULL) {
6572 /* NIC and Netmap rx */
6573 rc = t4_alloc_irq(sc, irq, rid,
6574 t4_vi_intr, irq, s);
6575 }
6576 #endif
6577 if (irq->rxq != NULL &&
6578 irq->nm_rxq == NULL) {
6579 /* NIC rx only */
6580 rc = t4_alloc_irq(sc, irq, rid,
6581 t4_intr, irq->rxq, s);
6582 }
6583 if (rc != 0)
6584 return (rc);
6585 #ifdef RSS
6586 if (q < vi->nrxq) {
6587 bus_bind_intr(sc->dev, irq->res,
6588 rss_getcpu(q % nbuckets));
6589 }
6590 #endif
6591 irq++;
6592 rid++;
6593 vi->nintr++;
6594 }
6595 } else {
6596 for_each_rxq(vi, q, rxq) {
6597 snprintf(s, sizeof(s), "%x%c%x", p,
6598 'a' + v, q);
6599 rc = t4_alloc_irq(sc, irq, rid,
6600 t4_intr, rxq, s);
6601 if (rc != 0)
6602 return (rc);
6603 #ifdef RSS
6604 bus_bind_intr(sc->dev, irq->res,
6605 rss_getcpu(q % nbuckets));
6606 #endif
6607 irq++;
6608 rid++;
6609 vi->nintr++;
6610 }
6611 }
6612 #ifdef TCP_OFFLOAD
6613 for_each_ofld_rxq(vi, q, ofld_rxq) {
6614 snprintf(s, sizeof(s), "%x%c%x", p, 'A' + v, q);
6615 rc = t4_alloc_irq(sc, irq, rid, t4_intr,
6616 ofld_rxq, s);
6617 if (rc != 0)
6618 return (rc);
6619 irq++;
6620 rid++;
6621 vi->nintr++;
6622 }
6623 #endif
6624 }
6625 }
6626 MPASS(irq == &sc->irq[sc->intr_count]);
6627
6628 return (0);
6629 }
6630
6631 static void
write_global_rss_key(struct adapter * sc)6632 write_global_rss_key(struct adapter *sc)
6633 {
6634 #ifdef RSS
6635 int i;
6636 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
6637 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
6638
6639 CTASSERT(RSS_KEYSIZE == 40);
6640
6641 rss_getkey((void *)&raw_rss_key[0]);
6642 for (i = 0; i < nitems(rss_key); i++) {
6643 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
6644 }
6645 t4_write_rss_key(sc, &rss_key[0], -1, 1);
6646 #endif
6647 }
6648
6649 /*
6650 * Idempotent.
6651 */
6652 static int
adapter_full_init(struct adapter * sc)6653 adapter_full_init(struct adapter *sc)
6654 {
6655 int rc, i;
6656
6657 ASSERT_SYNCHRONIZED_OP(sc);
6658
6659 /*
6660 * queues that belong to the adapter (not any particular port).
6661 */
6662 rc = t4_setup_adapter_queues(sc);
6663 if (rc != 0)
6664 return (rc);
6665
6666 MPASS(sc->params.nports <= nitems(sc->tq));
6667 for (i = 0; i < sc->params.nports; i++) {
6668 if (sc->tq[i] != NULL)
6669 continue;
6670 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
6671 taskqueue_thread_enqueue, &sc->tq[i]);
6672 if (sc->tq[i] == NULL) {
6673 CH_ERR(sc, "failed to allocate task queue %d\n", i);
6674 return (ENOMEM);
6675 }
6676 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
6677 device_get_nameunit(sc->dev), i);
6678 }
6679
6680 if (!(sc->flags & IS_VF)) {
6681 write_global_rss_key(sc);
6682 t4_intr_enable(sc);
6683 }
6684 return (0);
6685 }
6686
6687 int
adapter_init(struct adapter * sc)6688 adapter_init(struct adapter *sc)
6689 {
6690 int rc;
6691
6692 ASSERT_SYNCHRONIZED_OP(sc);
6693 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
6694 KASSERT((sc->flags & FULL_INIT_DONE) == 0,
6695 ("%s: FULL_INIT_DONE already", __func__));
6696
6697 rc = adapter_full_init(sc);
6698 if (rc != 0)
6699 adapter_full_uninit(sc);
6700 else
6701 sc->flags |= FULL_INIT_DONE;
6702
6703 return (rc);
6704 }
6705
6706 /*
6707 * Idempotent.
6708 */
6709 static void
adapter_full_uninit(struct adapter * sc)6710 adapter_full_uninit(struct adapter *sc)
6711 {
6712 int i;
6713
6714 t4_teardown_adapter_queues(sc);
6715
6716 for (i = 0; i < nitems(sc->tq); i++) {
6717 if (sc->tq[i] == NULL)
6718 continue;
6719 taskqueue_free(sc->tq[i]);
6720 sc->tq[i] = NULL;
6721 }
6722
6723 sc->flags &= ~FULL_INIT_DONE;
6724 }
6725
6726 #ifdef RSS
6727 #define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \
6728 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \
6729 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \
6730 RSS_HASHTYPE_RSS_UDP_IPV6)
6731
6732 /* Translates kernel hash types to hardware. */
6733 static int
hashconfig_to_hashen(int hashconfig)6734 hashconfig_to_hashen(int hashconfig)
6735 {
6736 int hashen = 0;
6737
6738 if (hashconfig & RSS_HASHTYPE_RSS_IPV4)
6739 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN;
6740 if (hashconfig & RSS_HASHTYPE_RSS_IPV6)
6741 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN;
6742 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) {
6743 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
6744 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
6745 }
6746 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) {
6747 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN |
6748 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
6749 }
6750 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4)
6751 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN;
6752 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6)
6753 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN;
6754
6755 return (hashen);
6756 }
6757
6758 /* Translates hardware hash types to kernel. */
6759 static int
hashen_to_hashconfig(int hashen)6760 hashen_to_hashconfig(int hashen)
6761 {
6762 int hashconfig = 0;
6763
6764 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) {
6765 /*
6766 * If UDP hashing was enabled it must have been enabled for
6767 * either IPv4 or IPv6 (inclusive or). Enabling UDP without
6768 * enabling any 4-tuple hash is nonsense configuration.
6769 */
6770 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
6771 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN));
6772
6773 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
6774 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4;
6775 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
6776 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6;
6777 }
6778 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
6779 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4;
6780 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
6781 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6;
6782 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
6783 hashconfig |= RSS_HASHTYPE_RSS_IPV4;
6784 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
6785 hashconfig |= RSS_HASHTYPE_RSS_IPV6;
6786
6787 return (hashconfig);
6788 }
6789 #endif
6790
6791 /*
6792 * Idempotent.
6793 */
6794 static int
vi_full_init(struct vi_info * vi)6795 vi_full_init(struct vi_info *vi)
6796 {
6797 struct adapter *sc = vi->adapter;
6798 struct sge_rxq *rxq;
6799 int rc, i, j;
6800 #ifdef RSS
6801 int nbuckets = rss_getnumbuckets();
6802 int hashconfig = rss_gethashconfig();
6803 int extra;
6804 #endif
6805
6806 ASSERT_SYNCHRONIZED_OP(sc);
6807
6808 /*
6809 * Allocate tx/rx/fl queues for this VI.
6810 */
6811 rc = t4_setup_vi_queues(vi);
6812 if (rc != 0)
6813 return (rc);
6814
6815 /*
6816 * Setup RSS for this VI. Save a copy of the RSS table for later use.
6817 */
6818 if (vi->nrxq > vi->rss_size) {
6819 CH_ALERT(vi, "nrxq (%d) > hw RSS table size (%d); "
6820 "some queues will never receive traffic.\n", vi->nrxq,
6821 vi->rss_size);
6822 } else if (vi->rss_size % vi->nrxq) {
6823 CH_ALERT(vi, "nrxq (%d), hw RSS table size (%d); "
6824 "expect uneven traffic distribution.\n", vi->nrxq,
6825 vi->rss_size);
6826 }
6827 #ifdef RSS
6828 if (vi->nrxq != nbuckets) {
6829 CH_ALERT(vi, "nrxq (%d) != kernel RSS buckets (%d);"
6830 "performance will be impacted.\n", vi->nrxq, nbuckets);
6831 }
6832 #endif
6833 if (vi->rss == NULL)
6834 vi->rss = malloc(vi->rss_size * sizeof (*vi->rss), M_CXGBE,
6835 M_ZERO | M_WAITOK);
6836 for (i = 0; i < vi->rss_size;) {
6837 #ifdef RSS
6838 j = rss_get_indirection_to_bucket(i);
6839 j %= vi->nrxq;
6840 rxq = &sc->sge.rxq[vi->first_rxq + j];
6841 vi->rss[i++] = rxq->iq.abs_id;
6842 #else
6843 for_each_rxq(vi, j, rxq) {
6844 vi->rss[i++] = rxq->iq.abs_id;
6845 if (i == vi->rss_size)
6846 break;
6847 }
6848 #endif
6849 }
6850
6851 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size,
6852 vi->rss, vi->rss_size);
6853 if (rc != 0) {
6854 CH_ERR(vi, "rss_config failed: %d\n", rc);
6855 return (rc);
6856 }
6857
6858 #ifdef RSS
6859 vi->hashen = hashconfig_to_hashen(hashconfig);
6860
6861 /*
6862 * We may have had to enable some hashes even though the global config
6863 * wants them disabled. This is a potential problem that must be
6864 * reported to the user.
6865 */
6866 extra = hashen_to_hashconfig(vi->hashen) ^ hashconfig;
6867
6868 /*
6869 * If we consider only the supported hash types, then the enabled hashes
6870 * are a superset of the requested hashes. In other words, there cannot
6871 * be any supported hash that was requested but not enabled, but there
6872 * can be hashes that were not requested but had to be enabled.
6873 */
6874 extra &= SUPPORTED_RSS_HASHTYPES;
6875 MPASS((extra & hashconfig) == 0);
6876
6877 if (extra) {
6878 CH_ALERT(vi,
6879 "global RSS config (0x%x) cannot be accommodated.\n",
6880 hashconfig);
6881 }
6882 if (extra & RSS_HASHTYPE_RSS_IPV4)
6883 CH_ALERT(vi, "IPv4 2-tuple hashing forced on.\n");
6884 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4)
6885 CH_ALERT(vi, "TCP/IPv4 4-tuple hashing forced on.\n");
6886 if (extra & RSS_HASHTYPE_RSS_IPV6)
6887 CH_ALERT(vi, "IPv6 2-tuple hashing forced on.\n");
6888 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6)
6889 CH_ALERT(vi, "TCP/IPv6 4-tuple hashing forced on.\n");
6890 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4)
6891 CH_ALERT(vi, "UDP/IPv4 4-tuple hashing forced on.\n");
6892 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6)
6893 CH_ALERT(vi, "UDP/IPv6 4-tuple hashing forced on.\n");
6894 #else
6895 vi->hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN |
6896 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN |
6897 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN |
6898 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN;
6899 #endif
6900 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, vi->rss[0],
6901 0, 0);
6902 if (rc != 0) {
6903 CH_ERR(vi, "rss hash/defaultq config failed: %d\n", rc);
6904 return (rc);
6905 }
6906
6907 return (0);
6908 }
6909
6910 int
vi_init(struct vi_info * vi)6911 vi_init(struct vi_info *vi)
6912 {
6913 int rc;
6914
6915 ASSERT_SYNCHRONIZED_OP(vi->adapter);
6916 KASSERT((vi->flags & VI_INIT_DONE) == 0,
6917 ("%s: VI_INIT_DONE already", __func__));
6918
6919 rc = vi_full_init(vi);
6920 if (rc != 0)
6921 vi_full_uninit(vi);
6922 else
6923 vi->flags |= VI_INIT_DONE;
6924
6925 return (rc);
6926 }
6927
6928 /*
6929 * Idempotent.
6930 */
6931 static void
vi_full_uninit(struct vi_info * vi)6932 vi_full_uninit(struct vi_info *vi)
6933 {
6934
6935 if (vi->flags & VI_INIT_DONE) {
6936 quiesce_vi(vi);
6937 free(vi->rss, M_CXGBE);
6938 free(vi->nm_rss, M_CXGBE);
6939 }
6940
6941 t4_teardown_vi_queues(vi);
6942 vi->flags &= ~VI_INIT_DONE;
6943 }
6944
6945 static void
quiesce_txq(struct sge_txq * txq)6946 quiesce_txq(struct sge_txq *txq)
6947 {
6948 struct sge_eq *eq = &txq->eq;
6949 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx];
6950
6951 MPASS(eq->flags & EQ_SW_ALLOCATED);
6952 MPASS(!(eq->flags & EQ_ENABLED));
6953
6954 /* Wait for the mp_ring to empty. */
6955 while (!mp_ring_is_idle(txq->r)) {
6956 mp_ring_check_drainage(txq->r, 4096);
6957 pause("rquiesce", 1);
6958 }
6959 MPASS(txq->txp.npkt == 0);
6960
6961 if (eq->flags & EQ_HW_ALLOCATED) {
6962 /*
6963 * Hardware is alive and working normally. Wait for it to
6964 * finish and then wait for the driver to catch up and reclaim
6965 * all descriptors.
6966 */
6967 while (spg->cidx != htobe16(eq->pidx))
6968 pause("equiesce", 1);
6969 while (eq->cidx != eq->pidx)
6970 pause("dquiesce", 1);
6971 } else {
6972 /*
6973 * Hardware is unavailable. Discard all pending tx and reclaim
6974 * descriptors directly.
6975 */
6976 TXQ_LOCK(txq);
6977 while (eq->cidx != eq->pidx) {
6978 struct mbuf *m, *nextpkt;
6979 struct tx_sdesc *txsd;
6980
6981 txsd = &txq->sdesc[eq->cidx];
6982 for (m = txsd->m; m != NULL; m = nextpkt) {
6983 nextpkt = m->m_nextpkt;
6984 m->m_nextpkt = NULL;
6985 m_freem(m);
6986 }
6987 IDXINCR(eq->cidx, txsd->desc_used, eq->sidx);
6988 }
6989 spg->pidx = spg->cidx = htobe16(eq->cidx);
6990 TXQ_UNLOCK(txq);
6991 }
6992 }
6993
6994 static void
quiesce_wrq(struct sge_wrq * wrq)6995 quiesce_wrq(struct sge_wrq *wrq)
6996 {
6997
6998 /* XXXTX */
6999 }
7000
7001 static void
quiesce_iq_fl(struct adapter * sc,struct sge_iq * iq,struct sge_fl * fl)7002 quiesce_iq_fl(struct adapter *sc, struct sge_iq *iq, struct sge_fl *fl)
7003 {
7004 /* Synchronize with the interrupt handler */
7005 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
7006 pause("iqfree", 1);
7007
7008 if (fl != NULL) {
7009 MPASS(iq->flags & IQ_HAS_FL);
7010
7011 mtx_lock(&sc->sfl_lock);
7012 FL_LOCK(fl);
7013 fl->flags |= FL_DOOMED;
7014 FL_UNLOCK(fl);
7015 callout_stop(&sc->sfl_callout);
7016 mtx_unlock(&sc->sfl_lock);
7017
7018 KASSERT((fl->flags & FL_STARVING) == 0,
7019 ("%s: still starving", __func__));
7020
7021 /* Release all buffers if hardware is no longer available. */
7022 if (!(iq->flags & IQ_HW_ALLOCATED))
7023 free_fl_buffers(sc, fl);
7024 }
7025 }
7026
7027 /*
7028 * Wait for all activity on all the queues of the VI to complete. It is assumed
7029 * that no new work is being enqueued by the hardware or the driver. That part
7030 * should be arranged before calling this function.
7031 */
7032 static void
quiesce_vi(struct vi_info * vi)7033 quiesce_vi(struct vi_info *vi)
7034 {
7035 int i;
7036 struct adapter *sc = vi->adapter;
7037 struct sge_rxq *rxq;
7038 struct sge_txq *txq;
7039 #ifdef TCP_OFFLOAD
7040 struct sge_ofld_rxq *ofld_rxq;
7041 #endif
7042 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
7043 struct sge_ofld_txq *ofld_txq;
7044 #endif
7045
7046 if (!(vi->flags & VI_INIT_DONE))
7047 return;
7048
7049 for_each_txq(vi, i, txq) {
7050 quiesce_txq(txq);
7051 }
7052
7053 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
7054 for_each_ofld_txq(vi, i, ofld_txq) {
7055 quiesce_wrq(&ofld_txq->wrq);
7056 }
7057 #endif
7058
7059 for_each_rxq(vi, i, rxq) {
7060 quiesce_iq_fl(sc, &rxq->iq, &rxq->fl);
7061 }
7062
7063 #ifdef TCP_OFFLOAD
7064 for_each_ofld_rxq(vi, i, ofld_rxq) {
7065 quiesce_iq_fl(sc, &ofld_rxq->iq, &ofld_rxq->fl);
7066 }
7067 #endif
7068 }
7069
7070 static int
t4_alloc_irq(struct adapter * sc,struct irq * irq,int rid,driver_intr_t * handler,void * arg,char * name)7071 t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
7072 driver_intr_t *handler, void *arg, char *name)
7073 {
7074 int rc;
7075
7076 irq->rid = rid;
7077 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
7078 RF_SHAREABLE | RF_ACTIVE);
7079 if (irq->res == NULL) {
7080 device_printf(sc->dev,
7081 "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
7082 return (ENOMEM);
7083 }
7084
7085 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
7086 NULL, handler, arg, &irq->tag);
7087 if (rc != 0) {
7088 device_printf(sc->dev,
7089 "failed to setup interrupt for rid %d, name %s: %d\n",
7090 rid, name, rc);
7091 } else if (name)
7092 bus_describe_intr(sc->dev, irq->res, irq->tag, "%s", name);
7093
7094 return (rc);
7095 }
7096
7097 static int
t4_free_irq(struct adapter * sc,struct irq * irq)7098 t4_free_irq(struct adapter *sc, struct irq *irq)
7099 {
7100 if (irq->tag)
7101 bus_teardown_intr(sc->dev, irq->res, irq->tag);
7102 if (irq->res)
7103 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
7104
7105 bzero(irq, sizeof(*irq));
7106
7107 return (0);
7108 }
7109
7110 static void
get_regs(struct adapter * sc,struct t4_regdump * regs,uint8_t * buf)7111 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
7112 {
7113
7114 regs->version = chip_id(sc) | chip_rev(sc) << 10;
7115 t4_get_regs(sc, buf, regs->len);
7116 }
7117
7118 #define A_PL_INDIR_CMD 0x1f8
7119
7120 #define S_PL_AUTOINC 31
7121 #define M_PL_AUTOINC 0x1U
7122 #define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC)
7123 #define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC)
7124
7125 #define S_PL_VFID 20
7126 #define M_PL_VFID 0xffU
7127 #define V_PL_VFID(x) ((x) << S_PL_VFID)
7128 #define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID)
7129
7130 #define S_PL_ADDR 0
7131 #define M_PL_ADDR 0xfffffU
7132 #define V_PL_ADDR(x) ((x) << S_PL_ADDR)
7133 #define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR)
7134
7135 #define A_PL_INDIR_DATA 0x1fc
7136
7137 static uint64_t
read_vf_stat(struct adapter * sc,u_int vin,int reg)7138 read_vf_stat(struct adapter *sc, u_int vin, int reg)
7139 {
7140 u32 stats[2];
7141
7142 if (sc->flags & IS_VF) {
7143 stats[0] = t4_read_reg(sc, VF_MPS_REG(reg));
7144 stats[1] = t4_read_reg(sc, VF_MPS_REG(reg + 4));
7145 } else {
7146 mtx_assert(&sc->reg_lock, MA_OWNED);
7147 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) |
7148 V_PL_VFID(vin) | V_PL_ADDR(VF_MPS_REG(reg)));
7149 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA);
7150 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA);
7151 }
7152 return (((uint64_t)stats[1]) << 32 | stats[0]);
7153 }
7154
7155 static void
t4_get_vi_stats(struct adapter * sc,u_int vin,struct fw_vi_stats_vf * stats)7156 t4_get_vi_stats(struct adapter *sc, u_int vin, struct fw_vi_stats_vf *stats)
7157 {
7158
7159 #define GET_STAT(name) \
7160 read_vf_stat(sc, vin, A_MPS_VF_STAT_##name##_L)
7161
7162 if (!(sc->flags & IS_VF))
7163 mtx_lock(&sc->reg_lock);
7164 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES);
7165 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES);
7166 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES);
7167 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES);
7168 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES);
7169 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES);
7170 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES);
7171 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES);
7172 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES);
7173 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES);
7174 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES);
7175 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES);
7176 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES);
7177 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES);
7178 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES);
7179 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES);
7180 if (!(sc->flags & IS_VF))
7181 mtx_unlock(&sc->reg_lock);
7182
7183 #undef GET_STAT
7184 }
7185
7186 static void
t4_clr_vi_stats(struct adapter * sc,u_int vin)7187 t4_clr_vi_stats(struct adapter *sc, u_int vin)
7188 {
7189 int reg;
7190
7191 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | V_PL_VFID(vin) |
7192 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L)));
7193 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
7194 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
7195 t4_write_reg(sc, A_PL_INDIR_DATA, 0);
7196 }
7197
7198 static void
vi_refresh_stats(struct vi_info * vi)7199 vi_refresh_stats(struct vi_info *vi)
7200 {
7201 struct timeval tv;
7202 const struct timeval interval = {0, 250000}; /* 250ms */
7203
7204 mtx_assert(&vi->tick_mtx, MA_OWNED);
7205
7206 if (vi->flags & VI_SKIP_STATS)
7207 return;
7208
7209 getmicrotime(&tv);
7210 timevalsub(&tv, &interval);
7211 if (timevalcmp(&tv, &vi->last_refreshed, <))
7212 return;
7213
7214 t4_get_vi_stats(vi->adapter, vi->vin, &vi->stats);
7215 getmicrotime(&vi->last_refreshed);
7216 }
7217
7218 static void
cxgbe_refresh_stats(struct vi_info * vi)7219 cxgbe_refresh_stats(struct vi_info *vi)
7220 {
7221 u_int i, v, tnl_cong_drops, chan_map;
7222 struct timeval tv;
7223 const struct timeval interval = {0, 250000}; /* 250ms */
7224 struct port_info *pi;
7225 struct adapter *sc;
7226
7227 mtx_assert(&vi->tick_mtx, MA_OWNED);
7228
7229 if (vi->flags & VI_SKIP_STATS)
7230 return;
7231
7232 getmicrotime(&tv);
7233 timevalsub(&tv, &interval);
7234 if (timevalcmp(&tv, &vi->last_refreshed, <))
7235 return;
7236
7237 pi = vi->pi;
7238 sc = vi->adapter;
7239 tnl_cong_drops = 0;
7240 t4_get_port_stats(sc, pi->port_id, &pi->stats);
7241 chan_map = pi->rx_e_chan_map;
7242 while (chan_map) {
7243 i = ffs(chan_map) - 1;
7244 mtx_lock(&sc->reg_lock);
7245 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 1,
7246 A_TP_MIB_TNL_CNG_DROP_0 + i);
7247 mtx_unlock(&sc->reg_lock);
7248 tnl_cong_drops += v;
7249 chan_map &= ~(1 << i);
7250 }
7251 pi->tnl_cong_drops = tnl_cong_drops;
7252 getmicrotime(&vi->last_refreshed);
7253 }
7254
7255 static void
cxgbe_tick(void * arg)7256 cxgbe_tick(void *arg)
7257 {
7258 struct vi_info *vi = arg;
7259
7260 MPASS(IS_MAIN_VI(vi));
7261 mtx_assert(&vi->tick_mtx, MA_OWNED);
7262
7263 cxgbe_refresh_stats(vi);
7264 callout_schedule(&vi->tick, hz);
7265 }
7266
7267 static void
vi_tick(void * arg)7268 vi_tick(void *arg)
7269 {
7270 struct vi_info *vi = arg;
7271
7272 mtx_assert(&vi->tick_mtx, MA_OWNED);
7273
7274 vi_refresh_stats(vi);
7275 callout_schedule(&vi->tick, hz);
7276 }
7277
7278 /*
7279 * Should match fw_caps_config_<foo> enums in t4fw_interface.h
7280 */
7281 static char *caps_decoder[] = {
7282 "\20\001IPMI\002NCSI", /* 0: NBM */
7283 "\20\001PPP\002QFC\003DCBX", /* 1: link */
7284 "\20\001INGRESS\002EGRESS", /* 2: switch */
7285 "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */
7286 "\006HASHFILTER\007ETHOFLD",
7287 "\20\001TOE", /* 4: TOE */
7288 "\20\001RDDP\002RDMAC", /* 5: RDMA */
7289 "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */
7290 "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD"
7291 "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD"
7292 "\007T10DIF"
7293 "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD",
7294 "\20\001LOOKASIDE\002TLSKEYS\003IPSEC_INLINE" /* 7: Crypto */
7295 "\004TLS_HW",
7296 "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */
7297 "\004PO_INITIATOR\005PO_TARGET",
7298 };
7299
7300 void
t4_sysctls(struct adapter * sc)7301 t4_sysctls(struct adapter *sc)
7302 {
7303 struct sysctl_ctx_list *ctx = &sc->ctx;
7304 struct sysctl_oid *oid;
7305 struct sysctl_oid_list *children, *c0;
7306 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"};
7307
7308 /*
7309 * dev.t4nex.X.
7310 */
7311 oid = device_get_sysctl_tree(sc->dev);
7312 c0 = children = SYSCTL_CHILDREN(oid);
7313
7314 sc->sc_do_rxcopy = 1;
7315 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW,
7316 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames");
7317
7318 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL,
7319 sc->params.nports, "# of ports");
7320
7321 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells",
7322 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, doorbells,
7323 (uintptr_t)&sc->doorbells, sysctl_bitfield_8b, "A",
7324 "available doorbells");
7325
7326 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL,
7327 sc->params.vpd.cclk, "core clock frequency (in KHz)");
7328
7329 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
7330 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
7331 sc->params.sge.timer_val, sizeof(sc->params.sge.timer_val),
7332 sysctl_int_array, "A", "interrupt holdoff timer values (us)");
7333
7334 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
7335 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
7336 sc->params.sge.counter_val, sizeof(sc->params.sge.counter_val),
7337 sysctl_int_array, "A", "interrupt holdoff packet counter values");
7338
7339 t4_sge_sysctls(sc, ctx, children);
7340
7341 sc->lro_timeout = 100;
7342 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW,
7343 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)");
7344
7345 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "dflags", CTLFLAG_RW,
7346 &sc->debug_flags, 0, "flags to enable runtime debugging");
7347
7348 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version",
7349 CTLFLAG_RD, sc->tp_version, 0, "TP microcode version");
7350
7351 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
7352 CTLFLAG_RD, sc->fw_version, 0, "firmware version");
7353
7354 if (sc->flags & IS_VF)
7355 return;
7356
7357 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
7358 NULL, chip_rev(sc), "chip hardware revision");
7359
7360 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "sn",
7361 CTLFLAG_RD, sc->params.vpd.sn, 0, "serial number");
7362
7363 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "pn",
7364 CTLFLAG_RD, sc->params.vpd.pn, 0, "part number");
7365
7366 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "ec",
7367 CTLFLAG_RD, sc->params.vpd.ec, 0, "engineering change");
7368
7369 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "md_version",
7370 CTLFLAG_RD, sc->params.vpd.md, 0, "manufacturing diags version");
7371
7372 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "na",
7373 CTLFLAG_RD, sc->params.vpd.na, 0, "network address");
7374
7375 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "er_version", CTLFLAG_RD,
7376 sc->er_version, 0, "expansion ROM version");
7377
7378 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "bs_version", CTLFLAG_RD,
7379 sc->bs_version, 0, "bootstrap firmware version");
7380
7381 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "scfg_version", CTLFLAG_RD,
7382 NULL, sc->params.scfg_vers, "serial config version");
7383
7384 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "vpd_version", CTLFLAG_RD,
7385 NULL, sc->params.vpd_vers, "VPD version");
7386
7387 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
7388 CTLFLAG_RD, sc->cfg_file, 0, "configuration file");
7389
7390 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL,
7391 sc->cfcsum, "config file checksum");
7392
7393 #define SYSCTL_CAP(name, n, text) \
7394 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \
7395 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, caps_decoder[n], \
7396 (uintptr_t)&sc->name, sysctl_bitfield_16b, "A", \
7397 "available " text " capabilities")
7398
7399 SYSCTL_CAP(nbmcaps, 0, "NBM");
7400 SYSCTL_CAP(linkcaps, 1, "link");
7401 SYSCTL_CAP(switchcaps, 2, "switch");
7402 SYSCTL_CAP(niccaps, 3, "NIC");
7403 SYSCTL_CAP(toecaps, 4, "TCP offload");
7404 SYSCTL_CAP(rdmacaps, 5, "RDMA");
7405 SYSCTL_CAP(iscsicaps, 6, "iSCSI");
7406 SYSCTL_CAP(cryptocaps, 7, "crypto");
7407 SYSCTL_CAP(fcoecaps, 8, "FCoE");
7408 #undef SYSCTL_CAP
7409
7410 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD,
7411 NULL, sc->tids.nftids, "number of filters");
7412
7413 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
7414 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7415 sysctl_temperature, "I", "chip temperature (in Celsius)");
7416 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset_sensor",
7417 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
7418 sysctl_reset_sensor, "I", "reset the chip's temperature sensor.");
7419
7420 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "loadavg",
7421 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7422 sysctl_loadavg, "A",
7423 "microprocessor load averages (debug firmwares only)");
7424
7425 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "core_vdd",
7426 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0, sysctl_vdd,
7427 "I", "core Vdd (in mV)");
7428
7429 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "local_cpus",
7430 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, LOCAL_CPUS,
7431 sysctl_cpus, "A", "local CPUs");
7432
7433 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_cpus",
7434 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, INTR_CPUS,
7435 sysctl_cpus, "A", "preferred CPUs for interrupts");
7436
7437 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "swintr", CTLFLAG_RW,
7438 &sc->swintr, 0, "software triggered interrupts");
7439
7440 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "reset",
7441 CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_reset, "I",
7442 "1 = reset adapter, 0 = zero reset counter");
7443
7444 /*
7445 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload.
7446 */
7447 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
7448 CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL,
7449 "logs and miscellaneous information");
7450 children = SYSCTL_CHILDREN(oid);
7451
7452 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
7453 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7454 sysctl_cctrl, "A", "congestion control");
7455
7456 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0",
7457 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7458 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)");
7459
7460 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1",
7461 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 1,
7462 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)");
7463
7464 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp",
7465 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 2,
7466 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)");
7467
7468 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0",
7469 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 3,
7470 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)");
7471
7472 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1",
7473 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 4,
7474 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)");
7475
7476 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi",
7477 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 5,
7478 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)");
7479
7480 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la",
7481 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7482 sysctl_cim_la, "A", "CIM logic analyzer");
7483
7484 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la",
7485 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7486 sysctl_cim_ma_la, "A", "CIM MA logic analyzer");
7487
7488 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0",
7489 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7490 0 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)");
7491
7492 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1",
7493 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7494 1 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)");
7495
7496 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2",
7497 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7498 2 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)");
7499
7500 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3",
7501 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7502 3 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)");
7503
7504 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge",
7505 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7506 4 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)");
7507
7508 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi",
7509 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7510 5 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)");
7511
7512 if (chip_id(sc) > CHELSIO_T4) {
7513 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx",
7514 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7515 6 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A",
7516 "CIM OBQ 6 (SGE0-RX)");
7517
7518 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx",
7519 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7520 7 + CIM_NUM_IBQ, sysctl_cim_ibq_obq, "A",
7521 "CIM OBQ 7 (SGE1-RX)");
7522 }
7523
7524 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la",
7525 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7526 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer");
7527
7528 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg",
7529 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7530 sysctl_cim_qcfg, "A", "CIM queue configuration");
7531
7532 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
7533 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7534 sysctl_cpl_stats, "A", "CPL statistics");
7535
7536 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
7537 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7538 sysctl_ddp_stats, "A", "non-TCP DDP statistics");
7539
7540 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tid_stats",
7541 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7542 sysctl_tid_stats, "A", "tid stats");
7543
7544 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
7545 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7546 sysctl_devlog, "A", "firmware's device log");
7547
7548 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
7549 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7550 sysctl_fcoe_stats, "A", "FCoE statistics");
7551
7552 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
7553 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7554 sysctl_hw_sched, "A", "hardware scheduler ");
7555
7556 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
7557 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7558 sysctl_l2t, "A", "hardware L2 table");
7559
7560 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "smt",
7561 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7562 sysctl_smt, "A", "hardware source MAC table");
7563
7564 #ifdef INET6
7565 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "clip",
7566 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7567 sysctl_clip, "A", "active CLIP table entries");
7568 #endif
7569
7570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
7571 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7572 sysctl_lb_stats, "A", "loopback statistics");
7573
7574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
7575 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7576 sysctl_meminfo, "A", "memory regions");
7577
7578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam",
7579 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7580 chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6,
7581 "A", "MPS TCAM entries");
7582
7583 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
7584 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7585 sysctl_path_mtus, "A", "path MTUs");
7586
7587 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
7588 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7589 sysctl_pm_stats, "A", "PM statistics");
7590
7591 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
7592 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7593 sysctl_rdma_stats, "A", "RDMA statistics");
7594
7595 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
7596 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7597 sysctl_tcp_stats, "A", "TCP statistics");
7598
7599 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
7600 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7601 sysctl_tids, "A", "TID information");
7602
7603 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
7604 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7605 sysctl_tp_err_stats, "A", "TP error statistics");
7606
7607 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tnl_stats",
7608 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7609 sysctl_tnl_stats, "A", "TP tunnel statistics");
7610
7611 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask",
7612 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
7613 sysctl_tp_la_mask, "I", "TP logic analyzer event capture mask");
7614
7615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la",
7616 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7617 sysctl_tp_la, "A", "TP logic analyzer");
7618
7619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
7620 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7621 sysctl_tx_rate, "A", "Tx rate");
7622
7623 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la",
7624 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7625 sysctl_ulprx_la, "A", "ULPRX logic analyzer");
7626
7627 if (chip_id(sc) >= CHELSIO_T5) {
7628 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats",
7629 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7630 sysctl_wcwr_stats, "A", "write combined work requests");
7631 }
7632
7633 #ifdef KERN_TLS
7634 if (is_ktls(sc)) {
7635 /*
7636 * dev.t4nex.0.tls.
7637 */
7638 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "tls",
7639 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "KERN_TLS parameters");
7640 children = SYSCTL_CHILDREN(oid);
7641
7642 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "inline_keys",
7643 CTLFLAG_RW, &sc->tlst.inline_keys, 0, "Always pass TLS "
7644 "keys in work requests (1) or attempt to store TLS keys "
7645 "in card memory.");
7646
7647 if (is_t6(sc))
7648 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "combo_wrs",
7649 CTLFLAG_RW, &sc->tlst.combo_wrs, 0, "Attempt to "
7650 "combine TCB field updates with TLS record work "
7651 "requests.");
7652 }
7653 #endif
7654
7655 #ifdef TCP_OFFLOAD
7656 if (is_offload(sc)) {
7657 int i;
7658 char s[4];
7659
7660 /*
7661 * dev.t4nex.X.toe.
7662 */
7663 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe",
7664 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE parameters");
7665 children = SYSCTL_CHILDREN(oid);
7666
7667 sc->tt.cong_algorithm = -1;
7668 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "cong_algorithm",
7669 CTLFLAG_RW, &sc->tt.cong_algorithm, 0, "congestion control "
7670 "(-1 = default, 0 = reno, 1 = tahoe, 2 = newreno, "
7671 "3 = highspeed)");
7672
7673 sc->tt.sndbuf = -1;
7674 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
7675 &sc->tt.sndbuf, 0, "hardware send buffer");
7676
7677 sc->tt.ddp = 0;
7678 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp",
7679 CTLFLAG_RW | CTLFLAG_SKIP, &sc->tt.ddp, 0, "");
7680 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_zcopy", CTLFLAG_RW,
7681 &sc->tt.ddp, 0, "Enable zero-copy aio_read(2)");
7682
7683 sc->tt.rx_coalesce = -1;
7684 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce",
7685 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing");
7686
7687 sc->tt.tls = 0;
7688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tls", CTLTYPE_INT |
7689 CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, sysctl_tls, "I",
7690 "Inline TLS allowed");
7691
7692 sc->tt.tx_align = -1;
7693 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align",
7694 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload");
7695
7696 sc->tt.tx_zcopy = 0;
7697 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_zcopy",
7698 CTLFLAG_RW, &sc->tt.tx_zcopy, 0,
7699 "Enable zero-copy aio_write(2)");
7700
7701 sc->tt.cop_managed_offloading = !!t4_cop_managed_offloading;
7702 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7703 "cop_managed_offloading", CTLFLAG_RW,
7704 &sc->tt.cop_managed_offloading, 0,
7705 "COP (Connection Offload Policy) controls all TOE offload");
7706
7707 sc->tt.autorcvbuf_inc = 16 * 1024;
7708 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "autorcvbuf_inc",
7709 CTLFLAG_RW, &sc->tt.autorcvbuf_inc, 0,
7710 "autorcvbuf increment");
7711
7712 sc->tt.update_hc_on_pmtu_change = 1;
7713 SYSCTL_ADD_INT(ctx, children, OID_AUTO,
7714 "update_hc_on_pmtu_change", CTLFLAG_RW,
7715 &sc->tt.update_hc_on_pmtu_change, 0,
7716 "Update hostcache entry if the PMTU changes");
7717
7718 sc->tt.iso = 1;
7719 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "iso", CTLFLAG_RW,
7720 &sc->tt.iso, 0, "Enable iSCSI segmentation offload");
7721
7722 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick",
7723 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7724 sysctl_tp_tick, "A", "TP timer tick (us)");
7725
7726 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick",
7727 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 1,
7728 sysctl_tp_tick, "A", "TCP timestamp tick (us)");
7729
7730 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick",
7731 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 2,
7732 sysctl_tp_tick, "A", "DACK tick (us)");
7733
7734 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer",
7735 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, 0,
7736 sysctl_tp_dack_timer, "IU", "DACK timer (us)");
7737
7738 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min",
7739 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7740 A_TP_RXT_MIN, sysctl_tp_timer, "LU",
7741 "Minimum retransmit interval (us)");
7742
7743 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max",
7744 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7745 A_TP_RXT_MAX, sysctl_tp_timer, "LU",
7746 "Maximum retransmit interval (us)");
7747
7748 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min",
7749 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7750 A_TP_PERS_MIN, sysctl_tp_timer, "LU",
7751 "Persist timer min (us)");
7752
7753 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max",
7754 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7755 A_TP_PERS_MAX, sysctl_tp_timer, "LU",
7756 "Persist timer max (us)");
7757
7758 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle",
7759 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7760 A_TP_KEEP_IDLE, sysctl_tp_timer, "LU",
7761 "Keepalive idle timer (us)");
7762
7763 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_interval",
7764 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7765 A_TP_KEEP_INTVL, sysctl_tp_timer, "LU",
7766 "Keepalive interval timer (us)");
7767
7768 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt",
7769 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7770 A_TP_INIT_SRTT, sysctl_tp_timer, "LU", "Initial SRTT (us)");
7771
7772 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer",
7773 CTLTYPE_ULONG | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7774 A_TP_FINWAIT2_TIMER, sysctl_tp_timer, "LU",
7775 "FINWAIT2 timer (us)");
7776
7777 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "syn_rexmt_count",
7778 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7779 S_SYNSHIFTMAX, sysctl_tp_shift_cnt, "IU",
7780 "Number of SYN retransmissions before abort");
7781
7782 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_count",
7783 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7784 S_RXTSHIFTMAXR2, sysctl_tp_shift_cnt, "IU",
7785 "Number of retransmissions before abort");
7786
7787 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_count",
7788 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7789 S_KEEPALIVEMAXR2, sysctl_tp_shift_cnt, "IU",
7790 "Number of keepalive probes before abort");
7791
7792 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "rexmt_backoff",
7793 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
7794 "TOE retransmit backoffs");
7795 children = SYSCTL_CHILDREN(oid);
7796 for (i = 0; i < 16; i++) {
7797 snprintf(s, sizeof(s), "%u", i);
7798 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, s,
7799 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
7800 i, sysctl_tp_backoff, "IU",
7801 "TOE retransmit backoff");
7802 }
7803 }
7804 #endif
7805 }
7806
7807 void
vi_sysctls(struct vi_info * vi)7808 vi_sysctls(struct vi_info *vi)
7809 {
7810 struct sysctl_ctx_list *ctx = &vi->ctx;
7811 struct sysctl_oid *oid;
7812 struct sysctl_oid_list *children;
7813
7814 /*
7815 * dev.v?(cxgbe|cxl).X.
7816 */
7817 oid = device_get_sysctl_tree(vi->dev);
7818 children = SYSCTL_CHILDREN(oid);
7819
7820 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL,
7821 vi->viid, "VI identifer");
7822 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
7823 &vi->nrxq, 0, "# of rx queues");
7824 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
7825 &vi->ntxq, 0, "# of tx queues");
7826 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
7827 &vi->first_rxq, 0, "index of first rx queue");
7828 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
7829 &vi->first_txq, 0, "index of first tx queue");
7830 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_base", CTLFLAG_RD, NULL,
7831 vi->rss_base, "start of RSS indirection table");
7832 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rss_size", CTLFLAG_RD, NULL,
7833 vi->rss_size, "size of RSS indirection table");
7834
7835 if (IS_MAIN_VI(vi)) {
7836 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq",
7837 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7838 sysctl_noflowq, "IU",
7839 "Reserve queue 0 for non-flowid packets");
7840 }
7841
7842 if (vi->adapter->flags & IS_VF) {
7843 MPASS(vi->flags & TX_USES_VM_WR);
7844 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_vm_wr", CTLFLAG_RD,
7845 NULL, 1, "use VM work requests for transmit");
7846 } else {
7847 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_vm_wr",
7848 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7849 sysctl_tx_vm_wr, "I", "use VM work requestes for transmit");
7850 }
7851
7852 #ifdef TCP_OFFLOAD
7853 if (vi->nofldrxq != 0) {
7854 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
7855 &vi->nofldrxq, 0,
7856 "# of rx queues for offloaded TCP connections");
7857 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
7858 CTLFLAG_RD, &vi->first_ofld_rxq, 0,
7859 "index of first TOE rx queue");
7860 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx_ofld",
7861 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7862 sysctl_holdoff_tmr_idx_ofld, "I",
7863 "holdoff timer index for TOE queues");
7864 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx_ofld",
7865 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7866 sysctl_holdoff_pktc_idx_ofld, "I",
7867 "holdoff packet counter index for TOE queues");
7868 }
7869 #endif
7870 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
7871 if (vi->nofldtxq != 0) {
7872 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
7873 &vi->nofldtxq, 0,
7874 "# of tx queues for TOE/ETHOFLD");
7875 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
7876 CTLFLAG_RD, &vi->first_ofld_txq, 0,
7877 "index of first TOE/ETHOFLD tx queue");
7878 }
7879 #endif
7880 #ifdef DEV_NETMAP
7881 if (vi->nnmrxq != 0) {
7882 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD,
7883 &vi->nnmrxq, 0, "# of netmap rx queues");
7884 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD,
7885 &vi->nnmtxq, 0, "# of netmap tx queues");
7886 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq",
7887 CTLFLAG_RD, &vi->first_nm_rxq, 0,
7888 "index of first netmap rx queue");
7889 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq",
7890 CTLFLAG_RD, &vi->first_nm_txq, 0,
7891 "index of first netmap tx queue");
7892 }
7893 #endif
7894
7895 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
7896 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7897 sysctl_holdoff_tmr_idx, "I", "holdoff timer index");
7898 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
7899 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7900 sysctl_holdoff_pktc_idx, "I", "holdoff packet counter index");
7901
7902 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
7903 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7904 sysctl_qsize_rxq, "I", "rx queue size");
7905 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
7906 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, vi, 0,
7907 sysctl_qsize_txq, "I", "tx queue size");
7908 }
7909
7910 static void
cxgbe_sysctls(struct port_info * pi)7911 cxgbe_sysctls(struct port_info *pi)
7912 {
7913 struct sysctl_ctx_list *ctx = &pi->ctx;
7914 struct sysctl_oid *oid;
7915 struct sysctl_oid_list *children, *children2;
7916 struct adapter *sc = pi->adapter;
7917 int i;
7918 char name[16];
7919 static char *tc_flags = {"\20\1USER"};
7920
7921 /*
7922 * dev.cxgbe.X.
7923 */
7924 oid = device_get_sysctl_tree(pi->dev);
7925 children = SYSCTL_CHILDREN(oid);
7926
7927 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc",
7928 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 0,
7929 sysctl_linkdnrc, "A", "reason why link is down");
7930 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) {
7931 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature",
7932 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 0,
7933 sysctl_btphy, "I", "PHY temperature (in Celsius)");
7934 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version",
7935 CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_MPSAFE, pi, 1,
7936 sysctl_btphy, "I", "PHY firmware version");
7937 }
7938
7939 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings",
7940 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0,
7941 sysctl_pause_settings, "A",
7942 "PAUSE settings (bit 0 = rx_pause, 1 = tx_pause, 2 = pause_autoneg)");
7943 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "link_fec",
7944 CTLTYPE_STRING | CTLFLAG_MPSAFE, pi, 0, sysctl_link_fec, "A",
7945 "FEC in use on the link");
7946 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "requested_fec",
7947 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0,
7948 sysctl_requested_fec, "A",
7949 "FECs to use (bit 0 = RS, 1 = FC, 2 = none, 5 = auto, 6 = module)");
7950 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "module_fec",
7951 CTLTYPE_STRING | CTLFLAG_MPSAFE, pi, 0, sysctl_module_fec, "A",
7952 "FEC recommended by the cable/transceiver");
7953 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "autoneg",
7954 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0,
7955 sysctl_autoneg, "I",
7956 "autonegotiation (-1 = not supported)");
7957 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "force_fec",
7958 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, pi, 0,
7959 sysctl_force_fec, "I", "when to use FORCE_FEC bit for link config");
7960
7961 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rcaps", CTLFLAG_RD,
7962 &pi->link_cfg.requested_caps, 0, "L1 config requested by driver");
7963 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "pcaps", CTLFLAG_RD,
7964 &pi->link_cfg.pcaps, 0, "port capabilities");
7965 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "acaps", CTLFLAG_RD,
7966 &pi->link_cfg.acaps, 0, "advertised capabilities");
7967 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lpacaps", CTLFLAG_RD,
7968 &pi->link_cfg.lpacaps, 0, "link partner advertised capabilities");
7969
7970 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL,
7971 port_top_speed(pi), "max speed (in Gbps)");
7972 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "mps_bg_map", CTLFLAG_RD, NULL,
7973 pi->mps_bg_map, "MPS buffer group map");
7974 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_e_chan_map", CTLFLAG_RD,
7975 NULL, pi->rx_e_chan_map, "TP rx e-channel map");
7976 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_chan", CTLFLAG_RD, NULL,
7977 pi->tx_chan, "TP tx c-channel");
7978 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_chan", CTLFLAG_RD, NULL,
7979 pi->rx_chan, "TP rx c-channel");
7980
7981 if (sc->flags & IS_VF)
7982 return;
7983
7984 /*
7985 * dev.(cxgbe|cxl).X.tc.
7986 */
7987 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc",
7988 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
7989 "Tx scheduler traffic classes (cl_rl)");
7990 children2 = SYSCTL_CHILDREN(oid);
7991 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "pktsize",
7992 CTLFLAG_RW, &pi->sched_params->pktsize, 0,
7993 "pktsize for per-flow cl-rl (0 means up to the driver )");
7994 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "burstsize",
7995 CTLFLAG_RW, &pi->sched_params->burstsize, 0,
7996 "burstsize for per-flow cl-rl (0 means up to the driver)");
7997 for (i = 0; i < sc->params.nsched_cls; i++) {
7998 struct tx_cl_rl_params *tc = &pi->sched_params->cl_rl[i];
7999
8000 snprintf(name, sizeof(name), "%d", i);
8001 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx,
8002 SYSCTL_CHILDREN(oid), OID_AUTO, name,
8003 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "traffic class"));
8004 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "state",
8005 CTLFLAG_RD, &tc->state, 0, "current state");
8006 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "flags",
8007 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, tc_flags,
8008 (uintptr_t)&tc->flags, sysctl_bitfield_8b, "A", "flags");
8009 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount",
8010 CTLFLAG_RD, &tc->refcount, 0, "references to this class");
8011 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params",
8012 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, sc,
8013 (pi->port_id << 16) | i, sysctl_tc_params, "A",
8014 "traffic class parameters");
8015 }
8016
8017 /*
8018 * dev.cxgbe.X.stats.
8019 */
8020 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
8021 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "port statistics");
8022 children = SYSCTL_CHILDREN(oid);
8023 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD,
8024 &pi->tx_parse_error, 0,
8025 "# of tx packets with invalid length or # of segments");
8026
8027 #define T4_REGSTAT(name, stat, desc) \
8028 SYSCTL_ADD_OID(ctx, children, OID_AUTO, #name, \
8029 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, sc, \
8030 t4_port_reg(sc, pi->tx_chan, A_MPS_PORT_STAT_##stat##_L), \
8031 sysctl_handle_t4_reg64, "QU", desc)
8032
8033 /* We get these from port_stats and they may be stale by up to 1s */
8034 #define T4_PORTSTAT(name, desc) \
8035 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
8036 &pi->stats.name, desc)
8037
8038 T4_REGSTAT(tx_octets, TX_PORT_BYTES, "# of octets in good frames");
8039 T4_REGSTAT(tx_frames, TX_PORT_FRAMES, "total # of good frames");
8040 T4_REGSTAT(tx_bcast_frames, TX_PORT_BCAST, "# of broadcast frames");
8041 T4_REGSTAT(tx_mcast_frames, TX_PORT_MCAST, "# of multicast frames");
8042 T4_REGSTAT(tx_ucast_frames, TX_PORT_UCAST, "# of unicast frames");
8043 T4_REGSTAT(tx_error_frames, TX_PORT_ERROR, "# of error frames");
8044 T4_REGSTAT(tx_frames_64, TX_PORT_64B, "# of tx frames in this range");
8045 T4_REGSTAT(tx_frames_65_127, TX_PORT_65B_127B, "# of tx frames in this range");
8046 T4_REGSTAT(tx_frames_128_255, TX_PORT_128B_255B, "# of tx frames in this range");
8047 T4_REGSTAT(tx_frames_256_511, TX_PORT_256B_511B, "# of tx frames in this range");
8048 T4_REGSTAT(tx_frames_512_1023, TX_PORT_512B_1023B, "# of tx frames in this range");
8049 T4_REGSTAT(tx_frames_1024_1518, TX_PORT_1024B_1518B, "# of tx frames in this range");
8050 T4_REGSTAT(tx_frames_1519_max, TX_PORT_1519B_MAX, "# of tx frames in this range");
8051 T4_REGSTAT(tx_drop, TX_PORT_DROP, "# of dropped tx frames");
8052 T4_REGSTAT(tx_pause, TX_PORT_PAUSE, "# of pause frames transmitted");
8053 T4_REGSTAT(tx_ppp0, TX_PORT_PPP0, "# of PPP prio 0 frames transmitted");
8054 T4_REGSTAT(tx_ppp1, TX_PORT_PPP1, "# of PPP prio 1 frames transmitted");
8055 T4_REGSTAT(tx_ppp2, TX_PORT_PPP2, "# of PPP prio 2 frames transmitted");
8056 T4_REGSTAT(tx_ppp3, TX_PORT_PPP3, "# of PPP prio 3 frames transmitted");
8057 T4_REGSTAT(tx_ppp4, TX_PORT_PPP4, "# of PPP prio 4 frames transmitted");
8058 T4_REGSTAT(tx_ppp5, TX_PORT_PPP5, "# of PPP prio 5 frames transmitted");
8059 T4_REGSTAT(tx_ppp6, TX_PORT_PPP6, "# of PPP prio 6 frames transmitted");
8060 T4_REGSTAT(tx_ppp7, TX_PORT_PPP7, "# of PPP prio 7 frames transmitted");
8061
8062 T4_REGSTAT(rx_octets, RX_PORT_BYTES, "# of octets in good frames");
8063 T4_REGSTAT(rx_frames, RX_PORT_FRAMES, "total # of good frames");
8064 T4_REGSTAT(rx_bcast_frames, RX_PORT_BCAST, "# of broadcast frames");
8065 T4_REGSTAT(rx_mcast_frames, RX_PORT_MCAST, "# of multicast frames");
8066 T4_REGSTAT(rx_ucast_frames, RX_PORT_UCAST, "# of unicast frames");
8067 T4_REGSTAT(rx_too_long, RX_PORT_MTU_ERROR, "# of frames exceeding MTU");
8068 T4_REGSTAT(rx_jabber, RX_PORT_MTU_CRC_ERROR, "# of jabber frames");
8069 if (is_t6(sc)) {
8070 T4_PORTSTAT(rx_fcs_err,
8071 "# of frames received with bad FCS since last link up");
8072 } else {
8073 T4_REGSTAT(rx_fcs_err, RX_PORT_CRC_ERROR,
8074 "# of frames received with bad FCS");
8075 }
8076 T4_REGSTAT(rx_len_err, RX_PORT_LEN_ERROR, "# of frames received with length error");
8077 T4_REGSTAT(rx_symbol_err, RX_PORT_SYM_ERROR, "symbol errors");
8078 T4_REGSTAT(rx_runt, RX_PORT_LESS_64B, "# of short frames received");
8079 T4_REGSTAT(rx_frames_64, RX_PORT_64B, "# of rx frames in this range");
8080 T4_REGSTAT(rx_frames_65_127, RX_PORT_65B_127B, "# of rx frames in this range");
8081 T4_REGSTAT(rx_frames_128_255, RX_PORT_128B_255B, "# of rx frames in this range");
8082 T4_REGSTAT(rx_frames_256_511, RX_PORT_256B_511B, "# of rx frames in this range");
8083 T4_REGSTAT(rx_frames_512_1023, RX_PORT_512B_1023B, "# of rx frames in this range");
8084 T4_REGSTAT(rx_frames_1024_1518, RX_PORT_1024B_1518B, "# of rx frames in this range");
8085 T4_REGSTAT(rx_frames_1519_max, RX_PORT_1519B_MAX, "# of rx frames in this range");
8086 T4_REGSTAT(rx_pause, RX_PORT_PAUSE, "# of pause frames received");
8087 T4_REGSTAT(rx_ppp0, RX_PORT_PPP0, "# of PPP prio 0 frames received");
8088 T4_REGSTAT(rx_ppp1, RX_PORT_PPP1, "# of PPP prio 1 frames received");
8089 T4_REGSTAT(rx_ppp2, RX_PORT_PPP2, "# of PPP prio 2 frames received");
8090 T4_REGSTAT(rx_ppp3, RX_PORT_PPP3, "# of PPP prio 3 frames received");
8091 T4_REGSTAT(rx_ppp4, RX_PORT_PPP4, "# of PPP prio 4 frames received");
8092 T4_REGSTAT(rx_ppp5, RX_PORT_PPP5, "# of PPP prio 5 frames received");
8093 T4_REGSTAT(rx_ppp6, RX_PORT_PPP6, "# of PPP prio 6 frames received");
8094 T4_REGSTAT(rx_ppp7, RX_PORT_PPP7, "# of PPP prio 7 frames received");
8095
8096 T4_PORTSTAT(rx_ovflow0, "# drops due to buffer-group 0 overflows");
8097 T4_PORTSTAT(rx_ovflow1, "# drops due to buffer-group 1 overflows");
8098 T4_PORTSTAT(rx_ovflow2, "# drops due to buffer-group 2 overflows");
8099 T4_PORTSTAT(rx_ovflow3, "# drops due to buffer-group 3 overflows");
8100 T4_PORTSTAT(rx_trunc0, "# of buffer-group 0 truncated packets");
8101 T4_PORTSTAT(rx_trunc1, "# of buffer-group 1 truncated packets");
8102 T4_PORTSTAT(rx_trunc2, "# of buffer-group 2 truncated packets");
8103 T4_PORTSTAT(rx_trunc3, "# of buffer-group 3 truncated packets");
8104
8105 #undef T4_REGSTAT
8106 #undef T4_PORTSTAT
8107 }
8108
8109 static int
sysctl_int_array(SYSCTL_HANDLER_ARGS)8110 sysctl_int_array(SYSCTL_HANDLER_ARGS)
8111 {
8112 int rc, *i, space = 0;
8113 struct sbuf sb;
8114
8115 sbuf_new_for_sysctl(&sb, NULL, 64, req);
8116 for (i = arg1; arg2; arg2 -= sizeof(int), i++) {
8117 if (space)
8118 sbuf_printf(&sb, " ");
8119 sbuf_printf(&sb, "%d", *i);
8120 space = 1;
8121 }
8122 rc = sbuf_finish(&sb);
8123 sbuf_delete(&sb);
8124 return (rc);
8125 }
8126
8127 static int
sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS)8128 sysctl_bitfield_8b(SYSCTL_HANDLER_ARGS)
8129 {
8130 int rc;
8131 struct sbuf *sb;
8132
8133 rc = sysctl_wire_old_buffer(req, 0);
8134 if (rc != 0)
8135 return(rc);
8136
8137 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
8138 if (sb == NULL)
8139 return (ENOMEM);
8140
8141 sbuf_printf(sb, "%b", *(uint8_t *)(uintptr_t)arg2, (char *)arg1);
8142 rc = sbuf_finish(sb);
8143 sbuf_delete(sb);
8144
8145 return (rc);
8146 }
8147
8148 static int
sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS)8149 sysctl_bitfield_16b(SYSCTL_HANDLER_ARGS)
8150 {
8151 int rc;
8152 struct sbuf *sb;
8153
8154 rc = sysctl_wire_old_buffer(req, 0);
8155 if (rc != 0)
8156 return(rc);
8157
8158 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
8159 if (sb == NULL)
8160 return (ENOMEM);
8161
8162 sbuf_printf(sb, "%b", *(uint16_t *)(uintptr_t)arg2, (char *)arg1);
8163 rc = sbuf_finish(sb);
8164 sbuf_delete(sb);
8165
8166 return (rc);
8167 }
8168
8169 static int
sysctl_btphy(SYSCTL_HANDLER_ARGS)8170 sysctl_btphy(SYSCTL_HANDLER_ARGS)
8171 {
8172 struct port_info *pi = arg1;
8173 int op = arg2;
8174 struct adapter *sc = pi->adapter;
8175 u_int v;
8176 int rc;
8177
8178 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt");
8179 if (rc)
8180 return (rc);
8181 if (hw_off_limits(sc))
8182 rc = ENXIO;
8183 else {
8184 /* XXX: magic numbers */
8185 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e,
8186 op ? 0x20 : 0xc820, &v);
8187 }
8188 end_synchronized_op(sc, 0);
8189 if (rc)
8190 return (rc);
8191 if (op == 0)
8192 v /= 256;
8193
8194 rc = sysctl_handle_int(oidp, &v, 0, req);
8195 return (rc);
8196 }
8197
8198 static int
sysctl_noflowq(SYSCTL_HANDLER_ARGS)8199 sysctl_noflowq(SYSCTL_HANDLER_ARGS)
8200 {
8201 struct vi_info *vi = arg1;
8202 int rc, val;
8203
8204 val = vi->rsrv_noflowq;
8205 rc = sysctl_handle_int(oidp, &val, 0, req);
8206 if (rc != 0 || req->newptr == NULL)
8207 return (rc);
8208
8209 if ((val >= 1) && (vi->ntxq > 1))
8210 vi->rsrv_noflowq = 1;
8211 else
8212 vi->rsrv_noflowq = 0;
8213
8214 return (rc);
8215 }
8216
8217 static int
sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS)8218 sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS)
8219 {
8220 struct vi_info *vi = arg1;
8221 struct adapter *sc = vi->adapter;
8222 int rc, val, i;
8223
8224 MPASS(!(sc->flags & IS_VF));
8225
8226 val = vi->flags & TX_USES_VM_WR ? 1 : 0;
8227 rc = sysctl_handle_int(oidp, &val, 0, req);
8228 if (rc != 0 || req->newptr == NULL)
8229 return (rc);
8230
8231 if (val != 0 && val != 1)
8232 return (EINVAL);
8233
8234 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
8235 "t4txvm");
8236 if (rc)
8237 return (rc);
8238 if (hw_off_limits(sc))
8239 rc = ENXIO;
8240 else if (if_getdrvflags(vi->ifp) & IFF_DRV_RUNNING) {
8241 /*
8242 * We don't want parse_pkt to run with one setting (VF or PF)
8243 * and then eth_tx to see a different setting but still use
8244 * stale information calculated by parse_pkt.
8245 */
8246 rc = EBUSY;
8247 } else {
8248 struct port_info *pi = vi->pi;
8249 struct sge_txq *txq;
8250 uint32_t ctrl0;
8251 uint8_t npkt = sc->params.max_pkts_per_eth_tx_pkts_wr;
8252
8253 if (val) {
8254 vi->flags |= TX_USES_VM_WR;
8255 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_VM_TSO);
8256 ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
8257 V_TXPKT_INTF(pi->tx_chan));
8258 if (!(sc->flags & IS_VF))
8259 npkt--;
8260 } else {
8261 vi->flags &= ~TX_USES_VM_WR;
8262 if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_TSO);
8263 ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) |
8264 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) |
8265 V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld));
8266 }
8267 for_each_txq(vi, i, txq) {
8268 txq->cpl_ctrl0 = ctrl0;
8269 txq->txp.max_npkt = npkt;
8270 }
8271 }
8272 end_synchronized_op(sc, LOCK_HELD);
8273 return (rc);
8274 }
8275
8276 static int
sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)8277 sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
8278 {
8279 struct vi_info *vi = arg1;
8280 struct adapter *sc = vi->adapter;
8281 int idx, rc, i;
8282 struct sge_rxq *rxq;
8283 uint8_t v;
8284
8285 idx = vi->tmr_idx;
8286
8287 rc = sysctl_handle_int(oidp, &idx, 0, req);
8288 if (rc != 0 || req->newptr == NULL)
8289 return (rc);
8290
8291 if (idx < 0 || idx >= SGE_NTIMERS)
8292 return (EINVAL);
8293
8294 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
8295 "t4tmr");
8296 if (rc)
8297 return (rc);
8298
8299 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1);
8300 for_each_rxq(vi, i, rxq) {
8301 #ifdef atomic_store_rel_8
8302 atomic_store_rel_8(&rxq->iq.intr_params, v);
8303 #else
8304 rxq->iq.intr_params = v;
8305 #endif
8306 }
8307 vi->tmr_idx = idx;
8308
8309 end_synchronized_op(sc, LOCK_HELD);
8310 return (0);
8311 }
8312
8313 static int
sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)8314 sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
8315 {
8316 struct vi_info *vi = arg1;
8317 struct adapter *sc = vi->adapter;
8318 int idx, rc;
8319
8320 idx = vi->pktc_idx;
8321
8322 rc = sysctl_handle_int(oidp, &idx, 0, req);
8323 if (rc != 0 || req->newptr == NULL)
8324 return (rc);
8325
8326 if (idx < -1 || idx >= SGE_NCOUNTERS)
8327 return (EINVAL);
8328
8329 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
8330 "t4pktc");
8331 if (rc)
8332 return (rc);
8333
8334 if (vi->flags & VI_INIT_DONE)
8335 rc = EBUSY; /* cannot be changed once the queues are created */
8336 else
8337 vi->pktc_idx = idx;
8338
8339 end_synchronized_op(sc, LOCK_HELD);
8340 return (rc);
8341 }
8342
8343 static int
sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)8344 sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
8345 {
8346 struct vi_info *vi = arg1;
8347 struct adapter *sc = vi->adapter;
8348 int qsize, rc;
8349
8350 qsize = vi->qsize_rxq;
8351
8352 rc = sysctl_handle_int(oidp, &qsize, 0, req);
8353 if (rc != 0 || req->newptr == NULL)
8354 return (rc);
8355
8356 if (qsize < 128 || (qsize & 7))
8357 return (EINVAL);
8358
8359 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
8360 "t4rxqs");
8361 if (rc)
8362 return (rc);
8363
8364 if (vi->flags & VI_INIT_DONE)
8365 rc = EBUSY; /* cannot be changed once the queues are created */
8366 else
8367 vi->qsize_rxq = qsize;
8368
8369 end_synchronized_op(sc, LOCK_HELD);
8370 return (rc);
8371 }
8372
8373 static int
sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)8374 sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
8375 {
8376 struct vi_info *vi = arg1;
8377 struct adapter *sc = vi->adapter;
8378 int qsize, rc;
8379
8380 qsize = vi->qsize_txq;
8381
8382 rc = sysctl_handle_int(oidp, &qsize, 0, req);
8383 if (rc != 0 || req->newptr == NULL)
8384 return (rc);
8385
8386 if (qsize < 128 || qsize > 65536)
8387 return (EINVAL);
8388
8389 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
8390 "t4txqs");
8391 if (rc)
8392 return (rc);
8393
8394 if (vi->flags & VI_INIT_DONE)
8395 rc = EBUSY; /* cannot be changed once the queues are created */
8396 else
8397 vi->qsize_txq = qsize;
8398
8399 end_synchronized_op(sc, LOCK_HELD);
8400 return (rc);
8401 }
8402
8403 static int
sysctl_pause_settings(SYSCTL_HANDLER_ARGS)8404 sysctl_pause_settings(SYSCTL_HANDLER_ARGS)
8405 {
8406 struct port_info *pi = arg1;
8407 struct adapter *sc = pi->adapter;
8408 struct link_config *lc = &pi->link_cfg;
8409 int rc;
8410
8411 if (req->newptr == NULL) {
8412 struct sbuf *sb;
8413 static char *bits = "\20\1RX\2TX\3AUTO";
8414
8415 rc = sysctl_wire_old_buffer(req, 0);
8416 if (rc != 0)
8417 return(rc);
8418
8419 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
8420 if (sb == NULL)
8421 return (ENOMEM);
8422
8423 if (lc->link_ok) {
8424 sbuf_printf(sb, "%b", (lc->fc & (PAUSE_TX | PAUSE_RX)) |
8425 (lc->requested_fc & PAUSE_AUTONEG), bits);
8426 } else {
8427 sbuf_printf(sb, "%b", lc->requested_fc & (PAUSE_TX |
8428 PAUSE_RX | PAUSE_AUTONEG), bits);
8429 }
8430 rc = sbuf_finish(sb);
8431 sbuf_delete(sb);
8432 } else {
8433 char s[2];
8434 int n;
8435
8436 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX |
8437 PAUSE_AUTONEG));
8438 s[1] = 0;
8439
8440 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
8441 if (rc != 0)
8442 return(rc);
8443
8444 if (s[1] != 0)
8445 return (EINVAL);
8446 if (s[0] < '0' || s[0] > '9')
8447 return (EINVAL); /* not a number */
8448 n = s[0] - '0';
8449 if (n & ~(PAUSE_TX | PAUSE_RX | PAUSE_AUTONEG))
8450 return (EINVAL); /* some other bit is set too */
8451
8452 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8453 "t4PAUSE");
8454 if (rc)
8455 return (rc);
8456 if (!hw_off_limits(sc)) {
8457 PORT_LOCK(pi);
8458 lc->requested_fc = n;
8459 fixup_link_config(pi);
8460 if (pi->up_vis > 0)
8461 rc = apply_link_config(pi);
8462 set_current_media(pi);
8463 PORT_UNLOCK(pi);
8464 }
8465 end_synchronized_op(sc, 0);
8466 }
8467
8468 return (rc);
8469 }
8470
8471 static int
sysctl_link_fec(SYSCTL_HANDLER_ARGS)8472 sysctl_link_fec(SYSCTL_HANDLER_ARGS)
8473 {
8474 struct port_info *pi = arg1;
8475 struct link_config *lc = &pi->link_cfg;
8476 int rc;
8477 struct sbuf *sb;
8478 static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD1\5RSVD2";
8479
8480 rc = sysctl_wire_old_buffer(req, 0);
8481 if (rc != 0)
8482 return(rc);
8483
8484 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
8485 if (sb == NULL)
8486 return (ENOMEM);
8487 if (lc->link_ok)
8488 sbuf_printf(sb, "%b", lc->fec, bits);
8489 else
8490 sbuf_printf(sb, "no link");
8491 rc = sbuf_finish(sb);
8492 sbuf_delete(sb);
8493
8494 return (rc);
8495 }
8496
8497 static int
sysctl_requested_fec(SYSCTL_HANDLER_ARGS)8498 sysctl_requested_fec(SYSCTL_HANDLER_ARGS)
8499 {
8500 struct port_info *pi = arg1;
8501 struct adapter *sc = pi->adapter;
8502 struct link_config *lc = &pi->link_cfg;
8503 int rc;
8504 int8_t old;
8505
8506 if (req->newptr == NULL) {
8507 struct sbuf *sb;
8508 static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2"
8509 "\5RSVD3\6auto\7module";
8510
8511 rc = sysctl_wire_old_buffer(req, 0);
8512 if (rc != 0)
8513 return(rc);
8514
8515 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
8516 if (sb == NULL)
8517 return (ENOMEM);
8518
8519 sbuf_printf(sb, "%b", lc->requested_fec, bits);
8520 rc = sbuf_finish(sb);
8521 sbuf_delete(sb);
8522 } else {
8523 char s[8];
8524 int n;
8525
8526 snprintf(s, sizeof(s), "%d",
8527 lc->requested_fec == FEC_AUTO ? -1 :
8528 lc->requested_fec & (M_FW_PORT_CAP32_FEC | FEC_MODULE));
8529
8530 rc = sysctl_handle_string(oidp, s, sizeof(s), req);
8531 if (rc != 0)
8532 return(rc);
8533
8534 n = strtol(&s[0], NULL, 0);
8535 if (n < 0 || n & FEC_AUTO)
8536 n = FEC_AUTO;
8537 else if (n & ~(M_FW_PORT_CAP32_FEC | FEC_MODULE))
8538 return (EINVAL);/* some other bit is set too */
8539
8540 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8541 "t4reqf");
8542 if (rc)
8543 return (rc);
8544 PORT_LOCK(pi);
8545 old = lc->requested_fec;
8546 if (n == FEC_AUTO)
8547 lc->requested_fec = FEC_AUTO;
8548 else if (n == 0 || n == FEC_NONE)
8549 lc->requested_fec = FEC_NONE;
8550 else {
8551 if ((lc->pcaps |
8552 V_FW_PORT_CAP32_FEC(n & M_FW_PORT_CAP32_FEC)) !=
8553 lc->pcaps) {
8554 rc = ENOTSUP;
8555 goto done;
8556 }
8557 lc->requested_fec = n & (M_FW_PORT_CAP32_FEC |
8558 FEC_MODULE);
8559 }
8560 if (!hw_off_limits(sc)) {
8561 fixup_link_config(pi);
8562 if (pi->up_vis > 0) {
8563 rc = apply_link_config(pi);
8564 if (rc != 0) {
8565 lc->requested_fec = old;
8566 if (rc == FW_EPROTO)
8567 rc = ENOTSUP;
8568 }
8569 }
8570 }
8571 done:
8572 PORT_UNLOCK(pi);
8573 end_synchronized_op(sc, 0);
8574 }
8575
8576 return (rc);
8577 }
8578
8579 static int
sysctl_module_fec(SYSCTL_HANDLER_ARGS)8580 sysctl_module_fec(SYSCTL_HANDLER_ARGS)
8581 {
8582 struct port_info *pi = arg1;
8583 struct adapter *sc = pi->adapter;
8584 struct link_config *lc = &pi->link_cfg;
8585 int rc;
8586 int8_t fec;
8587 struct sbuf *sb;
8588 static char *bits = "\20\1RS-FEC\2FC-FEC\3NO-FEC\4RSVD2\5RSVD3";
8589
8590 rc = sysctl_wire_old_buffer(req, 0);
8591 if (rc != 0)
8592 return (rc);
8593
8594 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
8595 if (sb == NULL)
8596 return (ENOMEM);
8597
8598 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4mfec") != 0) {
8599 rc = EBUSY;
8600 goto done;
8601 }
8602 if (hw_off_limits(sc)) {
8603 rc = ENXIO;
8604 goto done;
8605 }
8606 PORT_LOCK(pi);
8607 if (pi->up_vis == 0) {
8608 /*
8609 * If all the interfaces are administratively down the firmware
8610 * does not report transceiver changes. Refresh port info here.
8611 * This is the only reason we have a synchronized op in this
8612 * function. Just PORT_LOCK would have been enough otherwise.
8613 */
8614 t4_update_port_info(pi);
8615 }
8616
8617 fec = lc->fec_hint;
8618 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE ||
8619 !fec_supported(lc->pcaps)) {
8620 sbuf_printf(sb, "n/a");
8621 } else {
8622 if (fec == 0)
8623 fec = FEC_NONE;
8624 sbuf_printf(sb, "%b", fec & M_FW_PORT_CAP32_FEC, bits);
8625 }
8626 rc = sbuf_finish(sb);
8627 PORT_UNLOCK(pi);
8628 done:
8629 sbuf_delete(sb);
8630 end_synchronized_op(sc, 0);
8631
8632 return (rc);
8633 }
8634
8635 static int
sysctl_autoneg(SYSCTL_HANDLER_ARGS)8636 sysctl_autoneg(SYSCTL_HANDLER_ARGS)
8637 {
8638 struct port_info *pi = arg1;
8639 struct adapter *sc = pi->adapter;
8640 struct link_config *lc = &pi->link_cfg;
8641 int rc, val;
8642
8643 if (lc->pcaps & FW_PORT_CAP32_ANEG)
8644 val = lc->requested_aneg == AUTONEG_DISABLE ? 0 : 1;
8645 else
8646 val = -1;
8647 rc = sysctl_handle_int(oidp, &val, 0, req);
8648 if (rc != 0 || req->newptr == NULL)
8649 return (rc);
8650 if (val == 0)
8651 val = AUTONEG_DISABLE;
8652 else if (val == 1)
8653 val = AUTONEG_ENABLE;
8654 else
8655 val = AUTONEG_AUTO;
8656
8657 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK,
8658 "t4aneg");
8659 if (rc)
8660 return (rc);
8661 PORT_LOCK(pi);
8662 if (val == AUTONEG_ENABLE && !(lc->pcaps & FW_PORT_CAP32_ANEG)) {
8663 rc = ENOTSUP;
8664 goto done;
8665 }
8666 lc->requested_aneg = val;
8667 if (!hw_off_limits(sc)) {
8668 fixup_link_config(pi);
8669 if (pi->up_vis > 0)
8670 rc = apply_link_config(pi);
8671 set_current_media(pi);
8672 }
8673 done:
8674 PORT_UNLOCK(pi);
8675 end_synchronized_op(sc, 0);
8676 return (rc);
8677 }
8678
8679 static int
sysctl_force_fec(SYSCTL_HANDLER_ARGS)8680 sysctl_force_fec(SYSCTL_HANDLER_ARGS)
8681 {
8682 struct port_info *pi = arg1;
8683 struct adapter *sc = pi->adapter;
8684 struct link_config *lc = &pi->link_cfg;
8685 int rc, val;
8686
8687 val = lc->force_fec;
8688 MPASS(val >= -1 && val <= 1);
8689 rc = sysctl_handle_int(oidp, &val, 0, req);
8690 if (rc != 0 || req->newptr == NULL)
8691 return (rc);
8692 if (!(lc->pcaps & FW_PORT_CAP32_FORCE_FEC))
8693 return (ENOTSUP);
8694 if (val < -1 || val > 1)
8695 return (EINVAL);
8696
8697 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4ff");
8698 if (rc)
8699 return (rc);
8700 PORT_LOCK(pi);
8701 lc->force_fec = val;
8702 if (!hw_off_limits(sc)) {
8703 fixup_link_config(pi);
8704 if (pi->up_vis > 0)
8705 rc = apply_link_config(pi);
8706 }
8707 PORT_UNLOCK(pi);
8708 end_synchronized_op(sc, 0);
8709 return (rc);
8710 }
8711
8712 static int
sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)8713 sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
8714 {
8715 struct adapter *sc = arg1;
8716 int rc, reg = arg2;
8717 uint64_t val;
8718
8719 mtx_lock(&sc->reg_lock);
8720 if (hw_off_limits(sc))
8721 rc = ENXIO;
8722 else {
8723 rc = 0;
8724 val = t4_read_reg64(sc, reg);
8725 }
8726 mtx_unlock(&sc->reg_lock);
8727 if (rc == 0)
8728 rc = sysctl_handle_64(oidp, &val, 0, req);
8729 return (rc);
8730 }
8731
8732 static int
sysctl_temperature(SYSCTL_HANDLER_ARGS)8733 sysctl_temperature(SYSCTL_HANDLER_ARGS)
8734 {
8735 struct adapter *sc = arg1;
8736 int rc, t;
8737 uint32_t param, val;
8738
8739 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp");
8740 if (rc)
8741 return (rc);
8742 if (hw_off_limits(sc))
8743 rc = ENXIO;
8744 else {
8745 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
8746 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
8747 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP);
8748 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
8749 }
8750 end_synchronized_op(sc, 0);
8751 if (rc)
8752 return (rc);
8753
8754 /* unknown is returned as 0 but we display -1 in that case */
8755 t = val == 0 ? -1 : val;
8756
8757 rc = sysctl_handle_int(oidp, &t, 0, req);
8758 return (rc);
8759 }
8760
8761 static int
sysctl_vdd(SYSCTL_HANDLER_ARGS)8762 sysctl_vdd(SYSCTL_HANDLER_ARGS)
8763 {
8764 struct adapter *sc = arg1;
8765 int rc;
8766 uint32_t param, val;
8767
8768 if (sc->params.core_vdd == 0) {
8769 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
8770 "t4vdd");
8771 if (rc)
8772 return (rc);
8773 if (hw_off_limits(sc))
8774 rc = ENXIO;
8775 else {
8776 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
8777 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
8778 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_VDD);
8779 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1,
8780 ¶m, &val);
8781 }
8782 end_synchronized_op(sc, 0);
8783 if (rc)
8784 return (rc);
8785 sc->params.core_vdd = val;
8786 }
8787
8788 return (sysctl_handle_int(oidp, &sc->params.core_vdd, 0, req));
8789 }
8790
8791 static int
sysctl_reset_sensor(SYSCTL_HANDLER_ARGS)8792 sysctl_reset_sensor(SYSCTL_HANDLER_ARGS)
8793 {
8794 struct adapter *sc = arg1;
8795 int rc, v;
8796 uint32_t param, val;
8797
8798 v = sc->sensor_resets;
8799 rc = sysctl_handle_int(oidp, &v, 0, req);
8800 if (rc != 0 || req->newptr == NULL || v <= 0)
8801 return (rc);
8802
8803 if (sc->params.fw_vers < FW_VERSION32(1, 24, 7, 0) ||
8804 chip_id(sc) < CHELSIO_T5)
8805 return (ENOTSUP);
8806
8807 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4srst");
8808 if (rc)
8809 return (rc);
8810 if (hw_off_limits(sc))
8811 rc = ENXIO;
8812 else {
8813 param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
8814 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) |
8815 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_RESET_TMP_SENSOR));
8816 val = 1;
8817 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
8818 }
8819 end_synchronized_op(sc, 0);
8820 if (rc == 0)
8821 sc->sensor_resets++;
8822 return (rc);
8823 }
8824
8825 static int
sysctl_loadavg(SYSCTL_HANDLER_ARGS)8826 sysctl_loadavg(SYSCTL_HANDLER_ARGS)
8827 {
8828 struct adapter *sc = arg1;
8829 struct sbuf *sb;
8830 int rc;
8831 uint32_t param, val;
8832
8833 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4lavg");
8834 if (rc)
8835 return (rc);
8836 if (hw_off_limits(sc))
8837 rc = ENXIO;
8838 else {
8839 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
8840 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_LOAD);
8841 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val);
8842 }
8843 end_synchronized_op(sc, 0);
8844 if (rc)
8845 return (rc);
8846
8847 rc = sysctl_wire_old_buffer(req, 0);
8848 if (rc != 0)
8849 return (rc);
8850
8851 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8852 if (sb == NULL)
8853 return (ENOMEM);
8854
8855 if (val == 0xffffffff) {
8856 /* Only debug and custom firmwares report load averages. */
8857 sbuf_printf(sb, "not available");
8858 } else {
8859 sbuf_printf(sb, "%d %d %d", val & 0xff, (val >> 8) & 0xff,
8860 (val >> 16) & 0xff);
8861 }
8862 rc = sbuf_finish(sb);
8863 sbuf_delete(sb);
8864
8865 return (rc);
8866 }
8867
8868 static int
sysctl_cctrl(SYSCTL_HANDLER_ARGS)8869 sysctl_cctrl(SYSCTL_HANDLER_ARGS)
8870 {
8871 struct adapter *sc = arg1;
8872 struct sbuf *sb;
8873 int rc, i;
8874 uint16_t incr[NMTUS][NCCTRL_WIN];
8875 static const char *dec_fac[] = {
8876 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
8877 "0.9375"
8878 };
8879
8880 rc = sysctl_wire_old_buffer(req, 0);
8881 if (rc != 0)
8882 return (rc);
8883
8884 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
8885 if (sb == NULL)
8886 return (ENOMEM);
8887
8888 mtx_lock(&sc->reg_lock);
8889 if (hw_off_limits(sc))
8890 rc = ENXIO;
8891 else
8892 t4_read_cong_tbl(sc, incr);
8893 mtx_unlock(&sc->reg_lock);
8894 if (rc)
8895 goto done;
8896
8897 for (i = 0; i < NCCTRL_WIN; ++i) {
8898 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
8899 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
8900 incr[5][i], incr[6][i], incr[7][i]);
8901 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
8902 incr[8][i], incr[9][i], incr[10][i], incr[11][i],
8903 incr[12][i], incr[13][i], incr[14][i], incr[15][i],
8904 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
8905 }
8906
8907 rc = sbuf_finish(sb);
8908 done:
8909 sbuf_delete(sb);
8910 return (rc);
8911 }
8912
8913 static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = {
8914 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */
8915 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */
8916 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */
8917 };
8918
8919 static int
sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)8920 sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS)
8921 {
8922 struct adapter *sc = arg1;
8923 struct sbuf *sb;
8924 int rc, i, n, qid = arg2;
8925 uint32_t *buf, *p;
8926 char *qtype;
8927 u_int cim_num_obq = sc->chip_params->cim_num_obq;
8928
8929 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq,
8930 ("%s: bad qid %d\n", __func__, qid));
8931
8932 if (qid < CIM_NUM_IBQ) {
8933 /* inbound queue */
8934 qtype = "IBQ";
8935 n = 4 * CIM_IBQ_SIZE;
8936 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
8937 mtx_lock(&sc->reg_lock);
8938 if (hw_off_limits(sc))
8939 rc = -ENXIO;
8940 else
8941 rc = t4_read_cim_ibq(sc, qid, buf, n);
8942 mtx_unlock(&sc->reg_lock);
8943 } else {
8944 /* outbound queue */
8945 qtype = "OBQ";
8946 qid -= CIM_NUM_IBQ;
8947 n = 4 * cim_num_obq * CIM_OBQ_SIZE;
8948 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK);
8949 mtx_lock(&sc->reg_lock);
8950 if (hw_off_limits(sc))
8951 rc = -ENXIO;
8952 else
8953 rc = t4_read_cim_obq(sc, qid, buf, n);
8954 mtx_unlock(&sc->reg_lock);
8955 }
8956
8957 if (rc < 0) {
8958 rc = -rc;
8959 goto done;
8960 }
8961 n = rc * sizeof(uint32_t); /* rc has # of words actually read */
8962
8963 rc = sysctl_wire_old_buffer(req, 0);
8964 if (rc != 0)
8965 goto done;
8966
8967 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
8968 if (sb == NULL) {
8969 rc = ENOMEM;
8970 goto done;
8971 }
8972
8973 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]);
8974 for (i = 0, p = buf; i < n; i += 16, p += 4)
8975 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1],
8976 p[2], p[3]);
8977
8978 rc = sbuf_finish(sb);
8979 sbuf_delete(sb);
8980 done:
8981 free(buf, M_CXGBE);
8982 return (rc);
8983 }
8984
8985 static void
sbuf_cim_la4(struct adapter * sc,struct sbuf * sb,uint32_t * buf,uint32_t cfg)8986 sbuf_cim_la4(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg)
8987 {
8988 uint32_t *p;
8989
8990 sbuf_printf(sb, "Status Data PC%s",
8991 cfg & F_UPDBGLACAPTPCONLY ? "" :
8992 " LS0Stat LS0Addr LS0Data");
8993
8994 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) {
8995 if (cfg & F_UPDBGLACAPTPCONLY) {
8996 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff,
8997 p[6], p[7]);
8998 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x",
8999 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8,
9000 p[4] & 0xff, p[5] >> 8);
9001 sbuf_printf(sb, "\n %02x %x%07x %x%07x",
9002 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
9003 p[1] & 0xf, p[2] >> 4);
9004 } else {
9005 sbuf_printf(sb,
9006 "\n %02x %x%07x %x%07x %08x %08x "
9007 "%08x%08x%08x%08x",
9008 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4,
9009 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5],
9010 p[6], p[7]);
9011 }
9012 }
9013 }
9014
9015 static void
sbuf_cim_la6(struct adapter * sc,struct sbuf * sb,uint32_t * buf,uint32_t cfg)9016 sbuf_cim_la6(struct adapter *sc, struct sbuf *sb, uint32_t *buf, uint32_t cfg)
9017 {
9018 uint32_t *p;
9019
9020 sbuf_printf(sb, "Status Inst Data PC%s",
9021 cfg & F_UPDBGLACAPTPCONLY ? "" :
9022 " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data");
9023
9024 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) {
9025 if (cfg & F_UPDBGLACAPTPCONLY) {
9026 sbuf_printf(sb, "\n %02x %08x %08x %08x",
9027 p[3] & 0xff, p[2], p[1], p[0]);
9028 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x",
9029 (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8,
9030 p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8);
9031 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x",
9032 (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16,
9033 p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff,
9034 p[6] >> 16);
9035 } else {
9036 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x "
9037 "%08x %08x %08x %08x %08x %08x",
9038 (p[9] >> 16) & 0xff,
9039 p[9] & 0xffff, p[8] >> 16,
9040 p[8] & 0xffff, p[7] >> 16,
9041 p[7] & 0xffff, p[6] >> 16,
9042 p[2], p[1], p[0], p[5], p[4], p[3]);
9043 }
9044 }
9045 }
9046
9047 static int
sbuf_cim_la(struct adapter * sc,struct sbuf * sb,int flags)9048 sbuf_cim_la(struct adapter *sc, struct sbuf *sb, int flags)
9049 {
9050 uint32_t cfg, *buf;
9051 int rc;
9052
9053 MPASS(flags == M_WAITOK || flags == M_NOWAIT);
9054 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE,
9055 M_ZERO | flags);
9056 if (buf == NULL)
9057 return (ENOMEM);
9058
9059 mtx_lock(&sc->reg_lock);
9060 if (hw_off_limits(sc))
9061 rc = ENXIO;
9062 else {
9063 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg);
9064 if (rc == 0)
9065 rc = -t4_cim_read_la(sc, buf, NULL);
9066 }
9067 mtx_unlock(&sc->reg_lock);
9068 if (rc == 0) {
9069 if (chip_id(sc) < CHELSIO_T6)
9070 sbuf_cim_la4(sc, sb, buf, cfg);
9071 else
9072 sbuf_cim_la6(sc, sb, buf, cfg);
9073 }
9074 free(buf, M_CXGBE);
9075 return (rc);
9076 }
9077
9078 static int
sysctl_cim_la(SYSCTL_HANDLER_ARGS)9079 sysctl_cim_la(SYSCTL_HANDLER_ARGS)
9080 {
9081 struct adapter *sc = arg1;
9082 struct sbuf *sb;
9083 int rc;
9084
9085 rc = sysctl_wire_old_buffer(req, 0);
9086 if (rc != 0)
9087 return (rc);
9088 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
9089 if (sb == NULL)
9090 return (ENOMEM);
9091
9092 rc = sbuf_cim_la(sc, sb, M_WAITOK);
9093 if (rc == 0)
9094 rc = sbuf_finish(sb);
9095 sbuf_delete(sb);
9096 return (rc);
9097 }
9098
9099 static void
dump_cim_regs(struct adapter * sc)9100 dump_cim_regs(struct adapter *sc)
9101 {
9102 log(LOG_DEBUG, "%s: CIM debug regs1 %08x %08x %08x %08x %08x\n",
9103 device_get_nameunit(sc->dev),
9104 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0),
9105 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1),
9106 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA2),
9107 t4_read_reg(sc, A_EDC_H_BIST_DATA_PATTERN),
9108 t4_read_reg(sc, A_EDC_H_BIST_STATUS_RDATA));
9109 log(LOG_DEBUG, "%s: CIM debug regs2 %08x %08x %08x %08x %08x\n",
9110 device_get_nameunit(sc->dev),
9111 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0),
9112 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1),
9113 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA0 + 0x800),
9114 t4_read_reg(sc, A_EDC_H_BIST_USER_WDATA1 + 0x800),
9115 t4_read_reg(sc, A_EDC_H_BIST_CMD_LEN));
9116 }
9117
9118 static void
dump_cimla(struct adapter * sc)9119 dump_cimla(struct adapter *sc)
9120 {
9121 struct sbuf sb;
9122 int rc;
9123
9124 if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb) {
9125 log(LOG_DEBUG, "%s: failed to generate CIM LA dump.\n",
9126 device_get_nameunit(sc->dev));
9127 return;
9128 }
9129 rc = sbuf_cim_la(sc, &sb, M_WAITOK);
9130 if (rc == 0) {
9131 rc = sbuf_finish(&sb);
9132 if (rc == 0) {
9133 log(LOG_DEBUG, "%s: CIM LA dump follows.\n%s\n",
9134 device_get_nameunit(sc->dev), sbuf_data(&sb));
9135 }
9136 }
9137 sbuf_delete(&sb);
9138 }
9139
9140 void
t4_os_cim_err(struct adapter * sc)9141 t4_os_cim_err(struct adapter *sc)
9142 {
9143 atomic_set_int(&sc->error_flags, ADAP_CIM_ERR);
9144 }
9145
9146 static int
sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)9147 sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS)
9148 {
9149 struct adapter *sc = arg1;
9150 u_int i;
9151 struct sbuf *sb;
9152 uint32_t *buf, *p;
9153 int rc;
9154
9155 rc = sysctl_wire_old_buffer(req, 0);
9156 if (rc != 0)
9157 return (rc);
9158
9159 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
9160 if (sb == NULL)
9161 return (ENOMEM);
9162
9163 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE,
9164 M_ZERO | M_WAITOK);
9165
9166 mtx_lock(&sc->reg_lock);
9167 if (hw_off_limits(sc))
9168 rc = ENXIO;
9169 else
9170 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE);
9171 mtx_unlock(&sc->reg_lock);
9172 if (rc)
9173 goto done;
9174
9175 p = buf;
9176 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
9177 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2],
9178 p[1], p[0]);
9179 }
9180
9181 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD");
9182 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) {
9183 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u",
9184 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7,
9185 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1,
9186 (p[1] >> 2) | ((p[2] & 3) << 30),
9187 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1,
9188 p[0] & 1);
9189 }
9190 rc = sbuf_finish(sb);
9191 done:
9192 sbuf_delete(sb);
9193 free(buf, M_CXGBE);
9194 return (rc);
9195 }
9196
9197 static int
sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)9198 sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS)
9199 {
9200 struct adapter *sc = arg1;
9201 u_int i;
9202 struct sbuf *sb;
9203 uint32_t *buf, *p;
9204 int rc;
9205
9206 rc = sysctl_wire_old_buffer(req, 0);
9207 if (rc != 0)
9208 return (rc);
9209
9210 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
9211 if (sb == NULL)
9212 return (ENOMEM);
9213
9214 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE,
9215 M_ZERO | M_WAITOK);
9216
9217 mtx_lock(&sc->reg_lock);
9218 if (hw_off_limits(sc))
9219 rc = ENXIO;
9220 else
9221 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL);
9222 mtx_unlock(&sc->reg_lock);
9223 if (rc)
9224 goto done;
9225
9226 p = buf;
9227 sbuf_printf(sb, "Cntl ID DataBE Addr Data");
9228 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
9229 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x",
9230 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff,
9231 p[4], p[3], p[2], p[1], p[0]);
9232 }
9233
9234 sbuf_printf(sb, "\n\nCntl ID Data");
9235 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) {
9236 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x",
9237 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]);
9238 }
9239
9240 rc = sbuf_finish(sb);
9241 done:
9242 sbuf_delete(sb);
9243 free(buf, M_CXGBE);
9244 return (rc);
9245 }
9246
9247 static int
sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)9248 sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS)
9249 {
9250 struct adapter *sc = arg1;
9251 struct sbuf *sb;
9252 int rc, i;
9253 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
9254 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5];
9255 uint16_t thres[CIM_NUM_IBQ];
9256 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr;
9257 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat;
9258 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq;
9259
9260 cim_num_obq = sc->chip_params->cim_num_obq;
9261 if (is_t4(sc)) {
9262 ibq_rdaddr = A_UP_IBQ_0_RDADDR;
9263 obq_rdaddr = A_UP_OBQ_0_REALADDR;
9264 } else {
9265 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR;
9266 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR;
9267 }
9268 nq = CIM_NUM_IBQ + cim_num_obq;
9269
9270 mtx_lock(&sc->reg_lock);
9271 if (hw_off_limits(sc))
9272 rc = ENXIO;
9273 else {
9274 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat);
9275 if (rc == 0) {
9276 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq,
9277 obq_wr);
9278 if (rc == 0)
9279 t4_read_cimq_cfg(sc, base, size, thres);
9280 }
9281 }
9282 mtx_unlock(&sc->reg_lock);
9283 if (rc)
9284 return (rc);
9285
9286 rc = sysctl_wire_old_buffer(req, 0);
9287 if (rc != 0)
9288 return (rc);
9289
9290 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req);
9291 if (sb == NULL)
9292 return (ENOMEM);
9293
9294 sbuf_printf(sb,
9295 " Queue Base Size Thres RdPtr WrPtr SOP EOP Avail");
9296
9297 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4)
9298 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u",
9299 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]),
9300 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
9301 G_QUEREMFLITS(p[2]) * 16);
9302 for ( ; i < nq; i++, p += 4, wr += 2)
9303 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i],
9304 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff,
9305 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]),
9306 G_QUEREMFLITS(p[2]) * 16);
9307
9308 rc = sbuf_finish(sb);
9309 sbuf_delete(sb);
9310
9311 return (rc);
9312 }
9313
9314 static int
sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)9315 sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
9316 {
9317 struct adapter *sc = arg1;
9318 struct sbuf *sb;
9319 int rc;
9320 struct tp_cpl_stats stats;
9321
9322 rc = sysctl_wire_old_buffer(req, 0);
9323 if (rc != 0)
9324 return (rc);
9325
9326 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
9327 if (sb == NULL)
9328 return (ENOMEM);
9329
9330 mtx_lock(&sc->reg_lock);
9331 if (hw_off_limits(sc))
9332 rc = ENXIO;
9333 else
9334 t4_tp_get_cpl_stats(sc, &stats, 0);
9335 mtx_unlock(&sc->reg_lock);
9336 if (rc)
9337 goto done;
9338
9339 if (sc->chip_params->nchan > 2) {
9340 sbuf_printf(sb, " channel 0 channel 1"
9341 " channel 2 channel 3");
9342 sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u",
9343 stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
9344 sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u",
9345 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
9346 } else {
9347 sbuf_printf(sb, " channel 0 channel 1");
9348 sbuf_printf(sb, "\nCPL requests: %10u %10u",
9349 stats.req[0], stats.req[1]);
9350 sbuf_printf(sb, "\nCPL responses: %10u %10u",
9351 stats.rsp[0], stats.rsp[1]);
9352 }
9353
9354 rc = sbuf_finish(sb);
9355 done:
9356 sbuf_delete(sb);
9357 return (rc);
9358 }
9359
9360 static int
sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)9361 sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
9362 {
9363 struct adapter *sc = arg1;
9364 struct sbuf *sb;
9365 int rc;
9366 struct tp_usm_stats stats;
9367
9368 rc = sysctl_wire_old_buffer(req, 0);
9369 if (rc != 0)
9370 return(rc);
9371
9372 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
9373 if (sb == NULL)
9374 return (ENOMEM);
9375
9376 mtx_lock(&sc->reg_lock);
9377 if (hw_off_limits(sc))
9378 rc = ENXIO;
9379 else
9380 t4_get_usm_stats(sc, &stats, 1);
9381 mtx_unlock(&sc->reg_lock);
9382 if (rc == 0) {
9383 sbuf_printf(sb, "Frames: %u\n", stats.frames);
9384 sbuf_printf(sb, "Octets: %ju\n", stats.octets);
9385 sbuf_printf(sb, "Drops: %u", stats.drops);
9386 rc = sbuf_finish(sb);
9387 }
9388 sbuf_delete(sb);
9389
9390 return (rc);
9391 }
9392
9393 static int
sysctl_tid_stats(SYSCTL_HANDLER_ARGS)9394 sysctl_tid_stats(SYSCTL_HANDLER_ARGS)
9395 {
9396 struct adapter *sc = arg1;
9397 struct sbuf *sb;
9398 int rc;
9399 struct tp_tid_stats stats;
9400
9401 rc = sysctl_wire_old_buffer(req, 0);
9402 if (rc != 0)
9403 return(rc);
9404
9405 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
9406 if (sb == NULL)
9407 return (ENOMEM);
9408
9409 mtx_lock(&sc->reg_lock);
9410 if (hw_off_limits(sc))
9411 rc = ENXIO;
9412 else
9413 t4_tp_get_tid_stats(sc, &stats, 1);
9414 mtx_unlock(&sc->reg_lock);
9415 if (rc == 0) {
9416 sbuf_printf(sb, "Delete: %u\n", stats.del);
9417 sbuf_printf(sb, "Invalidate: %u\n", stats.inv);
9418 sbuf_printf(sb, "Active: %u\n", stats.act);
9419 sbuf_printf(sb, "Passive: %u", stats.pas);
9420 rc = sbuf_finish(sb);
9421 }
9422 sbuf_delete(sb);
9423
9424 return (rc);
9425 }
9426
9427 static const char * const devlog_level_strings[] = {
9428 [FW_DEVLOG_LEVEL_EMERG] = "EMERG",
9429 [FW_DEVLOG_LEVEL_CRIT] = "CRIT",
9430 [FW_DEVLOG_LEVEL_ERR] = "ERR",
9431 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE",
9432 [FW_DEVLOG_LEVEL_INFO] = "INFO",
9433 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG"
9434 };
9435
9436 static const char * const devlog_facility_strings[] = {
9437 [FW_DEVLOG_FACILITY_CORE] = "CORE",
9438 [FW_DEVLOG_FACILITY_CF] = "CF",
9439 [FW_DEVLOG_FACILITY_SCHED] = "SCHED",
9440 [FW_DEVLOG_FACILITY_TIMER] = "TIMER",
9441 [FW_DEVLOG_FACILITY_RES] = "RES",
9442 [FW_DEVLOG_FACILITY_HW] = "HW",
9443 [FW_DEVLOG_FACILITY_FLR] = "FLR",
9444 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ",
9445 [FW_DEVLOG_FACILITY_PHY] = "PHY",
9446 [FW_DEVLOG_FACILITY_MAC] = "MAC",
9447 [FW_DEVLOG_FACILITY_PORT] = "PORT",
9448 [FW_DEVLOG_FACILITY_VI] = "VI",
9449 [FW_DEVLOG_FACILITY_FILTER] = "FILTER",
9450 [FW_DEVLOG_FACILITY_ACL] = "ACL",
9451 [FW_DEVLOG_FACILITY_TM] = "TM",
9452 [FW_DEVLOG_FACILITY_QFC] = "QFC",
9453 [FW_DEVLOG_FACILITY_DCB] = "DCB",
9454 [FW_DEVLOG_FACILITY_ETH] = "ETH",
9455 [FW_DEVLOG_FACILITY_OFLD] = "OFLD",
9456 [FW_DEVLOG_FACILITY_RI] = "RI",
9457 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI",
9458 [FW_DEVLOG_FACILITY_FCOE] = "FCOE",
9459 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI",
9460 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE",
9461 [FW_DEVLOG_FACILITY_CHNET] = "CHNET",
9462 };
9463
9464 static int
sbuf_devlog(struct adapter * sc,struct sbuf * sb,int flags)9465 sbuf_devlog(struct adapter *sc, struct sbuf *sb, int flags)
9466 {
9467 int i, j, rc, nentries, first = 0;
9468 struct devlog_params *dparams = &sc->params.devlog;
9469 struct fw_devlog_e *buf, *e;
9470 uint64_t ftstamp = UINT64_MAX;
9471
9472 if (dparams->addr == 0)
9473 return (ENXIO);
9474
9475 MPASS(flags == M_WAITOK || flags == M_NOWAIT);
9476 buf = malloc(dparams->size, M_CXGBE, M_ZERO | flags);
9477 if (buf == NULL)
9478 return (ENOMEM);
9479
9480 mtx_lock(&sc->reg_lock);
9481 if (hw_off_limits(sc))
9482 rc = ENXIO;
9483 else
9484 rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf,
9485 dparams->size);
9486 mtx_unlock(&sc->reg_lock);
9487 if (rc != 0)
9488 goto done;
9489
9490 nentries = dparams->size / sizeof(struct fw_devlog_e);
9491 for (i = 0; i < nentries; i++) {
9492 e = &buf[i];
9493
9494 if (e->timestamp == 0)
9495 break; /* end */
9496
9497 e->timestamp = be64toh(e->timestamp);
9498 e->seqno = be32toh(e->seqno);
9499 for (j = 0; j < 8; j++)
9500 e->params[j] = be32toh(e->params[j]);
9501
9502 if (e->timestamp < ftstamp) {
9503 ftstamp = e->timestamp;
9504 first = i;
9505 }
9506 }
9507
9508 if (buf[first].timestamp == 0)
9509 goto done; /* nothing in the log */
9510
9511 sbuf_printf(sb, "%10s %15s %8s %8s %s\n",
9512 "Seq#", "Tstamp", "Level", "Facility", "Message");
9513
9514 i = first;
9515 do {
9516 e = &buf[i];
9517 if (e->timestamp == 0)
9518 break; /* end */
9519
9520 sbuf_printf(sb, "%10d %15ju %8s %8s ",
9521 e->seqno, e->timestamp,
9522 (e->level < nitems(devlog_level_strings) ?
9523 devlog_level_strings[e->level] : "UNKNOWN"),
9524 (e->facility < nitems(devlog_facility_strings) ?
9525 devlog_facility_strings[e->facility] : "UNKNOWN"));
9526 sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
9527 e->params[2], e->params[3], e->params[4],
9528 e->params[5], e->params[6], e->params[7]);
9529
9530 if (++i == nentries)
9531 i = 0;
9532 } while (i != first);
9533 done:
9534 free(buf, M_CXGBE);
9535 return (rc);
9536 }
9537
9538 static int
sysctl_devlog(SYSCTL_HANDLER_ARGS)9539 sysctl_devlog(SYSCTL_HANDLER_ARGS)
9540 {
9541 struct adapter *sc = arg1;
9542 int rc;
9543 struct sbuf *sb;
9544
9545 rc = sysctl_wire_old_buffer(req, 0);
9546 if (rc != 0)
9547 return (rc);
9548 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
9549 if (sb == NULL)
9550 return (ENOMEM);
9551
9552 rc = sbuf_devlog(sc, sb, M_WAITOK);
9553 if (rc == 0)
9554 rc = sbuf_finish(sb);
9555 sbuf_delete(sb);
9556 return (rc);
9557 }
9558
9559 static void
dump_devlog(struct adapter * sc)9560 dump_devlog(struct adapter *sc)
9561 {
9562 int rc;
9563 struct sbuf sb;
9564
9565 if (sbuf_new(&sb, NULL, 4096, SBUF_AUTOEXTEND) != &sb) {
9566 log(LOG_DEBUG, "%s: failed to generate devlog dump.\n",
9567 device_get_nameunit(sc->dev));
9568 return;
9569 }
9570 rc = sbuf_devlog(sc, &sb, M_WAITOK);
9571 if (rc == 0) {
9572 rc = sbuf_finish(&sb);
9573 if (rc == 0) {
9574 log(LOG_DEBUG, "%s: device log follows.\n%s",
9575 device_get_nameunit(sc->dev), sbuf_data(&sb));
9576 }
9577 }
9578 sbuf_delete(&sb);
9579 }
9580
9581 static int
sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)9582 sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
9583 {
9584 struct adapter *sc = arg1;
9585 struct sbuf *sb;
9586 int rc;
9587 struct tp_fcoe_stats stats[MAX_NCHAN];
9588 int i, nchan = sc->chip_params->nchan;
9589
9590 rc = sysctl_wire_old_buffer(req, 0);
9591 if (rc != 0)
9592 return (rc);
9593
9594 mtx_lock(&sc->reg_lock);
9595 if (hw_off_limits(sc))
9596 rc = ENXIO;
9597 else {
9598 for (i = 0; i < nchan; i++)
9599 t4_get_fcoe_stats(sc, i, &stats[i], 1);
9600 }
9601 mtx_unlock(&sc->reg_lock);
9602 if (rc != 0)
9603 return (rc);
9604
9605 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
9606 if (sb == NULL)
9607 return (ENOMEM);
9608
9609 if (nchan > 2) {
9610 sbuf_printf(sb, " channel 0 channel 1"
9611 " channel 2 channel 3");
9612 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju",
9613 stats[0].octets_ddp, stats[1].octets_ddp,
9614 stats[2].octets_ddp, stats[3].octets_ddp);
9615 sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u",
9616 stats[0].frames_ddp, stats[1].frames_ddp,
9617 stats[2].frames_ddp, stats[3].frames_ddp);
9618 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u",
9619 stats[0].frames_drop, stats[1].frames_drop,
9620 stats[2].frames_drop, stats[3].frames_drop);
9621 } else {
9622 sbuf_printf(sb, " channel 0 channel 1");
9623 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju",
9624 stats[0].octets_ddp, stats[1].octets_ddp);
9625 sbuf_printf(sb, "\nframesDDP: %16u %16u",
9626 stats[0].frames_ddp, stats[1].frames_ddp);
9627 sbuf_printf(sb, "\nframesDrop: %16u %16u",
9628 stats[0].frames_drop, stats[1].frames_drop);
9629 }
9630
9631 rc = sbuf_finish(sb);
9632 sbuf_delete(sb);
9633
9634 return (rc);
9635 }
9636
9637 static int
sysctl_hw_sched(SYSCTL_HANDLER_ARGS)9638 sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
9639 {
9640 struct adapter *sc = arg1;
9641 struct sbuf *sb;
9642 int rc, i;
9643 unsigned int map, kbps, ipg, mode;
9644 unsigned int pace_tab[NTX_SCHED];
9645
9646 rc = sysctl_wire_old_buffer(req, 0);
9647 if (rc != 0)
9648 return (rc);
9649
9650 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
9651 if (sb == NULL)
9652 return (ENOMEM);
9653
9654 mtx_lock(&sc->reg_lock);
9655 if (hw_off_limits(sc)) {
9656 rc = ENXIO;
9657 goto done;
9658 }
9659
9660 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
9661 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
9662 t4_read_pace_tbl(sc, pace_tab);
9663
9664 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) "
9665 "Class IPG (0.1 ns) Flow IPG (us)");
9666
9667 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
9668 t4_get_tx_sched(sc, i, &kbps, &ipg, 1);
9669 sbuf_printf(sb, "\n %u %-5s %u ", i,
9670 (mode & (1 << i)) ? "flow" : "class", map & 3);
9671 if (kbps)
9672 sbuf_printf(sb, "%9u ", kbps);
9673 else
9674 sbuf_printf(sb, " disabled ");
9675
9676 if (ipg)
9677 sbuf_printf(sb, "%13u ", ipg);
9678 else
9679 sbuf_printf(sb, " disabled ");
9680
9681 if (pace_tab[i])
9682 sbuf_printf(sb, "%10u", pace_tab[i]);
9683 else
9684 sbuf_printf(sb, " disabled");
9685 }
9686 rc = sbuf_finish(sb);
9687 done:
9688 mtx_unlock(&sc->reg_lock);
9689 sbuf_delete(sb);
9690 return (rc);
9691 }
9692
9693 static int
sysctl_lb_stats(SYSCTL_HANDLER_ARGS)9694 sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
9695 {
9696 struct adapter *sc = arg1;
9697 struct sbuf *sb;
9698 int rc, i, j;
9699 uint64_t *p0, *p1;
9700 struct lb_port_stats s[2];
9701 static const char *stat_name[] = {
9702 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
9703 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
9704 "Frames128To255:", "Frames256To511:", "Frames512To1023:",
9705 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
9706 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
9707 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
9708 "BG2FramesTrunc:", "BG3FramesTrunc:"
9709 };
9710
9711 rc = sysctl_wire_old_buffer(req, 0);
9712 if (rc != 0)
9713 return (rc);
9714
9715 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
9716 if (sb == NULL)
9717 return (ENOMEM);
9718
9719 memset(s, 0, sizeof(s));
9720
9721 for (i = 0; i < sc->chip_params->nchan; i += 2) {
9722 mtx_lock(&sc->reg_lock);
9723 if (hw_off_limits(sc))
9724 rc = ENXIO;
9725 else {
9726 t4_get_lb_stats(sc, i, &s[0]);
9727 t4_get_lb_stats(sc, i + 1, &s[1]);
9728 }
9729 mtx_unlock(&sc->reg_lock);
9730 if (rc != 0)
9731 break;
9732
9733 p0 = &s[0].octets;
9734 p1 = &s[1].octets;
9735 sbuf_printf(sb, "%s Loopback %u"
9736 " Loopback %u", i == 0 ? "" : "\n", i, i + 1);
9737
9738 for (j = 0; j < nitems(stat_name); j++)
9739 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
9740 *p0++, *p1++);
9741 }
9742
9743 rc = sbuf_finish(sb);
9744 sbuf_delete(sb);
9745
9746 return (rc);
9747 }
9748
9749 static int
sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)9750 sysctl_linkdnrc(SYSCTL_HANDLER_ARGS)
9751 {
9752 int rc = 0;
9753 struct port_info *pi = arg1;
9754 struct link_config *lc = &pi->link_cfg;
9755 struct sbuf *sb;
9756
9757 rc = sysctl_wire_old_buffer(req, 0);
9758 if (rc != 0)
9759 return(rc);
9760 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req);
9761 if (sb == NULL)
9762 return (ENOMEM);
9763
9764 if (lc->link_ok || lc->link_down_rc == 255)
9765 sbuf_printf(sb, "n/a");
9766 else
9767 sbuf_printf(sb, "%s", t4_link_down_rc_str(lc->link_down_rc));
9768
9769 rc = sbuf_finish(sb);
9770 sbuf_delete(sb);
9771
9772 return (rc);
9773 }
9774
9775 struct mem_desc {
9776 u_int base;
9777 u_int limit;
9778 u_int idx;
9779 };
9780
9781 static int
mem_desc_cmp(const void * a,const void * b)9782 mem_desc_cmp(const void *a, const void *b)
9783 {
9784 const u_int v1 = ((const struct mem_desc *)a)->base;
9785 const u_int v2 = ((const struct mem_desc *)b)->base;
9786
9787 if (v1 < v2)
9788 return (-1);
9789 else if (v1 > v2)
9790 return (1);
9791
9792 return (0);
9793 }
9794
9795 static void
mem_region_show(struct sbuf * sb,const char * name,unsigned int from,unsigned int to)9796 mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
9797 unsigned int to)
9798 {
9799 unsigned int size;
9800
9801 if (from == to)
9802 return;
9803
9804 size = to - from + 1;
9805 if (size == 0)
9806 return;
9807
9808 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
9809 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
9810 }
9811
9812 static int
sysctl_meminfo(SYSCTL_HANDLER_ARGS)9813 sysctl_meminfo(SYSCTL_HANDLER_ARGS)
9814 {
9815 struct adapter *sc = arg1;
9816 struct sbuf *sb;
9817 int rc, i, n;
9818 uint32_t lo, hi, used, free, alloc;
9819 static const char *memory[] = {
9820 "EDC0:", "EDC1:", "MC:", "MC0:", "MC1:", "HMA:"
9821 };
9822 static const char *region[] = {
9823 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
9824 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
9825 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
9826 "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
9827 "RQUDP region:", "PBL region:", "TXPBL region:",
9828 "TLSKey region:", "DBVFIFO region:", "ULPRX state:",
9829 "ULPTX state:", "On-chip queues:",
9830 };
9831 struct mem_desc avail[4];
9832 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */
9833 struct mem_desc *md = mem;
9834
9835 rc = sysctl_wire_old_buffer(req, 0);
9836 if (rc != 0)
9837 return (rc);
9838
9839 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
9840 if (sb == NULL)
9841 return (ENOMEM);
9842
9843 for (i = 0; i < nitems(mem); i++) {
9844 mem[i].limit = 0;
9845 mem[i].idx = i;
9846 }
9847
9848 mtx_lock(&sc->reg_lock);
9849 if (hw_off_limits(sc)) {
9850 rc = ENXIO;
9851 goto done;
9852 }
9853
9854 /* Find and sort the populated memory ranges */
9855 i = 0;
9856 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
9857 if (lo & F_EDRAM0_ENABLE) {
9858 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
9859 avail[i].base = G_EDRAM0_BASE(hi) << 20;
9860 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
9861 avail[i].idx = 0;
9862 i++;
9863 }
9864 if (lo & F_EDRAM1_ENABLE) {
9865 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
9866 avail[i].base = G_EDRAM1_BASE(hi) << 20;
9867 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
9868 avail[i].idx = 1;
9869 i++;
9870 }
9871 if (lo & F_EXT_MEM_ENABLE) {
9872 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
9873 avail[i].base = G_EXT_MEM_BASE(hi) << 20;
9874 avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
9875 avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */
9876 i++;
9877 }
9878 if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) {
9879 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
9880 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
9881 avail[i].limit = avail[i].base + (G_EXT_MEM1_SIZE(hi) << 20);
9882 avail[i].idx = 4;
9883 i++;
9884 }
9885 if (is_t6(sc) && lo & F_HMA_MUX) {
9886 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR);
9887 avail[i].base = G_EXT_MEM1_BASE(hi) << 20;
9888 avail[i].limit = avail[i].base + (G_EXT_MEM1_SIZE(hi) << 20);
9889 avail[i].idx = 5;
9890 i++;
9891 }
9892 MPASS(i <= nitems(avail));
9893 if (!i) /* no memory available */
9894 goto done;
9895 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
9896
9897 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
9898 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
9899 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
9900 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
9901 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
9902 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
9903 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
9904 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
9905 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
9906
9907 /* the next few have explicit upper bounds */
9908 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
9909 md->limit = md->base - 1 +
9910 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
9911 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
9912 md++;
9913
9914 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
9915 md->limit = md->base - 1 +
9916 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
9917 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
9918 md++;
9919
9920 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
9921 if (chip_id(sc) <= CHELSIO_T5)
9922 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
9923 else
9924 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR);
9925 md->limit = 0;
9926 } else {
9927 md->base = 0;
9928 md->idx = nitems(region); /* hide it */
9929 }
9930 md++;
9931
9932 #define ulp_region(reg) \
9933 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
9934 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
9935
9936 ulp_region(RX_ISCSI);
9937 ulp_region(RX_TDDP);
9938 ulp_region(TX_TPT);
9939 ulp_region(RX_STAG);
9940 ulp_region(RX_RQ);
9941 ulp_region(RX_RQUDP);
9942 ulp_region(RX_PBL);
9943 ulp_region(TX_PBL);
9944 if (sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS) {
9945 ulp_region(RX_TLS_KEY);
9946 }
9947 #undef ulp_region
9948
9949 md->base = 0;
9950 if (is_t4(sc))
9951 md->idx = nitems(region);
9952 else {
9953 uint32_t size = 0;
9954 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2);
9955 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE);
9956
9957 if (is_t5(sc)) {
9958 if (sge_ctrl & F_VFIFO_ENABLE)
9959 size = fifo_size << 2;
9960 } else
9961 size = G_T6_DBVFIFO_SIZE(fifo_size) << 6;
9962
9963 if (size) {
9964 md->base = t4_read_reg(sc, A_SGE_DBVFIFO_BADDR);
9965 md->limit = md->base + size - 1;
9966 } else
9967 md->idx = nitems(region);
9968 }
9969 md++;
9970
9971 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
9972 md->limit = 0;
9973 md++;
9974 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
9975 md->limit = 0;
9976 md++;
9977
9978 md->base = sc->vres.ocq.start;
9979 if (sc->vres.ocq.size)
9980 md->limit = md->base + sc->vres.ocq.size - 1;
9981 else
9982 md->idx = nitems(region); /* hide it */
9983 md++;
9984
9985 /* add any address-space holes, there can be up to 3 */
9986 for (n = 0; n < i - 1; n++)
9987 if (avail[n].limit < avail[n + 1].base)
9988 (md++)->base = avail[n].limit;
9989 if (avail[n].limit)
9990 (md++)->base = avail[n].limit;
9991
9992 n = md - mem;
9993 MPASS(n <= nitems(mem));
9994 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
9995
9996 for (lo = 0; lo < i; lo++)
9997 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
9998 avail[lo].limit - 1);
9999
10000 sbuf_printf(sb, "\n");
10001 for (i = 0; i < n; i++) {
10002 if (mem[i].idx >= nitems(region))
10003 continue; /* skip holes */
10004 if (!mem[i].limit)
10005 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
10006 mem_region_show(sb, region[mem[i].idx], mem[i].base,
10007 mem[i].limit);
10008 }
10009
10010 sbuf_printf(sb, "\n");
10011 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
10012 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
10013 mem_region_show(sb, "uP RAM:", lo, hi);
10014
10015 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
10016 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
10017 mem_region_show(sb, "uP Extmem2:", lo, hi);
10018
10019 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
10020 for (i = 0, free = 0; i < 2; i++)
10021 free += G_FREERXPAGECOUNT(t4_read_reg(sc, A_TP_FLM_FREE_RX_CNT));
10022 sbuf_printf(sb, "\n%u Rx pages (%u free) of size %uKiB for %u channels\n",
10023 G_PMRXMAXPAGE(lo), free,
10024 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
10025 (lo & F_PMRXNUMCHN) ? 2 : 1);
10026
10027 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
10028 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
10029 for (i = 0, free = 0; i < 4; i++)
10030 free += G_FREETXPAGECOUNT(t4_read_reg(sc, A_TP_FLM_FREE_TX_CNT));
10031 sbuf_printf(sb, "%u Tx pages (%u free) of size %u%ciB for %u channels\n",
10032 G_PMTXMAXPAGE(lo), free,
10033 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
10034 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
10035 sbuf_printf(sb, "%u p-structs (%u free)\n",
10036 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT),
10037 G_FREEPSTRUCTCOUNT(t4_read_reg(sc, A_TP_FLM_FREE_PS_CNT)));
10038
10039 for (i = 0; i < 4; i++) {
10040 if (chip_id(sc) > CHELSIO_T5)
10041 lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4);
10042 else
10043 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
10044 if (is_t5(sc)) {
10045 used = G_T5_USED(lo);
10046 alloc = G_T5_ALLOC(lo);
10047 } else {
10048 used = G_USED(lo);
10049 alloc = G_ALLOC(lo);
10050 }
10051 /* For T6 these are MAC buffer groups */
10052 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
10053 i, used, alloc);
10054 }
10055 for (i = 0; i < sc->chip_params->nchan; i++) {
10056 if (chip_id(sc) > CHELSIO_T5)
10057 lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4);
10058 else
10059 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
10060 if (is_t5(sc)) {
10061 used = G_T5_USED(lo);
10062 alloc = G_T5_ALLOC(lo);
10063 } else {
10064 used = G_USED(lo);
10065 alloc = G_ALLOC(lo);
10066 }
10067 /* For T6 these are MAC buffer groups */
10068 sbuf_printf(sb,
10069 "\nLoopback %d using %u pages out of %u allocated",
10070 i, used, alloc);
10071 }
10072 done:
10073 mtx_unlock(&sc->reg_lock);
10074 if (rc == 0)
10075 rc = sbuf_finish(sb);
10076 sbuf_delete(sb);
10077 return (rc);
10078 }
10079
10080 static inline void
tcamxy2valmask(uint64_t x,uint64_t y,uint8_t * addr,uint64_t * mask)10081 tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask)
10082 {
10083 *mask = x | y;
10084 y = htobe64(y);
10085 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN);
10086 }
10087
10088 static int
sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)10089 sysctl_mps_tcam(SYSCTL_HANDLER_ARGS)
10090 {
10091 struct adapter *sc = arg1;
10092 struct sbuf *sb;
10093 int rc, i;
10094
10095 MPASS(chip_id(sc) <= CHELSIO_T5);
10096
10097 rc = sysctl_wire_old_buffer(req, 0);
10098 if (rc != 0)
10099 return (rc);
10100
10101 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
10102 if (sb == NULL)
10103 return (ENOMEM);
10104
10105 sbuf_printf(sb,
10106 "Idx Ethernet address Mask Vld Ports PF"
10107 " VF Replication P0 P1 P2 P3 ML");
10108 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
10109 uint64_t tcamx, tcamy, mask;
10110 uint32_t cls_lo, cls_hi;
10111 uint8_t addr[ETHER_ADDR_LEN];
10112
10113 mtx_lock(&sc->reg_lock);
10114 if (hw_off_limits(sc))
10115 rc = ENXIO;
10116 else {
10117 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i));
10118 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i));
10119 }
10120 mtx_unlock(&sc->reg_lock);
10121 if (rc != 0)
10122 break;
10123 if (tcamx & tcamy)
10124 continue;
10125 tcamxy2valmask(tcamx, tcamy, addr, &mask);
10126 mtx_lock(&sc->reg_lock);
10127 if (hw_off_limits(sc))
10128 rc = ENXIO;
10129 else {
10130 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
10131 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
10132 }
10133 mtx_unlock(&sc->reg_lock);
10134 if (rc != 0)
10135 break;
10136 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx"
10137 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2],
10138 addr[3], addr[4], addr[5], (uintmax_t)mask,
10139 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N',
10140 G_PORTMAP(cls_hi), G_PF(cls_lo),
10141 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1);
10142
10143 if (cls_lo & F_REPLICATE) {
10144 struct fw_ldst_cmd ldst_cmd;
10145
10146 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
10147 ldst_cmd.op_to_addrspace =
10148 htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
10149 F_FW_CMD_REQUEST | F_FW_CMD_READ |
10150 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
10151 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
10152 ldst_cmd.u.mps.rplc.fid_idx =
10153 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
10154 V_FW_LDST_CMD_IDX(i));
10155
10156 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
10157 "t4mps");
10158 if (rc)
10159 break;
10160 if (hw_off_limits(sc))
10161 rc = ENXIO;
10162 else
10163 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
10164 sizeof(ldst_cmd), &ldst_cmd);
10165 end_synchronized_op(sc, 0);
10166 if (rc != 0)
10167 break;
10168 else {
10169 sbuf_printf(sb, " %08x %08x %08x %08x",
10170 be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
10171 be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
10172 be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
10173 be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
10174 }
10175 } else
10176 sbuf_printf(sb, "%36s", "");
10177
10178 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo),
10179 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo),
10180 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf);
10181 }
10182
10183 if (rc)
10184 (void) sbuf_finish(sb);
10185 else
10186 rc = sbuf_finish(sb);
10187 sbuf_delete(sb);
10188
10189 return (rc);
10190 }
10191
10192 static int
sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS)10193 sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS)
10194 {
10195 struct adapter *sc = arg1;
10196 struct sbuf *sb;
10197 int rc, i;
10198
10199 MPASS(chip_id(sc) > CHELSIO_T5);
10200
10201 rc = sysctl_wire_old_buffer(req, 0);
10202 if (rc != 0)
10203 return (rc);
10204
10205 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
10206 if (sb == NULL)
10207 return (ENOMEM);
10208
10209 sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask"
10210 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF"
10211 " Replication"
10212 " P0 P1 P2 P3 ML\n");
10213
10214 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) {
10215 uint8_t dip_hit, vlan_vld, lookup_type, port_num;
10216 uint16_t ivlan;
10217 uint64_t tcamx, tcamy, val, mask;
10218 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy;
10219 uint8_t addr[ETHER_ADDR_LEN];
10220
10221 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0);
10222 if (i < 256)
10223 ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0);
10224 else
10225 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1);
10226 mtx_lock(&sc->reg_lock);
10227 if (hw_off_limits(sc))
10228 rc = ENXIO;
10229 else {
10230 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
10231 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
10232 tcamy = G_DMACH(val) << 32;
10233 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
10234 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
10235 }
10236 mtx_unlock(&sc->reg_lock);
10237 if (rc != 0)
10238 break;
10239
10240 lookup_type = G_DATALKPTYPE(data2);
10241 port_num = G_DATAPORTNUM(data2);
10242 if (lookup_type && lookup_type != M_DATALKPTYPE) {
10243 /* Inner header VNI */
10244 vniy = ((data2 & F_DATAVIDH2) << 23) |
10245 (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
10246 dip_hit = data2 & F_DATADIPHIT;
10247 vlan_vld = 0;
10248 } else {
10249 vniy = 0;
10250 dip_hit = 0;
10251 vlan_vld = data2 & F_DATAVIDH2;
10252 ivlan = G_VIDL(val);
10253 }
10254
10255 ctl |= V_CTLXYBITSEL(1);
10256 mtx_lock(&sc->reg_lock);
10257 if (hw_off_limits(sc))
10258 rc = ENXIO;
10259 else {
10260 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl);
10261 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1);
10262 tcamx = G_DMACH(val) << 32;
10263 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1);
10264 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1);
10265 }
10266 mtx_unlock(&sc->reg_lock);
10267 if (rc != 0)
10268 break;
10269
10270 if (lookup_type && lookup_type != M_DATALKPTYPE) {
10271 /* Inner header VNI mask */
10272 vnix = ((data2 & F_DATAVIDH2) << 23) |
10273 (G_DATAVIDH1(data2) << 16) | G_VIDL(val);
10274 } else
10275 vnix = 0;
10276
10277 if (tcamx & tcamy)
10278 continue;
10279 tcamxy2valmask(tcamx, tcamy, addr, &mask);
10280
10281 mtx_lock(&sc->reg_lock);
10282 if (hw_off_limits(sc))
10283 rc = ENXIO;
10284 else {
10285 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i));
10286 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i));
10287 }
10288 mtx_unlock(&sc->reg_lock);
10289 if (rc != 0)
10290 break;
10291
10292 if (lookup_type && lookup_type != M_DATALKPTYPE) {
10293 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
10294 "%012jx %06x %06x - - %3c"
10295 " I %4x %3c %#x%4u%4d", i, addr[0],
10296 addr[1], addr[2], addr[3], addr[4], addr[5],
10297 (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N',
10298 port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
10299 G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
10300 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
10301 } else {
10302 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x "
10303 "%012jx - - ", i, addr[0], addr[1],
10304 addr[2], addr[3], addr[4], addr[5],
10305 (uintmax_t)mask);
10306
10307 if (vlan_vld)
10308 sbuf_printf(sb, "%4u Y ", ivlan);
10309 else
10310 sbuf_printf(sb, " - N ");
10311
10312 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d",
10313 lookup_type ? 'I' : 'O', port_num,
10314 cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N',
10315 G_PORTMAP(cls_hi), G_T6_PF(cls_lo),
10316 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1);
10317 }
10318
10319
10320 if (cls_lo & F_T6_REPLICATE) {
10321 struct fw_ldst_cmd ldst_cmd;
10322
10323 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
10324 ldst_cmd.op_to_addrspace =
10325 htobe32(V_FW_CMD_OP(FW_LDST_CMD) |
10326 F_FW_CMD_REQUEST | F_FW_CMD_READ |
10327 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS));
10328 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd));
10329 ldst_cmd.u.mps.rplc.fid_idx =
10330 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) |
10331 V_FW_LDST_CMD_IDX(i));
10332
10333 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK,
10334 "t6mps");
10335 if (rc)
10336 break;
10337 if (hw_off_limits(sc))
10338 rc = ENXIO;
10339 else
10340 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd,
10341 sizeof(ldst_cmd), &ldst_cmd);
10342 end_synchronized_op(sc, 0);
10343 if (rc != 0)
10344 break;
10345 else {
10346 sbuf_printf(sb, " %08x %08x %08x %08x"
10347 " %08x %08x %08x %08x",
10348 be32toh(ldst_cmd.u.mps.rplc.rplc255_224),
10349 be32toh(ldst_cmd.u.mps.rplc.rplc223_192),
10350 be32toh(ldst_cmd.u.mps.rplc.rplc191_160),
10351 be32toh(ldst_cmd.u.mps.rplc.rplc159_128),
10352 be32toh(ldst_cmd.u.mps.rplc.rplc127_96),
10353 be32toh(ldst_cmd.u.mps.rplc.rplc95_64),
10354 be32toh(ldst_cmd.u.mps.rplc.rplc63_32),
10355 be32toh(ldst_cmd.u.mps.rplc.rplc31_0));
10356 }
10357 } else
10358 sbuf_printf(sb, "%72s", "");
10359
10360 sbuf_printf(sb, "%4u%3u%3u%3u %#x",
10361 G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo),
10362 G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo),
10363 (cls_lo >> S_T6_MULTILISTEN0) & 0xf);
10364 }
10365
10366 if (rc)
10367 (void) sbuf_finish(sb);
10368 else
10369 rc = sbuf_finish(sb);
10370 sbuf_delete(sb);
10371
10372 return (rc);
10373 }
10374
10375 static int
sysctl_path_mtus(SYSCTL_HANDLER_ARGS)10376 sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
10377 {
10378 struct adapter *sc = arg1;
10379 struct sbuf *sb;
10380 int rc;
10381 uint16_t mtus[NMTUS];
10382
10383 rc = sysctl_wire_old_buffer(req, 0);
10384 if (rc != 0)
10385 return (rc);
10386
10387 mtx_lock(&sc->reg_lock);
10388 if (hw_off_limits(sc))
10389 rc = ENXIO;
10390 else
10391 t4_read_mtu_tbl(sc, mtus, NULL);
10392 mtx_unlock(&sc->reg_lock);
10393 if (rc != 0)
10394 return (rc);
10395
10396 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10397 if (sb == NULL)
10398 return (ENOMEM);
10399
10400 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
10401 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
10402 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
10403 mtus[14], mtus[15]);
10404
10405 rc = sbuf_finish(sb);
10406 sbuf_delete(sb);
10407
10408 return (rc);
10409 }
10410
10411 static int
sysctl_pm_stats(SYSCTL_HANDLER_ARGS)10412 sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
10413 {
10414 struct adapter *sc = arg1;
10415 struct sbuf *sb;
10416 int rc, i;
10417 uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS];
10418 uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS];
10419 static const char *tx_stats[MAX_PM_NSTATS] = {
10420 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:",
10421 "Tx FIFO wait", NULL, "Tx latency"
10422 };
10423 static const char *rx_stats[MAX_PM_NSTATS] = {
10424 "Read:", "Write bypass:", "Write mem:", "Flush:",
10425 "Rx FIFO wait", NULL, "Rx latency"
10426 };
10427
10428 rc = sysctl_wire_old_buffer(req, 0);
10429 if (rc != 0)
10430 return (rc);
10431
10432 mtx_lock(&sc->reg_lock);
10433 if (hw_off_limits(sc))
10434 rc = ENXIO;
10435 else {
10436 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
10437 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
10438 }
10439 mtx_unlock(&sc->reg_lock);
10440 if (rc != 0)
10441 return (rc);
10442
10443 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10444 if (sb == NULL)
10445 return (ENOMEM);
10446
10447 sbuf_printf(sb, " Tx pcmds Tx bytes");
10448 for (i = 0; i < 4; i++) {
10449 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
10450 tx_cyc[i]);
10451 }
10452
10453 sbuf_printf(sb, "\n Rx pcmds Rx bytes");
10454 for (i = 0; i < 4; i++) {
10455 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
10456 rx_cyc[i]);
10457 }
10458
10459 if (chip_id(sc) > CHELSIO_T5) {
10460 sbuf_printf(sb,
10461 "\n Total wait Total occupancy");
10462 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
10463 tx_cyc[i]);
10464 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
10465 rx_cyc[i]);
10466
10467 i += 2;
10468 MPASS(i < nitems(tx_stats));
10469
10470 sbuf_printf(sb,
10471 "\n Reads Total wait");
10472 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i],
10473 tx_cyc[i]);
10474 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i],
10475 rx_cyc[i]);
10476 }
10477
10478 rc = sbuf_finish(sb);
10479 sbuf_delete(sb);
10480
10481 return (rc);
10482 }
10483
10484 static int
sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)10485 sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
10486 {
10487 struct adapter *sc = arg1;
10488 struct sbuf *sb;
10489 int rc;
10490 struct tp_rdma_stats stats;
10491
10492 rc = sysctl_wire_old_buffer(req, 0);
10493 if (rc != 0)
10494 return (rc);
10495
10496 mtx_lock(&sc->reg_lock);
10497 if (hw_off_limits(sc))
10498 rc = ENXIO;
10499 else
10500 t4_tp_get_rdma_stats(sc, &stats, 0);
10501 mtx_unlock(&sc->reg_lock);
10502 if (rc != 0)
10503 return (rc);
10504
10505 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10506 if (sb == NULL)
10507 return (ENOMEM);
10508
10509 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
10510 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
10511
10512 rc = sbuf_finish(sb);
10513 sbuf_delete(sb);
10514
10515 return (rc);
10516 }
10517
10518 static int
sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)10519 sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
10520 {
10521 struct adapter *sc = arg1;
10522 struct sbuf *sb;
10523 int rc;
10524 struct tp_tcp_stats v4, v6;
10525
10526 rc = sysctl_wire_old_buffer(req, 0);
10527 if (rc != 0)
10528 return (rc);
10529
10530 mtx_lock(&sc->reg_lock);
10531 if (hw_off_limits(sc))
10532 rc = ENXIO;
10533 else
10534 t4_tp_get_tcp_stats(sc, &v4, &v6, 0);
10535 mtx_unlock(&sc->reg_lock);
10536 if (rc != 0)
10537 return (rc);
10538
10539 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10540 if (sb == NULL)
10541 return (ENOMEM);
10542
10543 sbuf_printf(sb,
10544 " IP IPv6\n");
10545 sbuf_printf(sb, "OutRsts: %20u %20u\n",
10546 v4.tcp_out_rsts, v6.tcp_out_rsts);
10547 sbuf_printf(sb, "InSegs: %20ju %20ju\n",
10548 v4.tcp_in_segs, v6.tcp_in_segs);
10549 sbuf_printf(sb, "OutSegs: %20ju %20ju\n",
10550 v4.tcp_out_segs, v6.tcp_out_segs);
10551 sbuf_printf(sb, "RetransSegs: %20ju %20ju",
10552 v4.tcp_retrans_segs, v6.tcp_retrans_segs);
10553
10554 rc = sbuf_finish(sb);
10555 sbuf_delete(sb);
10556
10557 return (rc);
10558 }
10559
10560 static int
sysctl_tids(SYSCTL_HANDLER_ARGS)10561 sysctl_tids(SYSCTL_HANDLER_ARGS)
10562 {
10563 struct adapter *sc = arg1;
10564 struct sbuf *sb;
10565 int rc;
10566 uint32_t x, y;
10567 struct tid_info *t = &sc->tids;
10568
10569 rc = sysctl_wire_old_buffer(req, 0);
10570 if (rc != 0)
10571 return (rc);
10572
10573 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10574 if (sb == NULL)
10575 return (ENOMEM);
10576
10577 if (t->natids) {
10578 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
10579 t->atids_in_use);
10580 }
10581
10582 if (t->nhpftids) {
10583 sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
10584 t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
10585 }
10586
10587 if (t->ntids) {
10588 bool hashen = false;
10589
10590 mtx_lock(&sc->reg_lock);
10591 if (hw_off_limits(sc))
10592 rc = ENXIO;
10593 else if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
10594 hashen = true;
10595 if (chip_id(sc) <= CHELSIO_T5) {
10596 x = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
10597 y = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
10598 } else {
10599 x = t4_read_reg(sc, A_LE_DB_SRVR_START_INDEX);
10600 y = t4_read_reg(sc, A_T6_LE_DB_HASH_TID_BASE);
10601 }
10602 }
10603 mtx_unlock(&sc->reg_lock);
10604 if (rc != 0)
10605 goto done;
10606
10607 sbuf_printf(sb, "TID range: ");
10608 if (hashen) {
10609 if (x)
10610 sbuf_printf(sb, "%u-%u, ", t->tid_base, x - 1);
10611 sbuf_printf(sb, "%u-%u", y, t->ntids - 1);
10612 } else {
10613 sbuf_printf(sb, "%u-%u", t->tid_base, t->tid_base +
10614 t->ntids - 1);
10615 }
10616 sbuf_printf(sb, ", in use: %u\n",
10617 atomic_load_acq_int(&t->tids_in_use));
10618 }
10619
10620 if (t->nstids) {
10621 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
10622 t->stid_base + t->nstids - 1, t->stids_in_use);
10623 }
10624
10625 if (t->nftids) {
10626 sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
10627 t->ftid_end, t->ftids_in_use);
10628 }
10629
10630 if (t->netids) {
10631 sbuf_printf(sb, "ETID range: %u-%u, in use: %u\n", t->etid_base,
10632 t->etid_base + t->netids - 1, t->etids_in_use);
10633 }
10634
10635 mtx_lock(&sc->reg_lock);
10636 if (hw_off_limits(sc))
10637 rc = ENXIO;
10638 else {
10639 x = t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4);
10640 y = t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6);
10641 }
10642 mtx_unlock(&sc->reg_lock);
10643 if (rc != 0)
10644 goto done;
10645 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", x, y);
10646 done:
10647 if (rc == 0)
10648 rc = sbuf_finish(sb);
10649 else
10650 (void)sbuf_finish(sb);
10651 sbuf_delete(sb);
10652
10653 return (rc);
10654 }
10655
10656 static int
sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)10657 sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
10658 {
10659 struct adapter *sc = arg1;
10660 struct sbuf *sb;
10661 int rc;
10662 struct tp_err_stats stats;
10663
10664 rc = sysctl_wire_old_buffer(req, 0);
10665 if (rc != 0)
10666 return (rc);
10667
10668 mtx_lock(&sc->reg_lock);
10669 if (hw_off_limits(sc))
10670 rc = ENXIO;
10671 else
10672 t4_tp_get_err_stats(sc, &stats, 0);
10673 mtx_unlock(&sc->reg_lock);
10674 if (rc != 0)
10675 return (rc);
10676
10677 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10678 if (sb == NULL)
10679 return (ENOMEM);
10680
10681 if (sc->chip_params->nchan > 2) {
10682 sbuf_printf(sb, " channel 0 channel 1"
10683 " channel 2 channel 3\n");
10684 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n",
10685 stats.mac_in_errs[0], stats.mac_in_errs[1],
10686 stats.mac_in_errs[2], stats.mac_in_errs[3]);
10687 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n",
10688 stats.hdr_in_errs[0], stats.hdr_in_errs[1],
10689 stats.hdr_in_errs[2], stats.hdr_in_errs[3]);
10690 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n",
10691 stats.tcp_in_errs[0], stats.tcp_in_errs[1],
10692 stats.tcp_in_errs[2], stats.tcp_in_errs[3]);
10693 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n",
10694 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1],
10695 stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]);
10696 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n",
10697 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1],
10698 stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]);
10699 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n",
10700 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1],
10701 stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]);
10702 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n",
10703 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1],
10704 stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]);
10705 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n",
10706 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1],
10707 stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]);
10708 } else {
10709 sbuf_printf(sb, " channel 0 channel 1\n");
10710 sbuf_printf(sb, "macInErrs: %10u %10u\n",
10711 stats.mac_in_errs[0], stats.mac_in_errs[1]);
10712 sbuf_printf(sb, "hdrInErrs: %10u %10u\n",
10713 stats.hdr_in_errs[0], stats.hdr_in_errs[1]);
10714 sbuf_printf(sb, "tcpInErrs: %10u %10u\n",
10715 stats.tcp_in_errs[0], stats.tcp_in_errs[1]);
10716 sbuf_printf(sb, "tcp6InErrs: %10u %10u\n",
10717 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]);
10718 sbuf_printf(sb, "tnlCongDrops: %10u %10u\n",
10719 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]);
10720 sbuf_printf(sb, "tnlTxDrops: %10u %10u\n",
10721 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]);
10722 sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n",
10723 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]);
10724 sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n",
10725 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]);
10726 }
10727
10728 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u",
10729 stats.ofld_no_neigh, stats.ofld_cong_defer);
10730
10731 rc = sbuf_finish(sb);
10732 sbuf_delete(sb);
10733
10734 return (rc);
10735 }
10736
10737 static int
sysctl_tnl_stats(SYSCTL_HANDLER_ARGS)10738 sysctl_tnl_stats(SYSCTL_HANDLER_ARGS)
10739 {
10740 struct adapter *sc = arg1;
10741 struct sbuf *sb;
10742 int rc;
10743 struct tp_tnl_stats stats;
10744
10745 rc = sysctl_wire_old_buffer(req, 0);
10746 if (rc != 0)
10747 return(rc);
10748
10749 mtx_lock(&sc->reg_lock);
10750 if (hw_off_limits(sc))
10751 rc = ENXIO;
10752 else
10753 t4_tp_get_tnl_stats(sc, &stats, 1);
10754 mtx_unlock(&sc->reg_lock);
10755 if (rc != 0)
10756 return (rc);
10757
10758 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
10759 if (sb == NULL)
10760 return (ENOMEM);
10761
10762 if (sc->chip_params->nchan > 2) {
10763 sbuf_printf(sb, " channel 0 channel 1"
10764 " channel 2 channel 3\n");
10765 sbuf_printf(sb, "OutPkts: %10u %10u %10u %10u\n",
10766 stats.out_pkt[0], stats.out_pkt[1],
10767 stats.out_pkt[2], stats.out_pkt[3]);
10768 sbuf_printf(sb, "InPkts: %10u %10u %10u %10u",
10769 stats.in_pkt[0], stats.in_pkt[1],
10770 stats.in_pkt[2], stats.in_pkt[3]);
10771 } else {
10772 sbuf_printf(sb, " channel 0 channel 1\n");
10773 sbuf_printf(sb, "OutPkts: %10u %10u\n",
10774 stats.out_pkt[0], stats.out_pkt[1]);
10775 sbuf_printf(sb, "InPkts: %10u %10u",
10776 stats.in_pkt[0], stats.in_pkt[1]);
10777 }
10778
10779 rc = sbuf_finish(sb);
10780 sbuf_delete(sb);
10781
10782 return (rc);
10783 }
10784
10785 static int
sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS)10786 sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS)
10787 {
10788 struct adapter *sc = arg1;
10789 struct tp_params *tpp = &sc->params.tp;
10790 u_int mask;
10791 int rc;
10792
10793 mask = tpp->la_mask >> 16;
10794 rc = sysctl_handle_int(oidp, &mask, 0, req);
10795 if (rc != 0 || req->newptr == NULL)
10796 return (rc);
10797 if (mask > 0xffff)
10798 return (EINVAL);
10799 mtx_lock(&sc->reg_lock);
10800 if (hw_off_limits(sc))
10801 rc = ENXIO;
10802 else {
10803 tpp->la_mask = mask << 16;
10804 t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U,
10805 tpp->la_mask);
10806 }
10807 mtx_unlock(&sc->reg_lock);
10808
10809 return (rc);
10810 }
10811
10812 struct field_desc {
10813 const char *name;
10814 u_int start;
10815 u_int width;
10816 };
10817
10818 static void
field_desc_show(struct sbuf * sb,uint64_t v,const struct field_desc * f)10819 field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f)
10820 {
10821 char buf[32];
10822 int line_size = 0;
10823
10824 while (f->name) {
10825 uint64_t mask = (1ULL << f->width) - 1;
10826 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name,
10827 ((uintmax_t)v >> f->start) & mask);
10828
10829 if (line_size + len >= 79) {
10830 line_size = 8;
10831 sbuf_printf(sb, "\n ");
10832 }
10833 sbuf_printf(sb, "%s ", buf);
10834 line_size += len + 1;
10835 f++;
10836 }
10837 sbuf_printf(sb, "\n");
10838 }
10839
10840 static const struct field_desc tp_la0[] = {
10841 { "RcfOpCodeOut", 60, 4 },
10842 { "State", 56, 4 },
10843 { "WcfState", 52, 4 },
10844 { "RcfOpcSrcOut", 50, 2 },
10845 { "CRxError", 49, 1 },
10846 { "ERxError", 48, 1 },
10847 { "SanityFailed", 47, 1 },
10848 { "SpuriousMsg", 46, 1 },
10849 { "FlushInputMsg", 45, 1 },
10850 { "FlushInputCpl", 44, 1 },
10851 { "RssUpBit", 43, 1 },
10852 { "RssFilterHit", 42, 1 },
10853 { "Tid", 32, 10 },
10854 { "InitTcb", 31, 1 },
10855 { "LineNumber", 24, 7 },
10856 { "Emsg", 23, 1 },
10857 { "EdataOut", 22, 1 },
10858 { "Cmsg", 21, 1 },
10859 { "CdataOut", 20, 1 },
10860 { "EreadPdu", 19, 1 },
10861 { "CreadPdu", 18, 1 },
10862 { "TunnelPkt", 17, 1 },
10863 { "RcfPeerFin", 16, 1 },
10864 { "RcfReasonOut", 12, 4 },
10865 { "TxCchannel", 10, 2 },
10866 { "RcfTxChannel", 8, 2 },
10867 { "RxEchannel", 6, 2 },
10868 { "RcfRxChannel", 5, 1 },
10869 { "RcfDataOutSrdy", 4, 1 },
10870 { "RxDvld", 3, 1 },
10871 { "RxOoDvld", 2, 1 },
10872 { "RxCongestion", 1, 1 },
10873 { "TxCongestion", 0, 1 },
10874 { NULL }
10875 };
10876
10877 static const struct field_desc tp_la1[] = {
10878 { "CplCmdIn", 56, 8 },
10879 { "CplCmdOut", 48, 8 },
10880 { "ESynOut", 47, 1 },
10881 { "EAckOut", 46, 1 },
10882 { "EFinOut", 45, 1 },
10883 { "ERstOut", 44, 1 },
10884 { "SynIn", 43, 1 },
10885 { "AckIn", 42, 1 },
10886 { "FinIn", 41, 1 },
10887 { "RstIn", 40, 1 },
10888 { "DataIn", 39, 1 },
10889 { "DataInVld", 38, 1 },
10890 { "PadIn", 37, 1 },
10891 { "RxBufEmpty", 36, 1 },
10892 { "RxDdp", 35, 1 },
10893 { "RxFbCongestion", 34, 1 },
10894 { "TxFbCongestion", 33, 1 },
10895 { "TxPktSumSrdy", 32, 1 },
10896 { "RcfUlpType", 28, 4 },
10897 { "Eread", 27, 1 },
10898 { "Ebypass", 26, 1 },
10899 { "Esave", 25, 1 },
10900 { "Static0", 24, 1 },
10901 { "Cread", 23, 1 },
10902 { "Cbypass", 22, 1 },
10903 { "Csave", 21, 1 },
10904 { "CPktOut", 20, 1 },
10905 { "RxPagePoolFull", 18, 2 },
10906 { "RxLpbkPkt", 17, 1 },
10907 { "TxLpbkPkt", 16, 1 },
10908 { "RxVfValid", 15, 1 },
10909 { "SynLearned", 14, 1 },
10910 { "SetDelEntry", 13, 1 },
10911 { "SetInvEntry", 12, 1 },
10912 { "CpcmdDvld", 11, 1 },
10913 { "CpcmdSave", 10, 1 },
10914 { "RxPstructsFull", 8, 2 },
10915 { "EpcmdDvld", 7, 1 },
10916 { "EpcmdFlush", 6, 1 },
10917 { "EpcmdTrimPrefix", 5, 1 },
10918 { "EpcmdTrimPostfix", 4, 1 },
10919 { "ERssIp4Pkt", 3, 1 },
10920 { "ERssIp6Pkt", 2, 1 },
10921 { "ERssTcpUdpPkt", 1, 1 },
10922 { "ERssFceFipPkt", 0, 1 },
10923 { NULL }
10924 };
10925
10926 static const struct field_desc tp_la2[] = {
10927 { "CplCmdIn", 56, 8 },
10928 { "MpsVfVld", 55, 1 },
10929 { "MpsPf", 52, 3 },
10930 { "MpsVf", 44, 8 },
10931 { "SynIn", 43, 1 },
10932 { "AckIn", 42, 1 },
10933 { "FinIn", 41, 1 },
10934 { "RstIn", 40, 1 },
10935 { "DataIn", 39, 1 },
10936 { "DataInVld", 38, 1 },
10937 { "PadIn", 37, 1 },
10938 { "RxBufEmpty", 36, 1 },
10939 { "RxDdp", 35, 1 },
10940 { "RxFbCongestion", 34, 1 },
10941 { "TxFbCongestion", 33, 1 },
10942 { "TxPktSumSrdy", 32, 1 },
10943 { "RcfUlpType", 28, 4 },
10944 { "Eread", 27, 1 },
10945 { "Ebypass", 26, 1 },
10946 { "Esave", 25, 1 },
10947 { "Static0", 24, 1 },
10948 { "Cread", 23, 1 },
10949 { "Cbypass", 22, 1 },
10950 { "Csave", 21, 1 },
10951 { "CPktOut", 20, 1 },
10952 { "RxPagePoolFull", 18, 2 },
10953 { "RxLpbkPkt", 17, 1 },
10954 { "TxLpbkPkt", 16, 1 },
10955 { "RxVfValid", 15, 1 },
10956 { "SynLearned", 14, 1 },
10957 { "SetDelEntry", 13, 1 },
10958 { "SetInvEntry", 12, 1 },
10959 { "CpcmdDvld", 11, 1 },
10960 { "CpcmdSave", 10, 1 },
10961 { "RxPstructsFull", 8, 2 },
10962 { "EpcmdDvld", 7, 1 },
10963 { "EpcmdFlush", 6, 1 },
10964 { "EpcmdTrimPrefix", 5, 1 },
10965 { "EpcmdTrimPostfix", 4, 1 },
10966 { "ERssIp4Pkt", 3, 1 },
10967 { "ERssIp6Pkt", 2, 1 },
10968 { "ERssTcpUdpPkt", 1, 1 },
10969 { "ERssFceFipPkt", 0, 1 },
10970 { NULL }
10971 };
10972
10973 static void
tp_la_show(struct sbuf * sb,uint64_t * p,int idx)10974 tp_la_show(struct sbuf *sb, uint64_t *p, int idx)
10975 {
10976
10977 field_desc_show(sb, *p, tp_la0);
10978 }
10979
10980 static void
tp_la_show2(struct sbuf * sb,uint64_t * p,int idx)10981 tp_la_show2(struct sbuf *sb, uint64_t *p, int idx)
10982 {
10983
10984 if (idx)
10985 sbuf_printf(sb, "\n");
10986 field_desc_show(sb, p[0], tp_la0);
10987 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
10988 field_desc_show(sb, p[1], tp_la0);
10989 }
10990
10991 static void
tp_la_show3(struct sbuf * sb,uint64_t * p,int idx)10992 tp_la_show3(struct sbuf *sb, uint64_t *p, int idx)
10993 {
10994
10995 if (idx)
10996 sbuf_printf(sb, "\n");
10997 field_desc_show(sb, p[0], tp_la0);
10998 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL)
10999 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1);
11000 }
11001
11002 static int
sysctl_tp_la(SYSCTL_HANDLER_ARGS)11003 sysctl_tp_la(SYSCTL_HANDLER_ARGS)
11004 {
11005 struct adapter *sc = arg1;
11006 struct sbuf *sb;
11007 uint64_t *buf, *p;
11008 int rc;
11009 u_int i, inc;
11010 void (*show_func)(struct sbuf *, uint64_t *, int);
11011
11012 rc = sysctl_wire_old_buffer(req, 0);
11013 if (rc != 0)
11014 return (rc);
11015
11016 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
11017 if (sb == NULL)
11018 return (ENOMEM);
11019
11020 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK);
11021
11022 mtx_lock(&sc->reg_lock);
11023 if (hw_off_limits(sc))
11024 rc = ENXIO;
11025 else {
11026 t4_tp_read_la(sc, buf, NULL);
11027 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) {
11028 case 2:
11029 inc = 2;
11030 show_func = tp_la_show2;
11031 break;
11032 case 3:
11033 inc = 2;
11034 show_func = tp_la_show3;
11035 break;
11036 default:
11037 inc = 1;
11038 show_func = tp_la_show;
11039 }
11040 }
11041 mtx_unlock(&sc->reg_lock);
11042 if (rc != 0)
11043 goto done;
11044
11045 p = buf;
11046 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc)
11047 (*show_func)(sb, p, i);
11048 rc = sbuf_finish(sb);
11049 done:
11050 sbuf_delete(sb);
11051 free(buf, M_CXGBE);
11052 return (rc);
11053 }
11054
11055 static int
sysctl_tx_rate(SYSCTL_HANDLER_ARGS)11056 sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
11057 {
11058 struct adapter *sc = arg1;
11059 struct sbuf *sb;
11060 int rc;
11061 u64 nrate[MAX_NCHAN], orate[MAX_NCHAN];
11062
11063 rc = sysctl_wire_old_buffer(req, 0);
11064 if (rc != 0)
11065 return (rc);
11066
11067 mtx_lock(&sc->reg_lock);
11068 if (hw_off_limits(sc))
11069 rc = ENXIO;
11070 else
11071 t4_get_chan_txrate(sc, nrate, orate);
11072 mtx_unlock(&sc->reg_lock);
11073 if (rc != 0)
11074 return (rc);
11075
11076 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
11077 if (sb == NULL)
11078 return (ENOMEM);
11079
11080 if (sc->chip_params->nchan > 2) {
11081 sbuf_printf(sb, " channel 0 channel 1"
11082 " channel 2 channel 3\n");
11083 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n",
11084 nrate[0], nrate[1], nrate[2], nrate[3]);
11085 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju",
11086 orate[0], orate[1], orate[2], orate[3]);
11087 } else {
11088 sbuf_printf(sb, " channel 0 channel 1\n");
11089 sbuf_printf(sb, "NIC B/s: %10ju %10ju\n",
11090 nrate[0], nrate[1]);
11091 sbuf_printf(sb, "Offload B/s: %10ju %10ju",
11092 orate[0], orate[1]);
11093 }
11094
11095 rc = sbuf_finish(sb);
11096 sbuf_delete(sb);
11097
11098 return (rc);
11099 }
11100
11101 static int
sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)11102 sysctl_ulprx_la(SYSCTL_HANDLER_ARGS)
11103 {
11104 struct adapter *sc = arg1;
11105 struct sbuf *sb;
11106 uint32_t *buf, *p;
11107 int rc, i;
11108
11109 rc = sysctl_wire_old_buffer(req, 0);
11110 if (rc != 0)
11111 return (rc);
11112
11113 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
11114 if (sb == NULL)
11115 return (ENOMEM);
11116
11117 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE,
11118 M_ZERO | M_WAITOK);
11119
11120 mtx_lock(&sc->reg_lock);
11121 if (hw_off_limits(sc))
11122 rc = ENXIO;
11123 else
11124 t4_ulprx_read_la(sc, buf);
11125 mtx_unlock(&sc->reg_lock);
11126 if (rc != 0)
11127 goto done;
11128
11129 p = buf;
11130 sbuf_printf(sb, " Pcmd Type Message"
11131 " Data");
11132 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) {
11133 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x",
11134 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]);
11135 }
11136 rc = sbuf_finish(sb);
11137 done:
11138 sbuf_delete(sb);
11139 free(buf, M_CXGBE);
11140 return (rc);
11141 }
11142
11143 static int
sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)11144 sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
11145 {
11146 struct adapter *sc = arg1;
11147 struct sbuf *sb;
11148 int rc;
11149 uint32_t cfg, s1, s2;
11150
11151 MPASS(chip_id(sc) >= CHELSIO_T5);
11152
11153 rc = sysctl_wire_old_buffer(req, 0);
11154 if (rc != 0)
11155 return (rc);
11156
11157 mtx_lock(&sc->reg_lock);
11158 if (hw_off_limits(sc))
11159 rc = ENXIO;
11160 else {
11161 cfg = t4_read_reg(sc, A_SGE_STAT_CFG);
11162 s1 = t4_read_reg(sc, A_SGE_STAT_TOTAL);
11163 s2 = t4_read_reg(sc, A_SGE_STAT_MATCH);
11164 }
11165 mtx_unlock(&sc->reg_lock);
11166 if (rc != 0)
11167 return (rc);
11168
11169 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
11170 if (sb == NULL)
11171 return (ENOMEM);
11172
11173 if (G_STATSOURCE_T5(cfg) == 7) {
11174 int mode;
11175
11176 mode = is_t5(sc) ? G_STATMODE(cfg) : G_T6_STATMODE(cfg);
11177 if (mode == 0)
11178 sbuf_printf(sb, "total %d, incomplete %d", s1, s2);
11179 else if (mode == 1)
11180 sbuf_printf(sb, "total %d, data overflow %d", s1, s2);
11181 else
11182 sbuf_printf(sb, "unknown mode %d", mode);
11183 }
11184 rc = sbuf_finish(sb);
11185 sbuf_delete(sb);
11186
11187 return (rc);
11188 }
11189
11190 static int
sysctl_cpus(SYSCTL_HANDLER_ARGS)11191 sysctl_cpus(SYSCTL_HANDLER_ARGS)
11192 {
11193 struct adapter *sc = arg1;
11194 enum cpu_sets op = arg2;
11195 cpuset_t cpuset;
11196 struct sbuf *sb;
11197 int i, rc;
11198
11199 MPASS(op == LOCAL_CPUS || op == INTR_CPUS);
11200
11201 CPU_ZERO(&cpuset);
11202 rc = bus_get_cpus(sc->dev, op, sizeof(cpuset), &cpuset);
11203 if (rc != 0)
11204 return (rc);
11205
11206 rc = sysctl_wire_old_buffer(req, 0);
11207 if (rc != 0)
11208 return (rc);
11209
11210 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
11211 if (sb == NULL)
11212 return (ENOMEM);
11213
11214 CPU_FOREACH(i)
11215 sbuf_printf(sb, "%d ", i);
11216 rc = sbuf_finish(sb);
11217 sbuf_delete(sb);
11218
11219 return (rc);
11220 }
11221
11222 static int
sysctl_reset(SYSCTL_HANDLER_ARGS)11223 sysctl_reset(SYSCTL_HANDLER_ARGS)
11224 {
11225 struct adapter *sc = arg1;
11226 u_int val;
11227 int rc;
11228
11229 val = atomic_load_int(&sc->num_resets);
11230 rc = sysctl_handle_int(oidp, &val, 0, req);
11231 if (rc != 0 || req->newptr == NULL)
11232 return (rc);
11233
11234 if (val == 0) {
11235 /* Zero out the counter that tracks reset. */
11236 atomic_store_int(&sc->num_resets, 0);
11237 return (0);
11238 }
11239
11240 if (val != 1)
11241 return (EINVAL); /* 0 or 1 are the only legal values */
11242
11243 if (hw_off_limits(sc)) /* harmless race */
11244 return (EALREADY);
11245
11246 taskqueue_enqueue(reset_tq, &sc->reset_task);
11247 return (0);
11248 }
11249
11250 #ifdef TCP_OFFLOAD
11251 static int
sysctl_tls(SYSCTL_HANDLER_ARGS)11252 sysctl_tls(SYSCTL_HANDLER_ARGS)
11253 {
11254 struct adapter *sc = arg1;
11255 int i, j, v, rc;
11256 struct vi_info *vi;
11257
11258 v = sc->tt.tls;
11259 rc = sysctl_handle_int(oidp, &v, 0, req);
11260 if (rc != 0 || req->newptr == NULL)
11261 return (rc);
11262
11263 if (v != 0 && !(sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS))
11264 return (ENOTSUP);
11265
11266 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4stls");
11267 if (rc)
11268 return (rc);
11269 if (hw_off_limits(sc))
11270 rc = ENXIO;
11271 else {
11272 sc->tt.tls = !!v;
11273 for_each_port(sc, i) {
11274 for_each_vi(sc->port[i], j, vi) {
11275 if (vi->flags & VI_INIT_DONE)
11276 t4_update_fl_bufsize(vi->ifp);
11277 }
11278 }
11279 }
11280 end_synchronized_op(sc, 0);
11281
11282 return (rc);
11283
11284 }
11285
11286 static void
unit_conv(char * buf,size_t len,u_int val,u_int factor)11287 unit_conv(char *buf, size_t len, u_int val, u_int factor)
11288 {
11289 u_int rem = val % factor;
11290
11291 if (rem == 0)
11292 snprintf(buf, len, "%u", val / factor);
11293 else {
11294 while (rem % 10 == 0)
11295 rem /= 10;
11296 snprintf(buf, len, "%u.%u", val / factor, rem);
11297 }
11298 }
11299
11300 static int
sysctl_tp_tick(SYSCTL_HANDLER_ARGS)11301 sysctl_tp_tick(SYSCTL_HANDLER_ARGS)
11302 {
11303 struct adapter *sc = arg1;
11304 char buf[16];
11305 u_int res, re;
11306 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
11307
11308 mtx_lock(&sc->reg_lock);
11309 if (hw_off_limits(sc))
11310 res = (u_int)-1;
11311 else
11312 res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
11313 mtx_unlock(&sc->reg_lock);
11314 if (res == (u_int)-1)
11315 return (ENXIO);
11316
11317 switch (arg2) {
11318 case 0:
11319 /* timer_tick */
11320 re = G_TIMERRESOLUTION(res);
11321 break;
11322 case 1:
11323 /* TCP timestamp tick */
11324 re = G_TIMESTAMPRESOLUTION(res);
11325 break;
11326 case 2:
11327 /* DACK tick */
11328 re = G_DELAYEDACKRESOLUTION(res);
11329 break;
11330 default:
11331 return (EDOOFUS);
11332 }
11333
11334 unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000);
11335
11336 return (sysctl_handle_string(oidp, buf, sizeof(buf), req));
11337 }
11338
11339 static int
sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS)11340 sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS)
11341 {
11342 struct adapter *sc = arg1;
11343 int rc;
11344 u_int dack_tmr, dack_re, v;
11345 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
11346
11347 mtx_lock(&sc->reg_lock);
11348 if (hw_off_limits(sc))
11349 rc = ENXIO;
11350 else {
11351 rc = 0;
11352 dack_re = G_DELAYEDACKRESOLUTION(t4_read_reg(sc,
11353 A_TP_TIMER_RESOLUTION));
11354 dack_tmr = t4_read_reg(sc, A_TP_DACK_TIMER);
11355 }
11356 mtx_unlock(&sc->reg_lock);
11357 if (rc != 0)
11358 return (rc);
11359
11360 v = ((cclk_ps << dack_re) / 1000000) * dack_tmr;
11361
11362 return (sysctl_handle_int(oidp, &v, 0, req));
11363 }
11364
11365 static int
sysctl_tp_timer(SYSCTL_HANDLER_ARGS)11366 sysctl_tp_timer(SYSCTL_HANDLER_ARGS)
11367 {
11368 struct adapter *sc = arg1;
11369 int rc, reg = arg2;
11370 u_int tre;
11371 u_long tp_tick_us, v;
11372 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk;
11373
11374 MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX ||
11375 reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX ||
11376 reg == A_TP_KEEP_IDLE || reg == A_TP_KEEP_INTVL ||
11377 reg == A_TP_INIT_SRTT || reg == A_TP_FINWAIT2_TIMER);
11378
11379 mtx_lock(&sc->reg_lock);
11380 if (hw_off_limits(sc))
11381 rc = ENXIO;
11382 else {
11383 rc = 0;
11384 tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION));
11385 tp_tick_us = (cclk_ps << tre) / 1000000;
11386 if (reg == A_TP_INIT_SRTT)
11387 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg));
11388 else
11389 v = tp_tick_us * t4_read_reg(sc, reg);
11390 }
11391 mtx_unlock(&sc->reg_lock);
11392 if (rc != 0)
11393 return (rc);
11394 else
11395 return (sysctl_handle_long(oidp, &v, 0, req));
11396 }
11397
11398 /*
11399 * All fields in TP_SHIFT_CNT are 4b and the starting location of the field is
11400 * passed to this function.
11401 */
11402 static int
sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS)11403 sysctl_tp_shift_cnt(SYSCTL_HANDLER_ARGS)
11404 {
11405 struct adapter *sc = arg1;
11406 int rc, idx = arg2;
11407 u_int v;
11408
11409 MPASS(idx >= 0 && idx <= 24);
11410
11411 mtx_lock(&sc->reg_lock);
11412 if (hw_off_limits(sc))
11413 rc = ENXIO;
11414 else {
11415 rc = 0;
11416 v = (t4_read_reg(sc, A_TP_SHIFT_CNT) >> idx) & 0xf;
11417 }
11418 mtx_unlock(&sc->reg_lock);
11419 if (rc != 0)
11420 return (rc);
11421 else
11422 return (sysctl_handle_int(oidp, &v, 0, req));
11423 }
11424
11425 static int
sysctl_tp_backoff(SYSCTL_HANDLER_ARGS)11426 sysctl_tp_backoff(SYSCTL_HANDLER_ARGS)
11427 {
11428 struct adapter *sc = arg1;
11429 int rc, idx = arg2;
11430 u_int shift, v, r;
11431
11432 MPASS(idx >= 0 && idx < 16);
11433
11434 r = A_TP_TCP_BACKOFF_REG0 + (idx & ~3);
11435 shift = (idx & 3) << 3;
11436 mtx_lock(&sc->reg_lock);
11437 if (hw_off_limits(sc))
11438 rc = ENXIO;
11439 else {
11440 rc = 0;
11441 v = (t4_read_reg(sc, r) >> shift) & M_TIMERBACKOFFINDEX0;
11442 }
11443 mtx_unlock(&sc->reg_lock);
11444 if (rc != 0)
11445 return (rc);
11446 else
11447 return (sysctl_handle_int(oidp, &v, 0, req));
11448 }
11449
11450 static int
sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS)11451 sysctl_holdoff_tmr_idx_ofld(SYSCTL_HANDLER_ARGS)
11452 {
11453 struct vi_info *vi = arg1;
11454 struct adapter *sc = vi->adapter;
11455 int idx, rc, i;
11456 struct sge_ofld_rxq *ofld_rxq;
11457 uint8_t v;
11458
11459 idx = vi->ofld_tmr_idx;
11460
11461 rc = sysctl_handle_int(oidp, &idx, 0, req);
11462 if (rc != 0 || req->newptr == NULL)
11463 return (rc);
11464
11465 if (idx < 0 || idx >= SGE_NTIMERS)
11466 return (EINVAL);
11467
11468 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
11469 "t4otmr");
11470 if (rc)
11471 return (rc);
11472
11473 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->ofld_pktc_idx != -1);
11474 for_each_ofld_rxq(vi, i, ofld_rxq) {
11475 #ifdef atomic_store_rel_8
11476 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v);
11477 #else
11478 ofld_rxq->iq.intr_params = v;
11479 #endif
11480 }
11481 vi->ofld_tmr_idx = idx;
11482
11483 end_synchronized_op(sc, LOCK_HELD);
11484 return (0);
11485 }
11486
11487 static int
sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS)11488 sysctl_holdoff_pktc_idx_ofld(SYSCTL_HANDLER_ARGS)
11489 {
11490 struct vi_info *vi = arg1;
11491 struct adapter *sc = vi->adapter;
11492 int idx, rc;
11493
11494 idx = vi->ofld_pktc_idx;
11495
11496 rc = sysctl_handle_int(oidp, &idx, 0, req);
11497 if (rc != 0 || req->newptr == NULL)
11498 return (rc);
11499
11500 if (idx < -1 || idx >= SGE_NCOUNTERS)
11501 return (EINVAL);
11502
11503 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK,
11504 "t4opktc");
11505 if (rc)
11506 return (rc);
11507
11508 if (vi->flags & VI_INIT_DONE)
11509 rc = EBUSY; /* cannot be changed once the queues are created */
11510 else
11511 vi->ofld_pktc_idx = idx;
11512
11513 end_synchronized_op(sc, LOCK_HELD);
11514 return (rc);
11515 }
11516 #endif
11517
11518 static int
get_sge_context(struct adapter * sc,struct t4_sge_context * cntxt)11519 get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
11520 {
11521 int rc;
11522
11523 if (cntxt->cid > M_CTXTQID)
11524 return (EINVAL);
11525
11526 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
11527 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
11528 return (EINVAL);
11529
11530 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt");
11531 if (rc)
11532 return (rc);
11533
11534 if (hw_off_limits(sc)) {
11535 rc = ENXIO;
11536 goto done;
11537 }
11538
11539 if (sc->flags & FW_OK) {
11540 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
11541 &cntxt->data[0]);
11542 if (rc == 0)
11543 goto done;
11544 }
11545
11546 /*
11547 * Read via firmware failed or wasn't even attempted. Read directly via
11548 * the backdoor.
11549 */
11550 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]);
11551 done:
11552 end_synchronized_op(sc, 0);
11553 return (rc);
11554 }
11555
11556 static int
load_fw(struct adapter * sc,struct t4_data * fw)11557 load_fw(struct adapter *sc, struct t4_data *fw)
11558 {
11559 int rc;
11560 uint8_t *fw_data;
11561
11562 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
11563 if (rc)
11564 return (rc);
11565
11566 if (hw_off_limits(sc)) {
11567 rc = ENXIO;
11568 goto done;
11569 }
11570
11571 /*
11572 * The firmware, with the sole exception of the memory parity error
11573 * handler, runs from memory and not flash. It is almost always safe to
11574 * install a new firmware on a running system. Just set bit 1 in
11575 * hw.cxgbe.dflags or dev.<nexus>.<n>.dflags first.
11576 */
11577 if (sc->flags & FULL_INIT_DONE &&
11578 (sc->debug_flags & DF_LOAD_FW_ANYTIME) == 0) {
11579 rc = EBUSY;
11580 goto done;
11581 }
11582
11583 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
11584
11585 rc = copyin(fw->data, fw_data, fw->len);
11586 if (rc == 0)
11587 rc = -t4_load_fw(sc, fw_data, fw->len);
11588
11589 free(fw_data, M_CXGBE);
11590 done:
11591 end_synchronized_op(sc, 0);
11592 return (rc);
11593 }
11594
11595 static int
load_cfg(struct adapter * sc,struct t4_data * cfg)11596 load_cfg(struct adapter *sc, struct t4_data *cfg)
11597 {
11598 int rc;
11599 uint8_t *cfg_data = NULL;
11600
11601 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf");
11602 if (rc)
11603 return (rc);
11604
11605 if (hw_off_limits(sc)) {
11606 rc = ENXIO;
11607 goto done;
11608 }
11609
11610 if (cfg->len == 0) {
11611 /* clear */
11612 rc = -t4_load_cfg(sc, NULL, 0);
11613 goto done;
11614 }
11615
11616 cfg_data = malloc(cfg->len, M_CXGBE, M_WAITOK);
11617
11618 rc = copyin(cfg->data, cfg_data, cfg->len);
11619 if (rc == 0)
11620 rc = -t4_load_cfg(sc, cfg_data, cfg->len);
11621
11622 free(cfg_data, M_CXGBE);
11623 done:
11624 end_synchronized_op(sc, 0);
11625 return (rc);
11626 }
11627
11628 static int
load_boot(struct adapter * sc,struct t4_bootrom * br)11629 load_boot(struct adapter *sc, struct t4_bootrom *br)
11630 {
11631 int rc;
11632 uint8_t *br_data = NULL;
11633 u_int offset;
11634
11635 if (br->len > 1024 * 1024)
11636 return (EFBIG);
11637
11638 if (br->pf_offset == 0) {
11639 /* pfidx */
11640 if (br->pfidx_addr > 7)
11641 return (EINVAL);
11642 offset = G_OFFSET(t4_read_reg(sc, PF_REG(br->pfidx_addr,
11643 A_PCIE_PF_EXPROM_OFST)));
11644 } else if (br->pf_offset == 1) {
11645 /* offset */
11646 offset = G_OFFSET(br->pfidx_addr);
11647 } else {
11648 return (EINVAL);
11649 }
11650
11651 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldbr");
11652 if (rc)
11653 return (rc);
11654
11655 if (hw_off_limits(sc)) {
11656 rc = ENXIO;
11657 goto done;
11658 }
11659
11660 if (br->len == 0) {
11661 /* clear */
11662 rc = -t4_load_boot(sc, NULL, offset, 0);
11663 goto done;
11664 }
11665
11666 br_data = malloc(br->len, M_CXGBE, M_WAITOK);
11667
11668 rc = copyin(br->data, br_data, br->len);
11669 if (rc == 0)
11670 rc = -t4_load_boot(sc, br_data, offset, br->len);
11671
11672 free(br_data, M_CXGBE);
11673 done:
11674 end_synchronized_op(sc, 0);
11675 return (rc);
11676 }
11677
11678 static int
load_bootcfg(struct adapter * sc,struct t4_data * bc)11679 load_bootcfg(struct adapter *sc, struct t4_data *bc)
11680 {
11681 int rc;
11682 uint8_t *bc_data = NULL;
11683
11684 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldcf");
11685 if (rc)
11686 return (rc);
11687
11688 if (hw_off_limits(sc)) {
11689 rc = ENXIO;
11690 goto done;
11691 }
11692
11693 if (bc->len == 0) {
11694 /* clear */
11695 rc = -t4_load_bootcfg(sc, NULL, 0);
11696 goto done;
11697 }
11698
11699 bc_data = malloc(bc->len, M_CXGBE, M_WAITOK);
11700
11701 rc = copyin(bc->data, bc_data, bc->len);
11702 if (rc == 0)
11703 rc = -t4_load_bootcfg(sc, bc_data, bc->len);
11704
11705 free(bc_data, M_CXGBE);
11706 done:
11707 end_synchronized_op(sc, 0);
11708 return (rc);
11709 }
11710
11711 static int
cudbg_dump(struct adapter * sc,struct t4_cudbg_dump * dump)11712 cudbg_dump(struct adapter *sc, struct t4_cudbg_dump *dump)
11713 {
11714 int rc;
11715 struct cudbg_init *cudbg;
11716 void *handle, *buf;
11717
11718 /* buf is large, don't block if no memory is available */
11719 buf = malloc(dump->len, M_CXGBE, M_NOWAIT | M_ZERO);
11720 if (buf == NULL)
11721 return (ENOMEM);
11722
11723 handle = cudbg_alloc_handle();
11724 if (handle == NULL) {
11725 rc = ENOMEM;
11726 goto done;
11727 }
11728
11729 cudbg = cudbg_get_init(handle);
11730 cudbg->adap = sc;
11731 cudbg->print = (cudbg_print_cb)printf;
11732
11733 #ifndef notyet
11734 device_printf(sc->dev, "%s: wr_flash %u, len %u, data %p.\n",
11735 __func__, dump->wr_flash, dump->len, dump->data);
11736 #endif
11737
11738 if (dump->wr_flash)
11739 cudbg->use_flash = 1;
11740 MPASS(sizeof(cudbg->dbg_bitmap) == sizeof(dump->bitmap));
11741 memcpy(cudbg->dbg_bitmap, dump->bitmap, sizeof(cudbg->dbg_bitmap));
11742
11743 rc = cudbg_collect(handle, buf, &dump->len);
11744 if (rc != 0)
11745 goto done;
11746
11747 rc = copyout(buf, dump->data, dump->len);
11748 done:
11749 cudbg_free_handle(handle);
11750 free(buf, M_CXGBE);
11751 return (rc);
11752 }
11753
11754 static void
free_offload_policy(struct t4_offload_policy * op)11755 free_offload_policy(struct t4_offload_policy *op)
11756 {
11757 struct offload_rule *r;
11758 int i;
11759
11760 if (op == NULL)
11761 return;
11762
11763 r = &op->rule[0];
11764 for (i = 0; i < op->nrules; i++, r++) {
11765 free(r->bpf_prog.bf_insns, M_CXGBE);
11766 }
11767 free(op->rule, M_CXGBE);
11768 free(op, M_CXGBE);
11769 }
11770
11771 static int
set_offload_policy(struct adapter * sc,struct t4_offload_policy * uop)11772 set_offload_policy(struct adapter *sc, struct t4_offload_policy *uop)
11773 {
11774 int i, rc, len;
11775 struct t4_offload_policy *op, *old;
11776 struct bpf_program *bf;
11777 const struct offload_settings *s;
11778 struct offload_rule *r;
11779 void *u;
11780
11781 if (!is_offload(sc))
11782 return (ENODEV);
11783
11784 if (uop->nrules == 0) {
11785 /* Delete installed policies. */
11786 op = NULL;
11787 goto set_policy;
11788 } else if (uop->nrules > 256) { /* arbitrary */
11789 return (E2BIG);
11790 }
11791
11792 /* Copy userspace offload policy to kernel */
11793 op = malloc(sizeof(*op), M_CXGBE, M_ZERO | M_WAITOK);
11794 op->nrules = uop->nrules;
11795 len = op->nrules * sizeof(struct offload_rule);
11796 op->rule = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
11797 rc = copyin(uop->rule, op->rule, len);
11798 if (rc) {
11799 free(op->rule, M_CXGBE);
11800 free(op, M_CXGBE);
11801 return (rc);
11802 }
11803
11804 r = &op->rule[0];
11805 for (i = 0; i < op->nrules; i++, r++) {
11806
11807 /* Validate open_type */
11808 if (r->open_type != OPEN_TYPE_LISTEN &&
11809 r->open_type != OPEN_TYPE_ACTIVE &&
11810 r->open_type != OPEN_TYPE_PASSIVE &&
11811 r->open_type != OPEN_TYPE_DONTCARE) {
11812 error:
11813 /*
11814 * Rules 0 to i have malloc'd filters that need to be
11815 * freed. Rules i+1 to nrules have userspace pointers
11816 * and should be left alone.
11817 */
11818 op->nrules = i;
11819 free_offload_policy(op);
11820 return (rc);
11821 }
11822
11823 /* Validate settings */
11824 s = &r->settings;
11825 if ((s->offload != 0 && s->offload != 1) ||
11826 s->cong_algo < -1 || s->cong_algo > CONG_ALG_HIGHSPEED ||
11827 s->sched_class < -1 ||
11828 s->sched_class >= sc->params.nsched_cls) {
11829 rc = EINVAL;
11830 goto error;
11831 }
11832
11833 bf = &r->bpf_prog;
11834 u = bf->bf_insns; /* userspace ptr */
11835 bf->bf_insns = NULL;
11836 if (bf->bf_len == 0) {
11837 /* legal, matches everything */
11838 continue;
11839 }
11840 len = bf->bf_len * sizeof(*bf->bf_insns);
11841 bf->bf_insns = malloc(len, M_CXGBE, M_ZERO | M_WAITOK);
11842 rc = copyin(u, bf->bf_insns, len);
11843 if (rc != 0)
11844 goto error;
11845
11846 if (!bpf_validate(bf->bf_insns, bf->bf_len)) {
11847 rc = EINVAL;
11848 goto error;
11849 }
11850 }
11851 set_policy:
11852 rw_wlock(&sc->policy_lock);
11853 old = sc->policy;
11854 sc->policy = op;
11855 rw_wunlock(&sc->policy_lock);
11856 free_offload_policy(old);
11857
11858 return (0);
11859 }
11860
11861 #define MAX_READ_BUF_SIZE (128 * 1024)
11862 static int
read_card_mem(struct adapter * sc,int win,struct t4_mem_range * mr)11863 read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr)
11864 {
11865 uint32_t addr, remaining, n;
11866 uint32_t *buf;
11867 int rc;
11868 uint8_t *dst;
11869
11870 mtx_lock(&sc->reg_lock);
11871 if (hw_off_limits(sc))
11872 rc = ENXIO;
11873 else
11874 rc = validate_mem_range(sc, mr->addr, mr->len);
11875 mtx_unlock(&sc->reg_lock);
11876 if (rc != 0)
11877 return (rc);
11878
11879 buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK);
11880 addr = mr->addr;
11881 remaining = mr->len;
11882 dst = (void *)mr->data;
11883
11884 while (remaining) {
11885 n = min(remaining, MAX_READ_BUF_SIZE);
11886 mtx_lock(&sc->reg_lock);
11887 if (hw_off_limits(sc))
11888 rc = ENXIO;
11889 else
11890 read_via_memwin(sc, 2, addr, buf, n);
11891 mtx_unlock(&sc->reg_lock);
11892 if (rc != 0)
11893 break;
11894
11895 rc = copyout(buf, dst, n);
11896 if (rc != 0)
11897 break;
11898
11899 dst += n;
11900 remaining -= n;
11901 addr += n;
11902 }
11903
11904 free(buf, M_CXGBE);
11905 return (rc);
11906 }
11907 #undef MAX_READ_BUF_SIZE
11908
11909 static int
read_i2c(struct adapter * sc,struct t4_i2c_data * i2cd)11910 read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
11911 {
11912 int rc;
11913
11914 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
11915 return (EINVAL);
11916
11917 if (i2cd->len > sizeof(i2cd->data))
11918 return (EFBIG);
11919
11920 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
11921 if (rc)
11922 return (rc);
11923 if (hw_off_limits(sc))
11924 rc = ENXIO;
11925 else
11926 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
11927 i2cd->offset, i2cd->len, &i2cd->data[0]);
11928 end_synchronized_op(sc, 0);
11929
11930 return (rc);
11931 }
11932
11933 static int
clear_stats(struct adapter * sc,u_int port_id)11934 clear_stats(struct adapter *sc, u_int port_id)
11935 {
11936 int i, v, chan_map;
11937 struct port_info *pi;
11938 struct vi_info *vi;
11939 struct sge_rxq *rxq;
11940 struct sge_txq *txq;
11941 struct sge_wrq *wrq;
11942 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
11943 struct sge_ofld_txq *ofld_txq;
11944 #endif
11945 #ifdef TCP_OFFLOAD
11946 struct sge_ofld_rxq *ofld_rxq;
11947 #endif
11948
11949 if (port_id >= sc->params.nports)
11950 return (EINVAL);
11951 pi = sc->port[port_id];
11952 if (pi == NULL)
11953 return (EIO);
11954
11955 mtx_lock(&sc->reg_lock);
11956 if (!hw_off_limits(sc)) {
11957 /* MAC stats */
11958 t4_clr_port_stats(sc, pi->tx_chan);
11959 if (is_t6(sc)) {
11960 if (pi->fcs_reg != -1)
11961 pi->fcs_base = t4_read_reg64(sc, pi->fcs_reg);
11962 else
11963 pi->stats.rx_fcs_err = 0;
11964 }
11965 for_each_vi(pi, v, vi) {
11966 if (vi->flags & VI_INIT_DONE)
11967 t4_clr_vi_stats(sc, vi->vin);
11968 }
11969 chan_map = pi->rx_e_chan_map;
11970 v = 0; /* reuse */
11971 while (chan_map) {
11972 i = ffs(chan_map) - 1;
11973 t4_write_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v,
11974 1, A_TP_MIB_TNL_CNG_DROP_0 + i);
11975 chan_map &= ~(1 << i);
11976 }
11977 }
11978 mtx_unlock(&sc->reg_lock);
11979 pi->tx_parse_error = 0;
11980 pi->tnl_cong_drops = 0;
11981
11982 /*
11983 * Since this command accepts a port, clear stats for
11984 * all VIs on this port.
11985 */
11986 for_each_vi(pi, v, vi) {
11987 if (vi->flags & VI_INIT_DONE) {
11988
11989 for_each_rxq(vi, i, rxq) {
11990 #if defined(INET) || defined(INET6)
11991 rxq->lro.lro_queued = 0;
11992 rxq->lro.lro_flushed = 0;
11993 #endif
11994 rxq->rxcsum = 0;
11995 rxq->vlan_extraction = 0;
11996 rxq->vxlan_rxcsum = 0;
11997
11998 rxq->fl.cl_allocated = 0;
11999 rxq->fl.cl_recycled = 0;
12000 rxq->fl.cl_fast_recycled = 0;
12001 }
12002
12003 for_each_txq(vi, i, txq) {
12004 txq->txcsum = 0;
12005 txq->tso_wrs = 0;
12006 txq->vlan_insertion = 0;
12007 txq->imm_wrs = 0;
12008 txq->sgl_wrs = 0;
12009 txq->txpkt_wrs = 0;
12010 txq->txpkts0_wrs = 0;
12011 txq->txpkts1_wrs = 0;
12012 txq->txpkts0_pkts = 0;
12013 txq->txpkts1_pkts = 0;
12014 txq->txpkts_flush = 0;
12015 txq->raw_wrs = 0;
12016 txq->vxlan_tso_wrs = 0;
12017 txq->vxlan_txcsum = 0;
12018 txq->kern_tls_records = 0;
12019 txq->kern_tls_short = 0;
12020 txq->kern_tls_partial = 0;
12021 txq->kern_tls_full = 0;
12022 txq->kern_tls_octets = 0;
12023 txq->kern_tls_waste = 0;
12024 txq->kern_tls_options = 0;
12025 txq->kern_tls_header = 0;
12026 txq->kern_tls_fin = 0;
12027 txq->kern_tls_fin_short = 0;
12028 txq->kern_tls_cbc = 0;
12029 txq->kern_tls_gcm = 0;
12030 mp_ring_reset_stats(txq->r);
12031 }
12032
12033 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
12034 for_each_ofld_txq(vi, i, ofld_txq) {
12035 ofld_txq->wrq.tx_wrs_direct = 0;
12036 ofld_txq->wrq.tx_wrs_copied = 0;
12037 counter_u64_zero(ofld_txq->tx_iscsi_pdus);
12038 counter_u64_zero(ofld_txq->tx_iscsi_octets);
12039 counter_u64_zero(ofld_txq->tx_iscsi_iso_wrs);
12040 counter_u64_zero(ofld_txq->tx_aio_jobs);
12041 counter_u64_zero(ofld_txq->tx_aio_octets);
12042 counter_u64_zero(ofld_txq->tx_toe_tls_records);
12043 counter_u64_zero(ofld_txq->tx_toe_tls_octets);
12044 }
12045 #endif
12046 #ifdef TCP_OFFLOAD
12047 for_each_ofld_rxq(vi, i, ofld_rxq) {
12048 ofld_rxq->fl.cl_allocated = 0;
12049 ofld_rxq->fl.cl_recycled = 0;
12050 ofld_rxq->fl.cl_fast_recycled = 0;
12051 counter_u64_zero(
12052 ofld_rxq->rx_iscsi_ddp_setup_ok);
12053 counter_u64_zero(
12054 ofld_rxq->rx_iscsi_ddp_setup_error);
12055 ofld_rxq->rx_iscsi_ddp_pdus = 0;
12056 ofld_rxq->rx_iscsi_ddp_octets = 0;
12057 ofld_rxq->rx_iscsi_fl_pdus = 0;
12058 ofld_rxq->rx_iscsi_fl_octets = 0;
12059 ofld_rxq->rx_aio_ddp_jobs = 0;
12060 ofld_rxq->rx_aio_ddp_octets = 0;
12061 ofld_rxq->rx_toe_tls_records = 0;
12062 ofld_rxq->rx_toe_tls_octets = 0;
12063 ofld_rxq->rx_toe_ddp_octets = 0;
12064 counter_u64_zero(ofld_rxq->ddp_buffer_alloc);
12065 counter_u64_zero(ofld_rxq->ddp_buffer_reuse);
12066 counter_u64_zero(ofld_rxq->ddp_buffer_free);
12067 }
12068 #endif
12069
12070 if (IS_MAIN_VI(vi)) {
12071 wrq = &sc->sge.ctrlq[pi->port_id];
12072 wrq->tx_wrs_direct = 0;
12073 wrq->tx_wrs_copied = 0;
12074 }
12075 }
12076 }
12077
12078 return (0);
12079 }
12080
12081 static int
hold_clip_addr(struct adapter * sc,struct t4_clip_addr * ca)12082 hold_clip_addr(struct adapter *sc, struct t4_clip_addr *ca)
12083 {
12084 #ifdef INET6
12085 struct in6_addr in6;
12086
12087 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
12088 if (t4_get_clip_entry(sc, &in6, true) != NULL)
12089 return (0);
12090 else
12091 return (EIO);
12092 #else
12093 return (ENOTSUP);
12094 #endif
12095 }
12096
12097 static int
release_clip_addr(struct adapter * sc,struct t4_clip_addr * ca)12098 release_clip_addr(struct adapter *sc, struct t4_clip_addr *ca)
12099 {
12100 #ifdef INET6
12101 struct in6_addr in6;
12102
12103 bcopy(&ca->addr[0], &in6.s6_addr[0], sizeof(in6.s6_addr));
12104 return (t4_release_clip_addr(sc, &in6));
12105 #else
12106 return (ENOTSUP);
12107 #endif
12108 }
12109
12110 int
t4_os_find_pci_capability(struct adapter * sc,int cap)12111 t4_os_find_pci_capability(struct adapter *sc, int cap)
12112 {
12113 int i;
12114
12115 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
12116 }
12117
12118 int
t4_os_pci_save_state(struct adapter * sc)12119 t4_os_pci_save_state(struct adapter *sc)
12120 {
12121 device_t dev;
12122 struct pci_devinfo *dinfo;
12123
12124 dev = sc->dev;
12125 dinfo = device_get_ivars(dev);
12126
12127 pci_cfg_save(dev, dinfo, 0);
12128 return (0);
12129 }
12130
12131 int
t4_os_pci_restore_state(struct adapter * sc)12132 t4_os_pci_restore_state(struct adapter *sc)
12133 {
12134 device_t dev;
12135 struct pci_devinfo *dinfo;
12136
12137 dev = sc->dev;
12138 dinfo = device_get_ivars(dev);
12139
12140 pci_cfg_restore(dev, dinfo);
12141 return (0);
12142 }
12143
12144 void
t4_os_portmod_changed(struct port_info * pi)12145 t4_os_portmod_changed(struct port_info *pi)
12146 {
12147 struct adapter *sc = pi->adapter;
12148 struct vi_info *vi;
12149 if_t ifp;
12150 static const char *mod_str[] = {
12151 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
12152 };
12153
12154 KASSERT((pi->flags & FIXED_IFMEDIA) == 0,
12155 ("%s: port_type %u", __func__, pi->port_type));
12156
12157 vi = &pi->vi[0];
12158 if (begin_synchronized_op(sc, vi, HOLD_LOCK, "t4mod") == 0) {
12159 PORT_LOCK(pi);
12160 build_medialist(pi);
12161 if (pi->mod_type != FW_PORT_MOD_TYPE_NONE) {
12162 fixup_link_config(pi);
12163 apply_link_config(pi);
12164 }
12165 PORT_UNLOCK(pi);
12166 end_synchronized_op(sc, LOCK_HELD);
12167 }
12168
12169 ifp = vi->ifp;
12170 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
12171 if_printf(ifp, "transceiver unplugged.\n");
12172 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
12173 if_printf(ifp, "unknown transceiver inserted.\n");
12174 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
12175 if_printf(ifp, "unsupported transceiver inserted.\n");
12176 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
12177 if_printf(ifp, "%dGbps %s transceiver inserted.\n",
12178 port_top_speed(pi), mod_str[pi->mod_type]);
12179 } else {
12180 if_printf(ifp, "transceiver (type %d) inserted.\n",
12181 pi->mod_type);
12182 }
12183 }
12184
12185 void
t4_os_link_changed(struct port_info * pi)12186 t4_os_link_changed(struct port_info *pi)
12187 {
12188 struct vi_info *vi;
12189 if_t ifp;
12190 struct link_config *lc = &pi->link_cfg;
12191 struct adapter *sc = pi->adapter;
12192 int v;
12193
12194 PORT_LOCK_ASSERT_OWNED(pi);
12195
12196 if (is_t6(sc)) {
12197 if (lc->link_ok) {
12198 if (lc->speed > 25000 ||
12199 (lc->speed == 25000 && lc->fec == FEC_RS)) {
12200 pi->fcs_reg = T5_PORT_REG(pi->tx_chan,
12201 A_MAC_PORT_AFRAMECHECKSEQUENCEERRORS);
12202 } else {
12203 pi->fcs_reg = T5_PORT_REG(pi->tx_chan,
12204 A_MAC_PORT_MTIP_1G10G_RX_CRCERRORS);
12205 }
12206 pi->fcs_base = t4_read_reg64(sc, pi->fcs_reg);
12207 pi->stats.rx_fcs_err = 0;
12208 } else {
12209 pi->fcs_reg = -1;
12210 }
12211 } else {
12212 MPASS(pi->fcs_reg != -1);
12213 MPASS(pi->fcs_base == 0);
12214 }
12215
12216 for_each_vi(pi, v, vi) {
12217 ifp = vi->ifp;
12218 if (ifp == NULL)
12219 continue;
12220
12221 if (lc->link_ok) {
12222 if_setbaudrate(ifp, IF_Mbps(lc->speed));
12223 if_link_state_change(ifp, LINK_STATE_UP);
12224 } else {
12225 if_link_state_change(ifp, LINK_STATE_DOWN);
12226 }
12227 }
12228 }
12229
12230 void
t4_iterate(void (* func)(struct adapter *,void *),void * arg)12231 t4_iterate(void (*func)(struct adapter *, void *), void *arg)
12232 {
12233 struct adapter *sc;
12234
12235 sx_slock(&t4_list_lock);
12236 SLIST_FOREACH(sc, &t4_list, link) {
12237 /*
12238 * func should not make any assumptions about what state sc is
12239 * in - the only guarantee is that sc->sc_lock is a valid lock.
12240 */
12241 func(sc, arg);
12242 }
12243 sx_sunlock(&t4_list_lock);
12244 }
12245
12246 static int
t4_ioctl(struct cdev * dev,unsigned long cmd,caddr_t data,int fflag,struct thread * td)12247 t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
12248 struct thread *td)
12249 {
12250 int rc;
12251 struct adapter *sc = dev->si_drv1;
12252
12253 rc = priv_check(td, PRIV_DRIVER);
12254 if (rc != 0)
12255 return (rc);
12256
12257 switch (cmd) {
12258 case CHELSIO_T4_GETREG: {
12259 struct t4_reg *edata = (struct t4_reg *)data;
12260
12261 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
12262 return (EFAULT);
12263
12264 mtx_lock(&sc->reg_lock);
12265 if (hw_off_limits(sc))
12266 rc = ENXIO;
12267 else if (edata->size == 4)
12268 edata->val = t4_read_reg(sc, edata->addr);
12269 else if (edata->size == 8)
12270 edata->val = t4_read_reg64(sc, edata->addr);
12271 else
12272 rc = EINVAL;
12273 mtx_unlock(&sc->reg_lock);
12274
12275 break;
12276 }
12277 case CHELSIO_T4_SETREG: {
12278 struct t4_reg *edata = (struct t4_reg *)data;
12279
12280 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
12281 return (EFAULT);
12282
12283 mtx_lock(&sc->reg_lock);
12284 if (hw_off_limits(sc))
12285 rc = ENXIO;
12286 else if (edata->size == 4) {
12287 if (edata->val & 0xffffffff00000000)
12288 rc = EINVAL;
12289 t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
12290 } else if (edata->size == 8)
12291 t4_write_reg64(sc, edata->addr, edata->val);
12292 else
12293 rc = EINVAL;
12294 mtx_unlock(&sc->reg_lock);
12295
12296 break;
12297 }
12298 case CHELSIO_T4_REGDUMP: {
12299 struct t4_regdump *regs = (struct t4_regdump *)data;
12300 int reglen = t4_get_regs_len(sc);
12301 uint8_t *buf;
12302
12303 if (regs->len < reglen) {
12304 regs->len = reglen; /* hint to the caller */
12305 return (ENOBUFS);
12306 }
12307
12308 regs->len = reglen;
12309 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
12310 mtx_lock(&sc->reg_lock);
12311 if (hw_off_limits(sc))
12312 rc = ENXIO;
12313 else
12314 get_regs(sc, regs, buf);
12315 mtx_unlock(&sc->reg_lock);
12316 if (rc == 0)
12317 rc = copyout(buf, regs->data, reglen);
12318 free(buf, M_CXGBE);
12319 break;
12320 }
12321 case CHELSIO_T4_GET_FILTER_MODE:
12322 rc = get_filter_mode(sc, (uint32_t *)data);
12323 break;
12324 case CHELSIO_T4_SET_FILTER_MODE:
12325 rc = set_filter_mode(sc, *(uint32_t *)data);
12326 break;
12327 case CHELSIO_T4_SET_FILTER_MASK:
12328 rc = set_filter_mask(sc, *(uint32_t *)data);
12329 break;
12330 case CHELSIO_T4_GET_FILTER:
12331 rc = get_filter(sc, (struct t4_filter *)data);
12332 break;
12333 case CHELSIO_T4_SET_FILTER:
12334 rc = set_filter(sc, (struct t4_filter *)data);
12335 break;
12336 case CHELSIO_T4_DEL_FILTER:
12337 rc = del_filter(sc, (struct t4_filter *)data);
12338 break;
12339 case CHELSIO_T4_GET_SGE_CONTEXT:
12340 rc = get_sge_context(sc, (struct t4_sge_context *)data);
12341 break;
12342 case CHELSIO_T4_LOAD_FW:
12343 rc = load_fw(sc, (struct t4_data *)data);
12344 break;
12345 case CHELSIO_T4_GET_MEM:
12346 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data);
12347 break;
12348 case CHELSIO_T4_GET_I2C:
12349 rc = read_i2c(sc, (struct t4_i2c_data *)data);
12350 break;
12351 case CHELSIO_T4_CLEAR_STATS:
12352 rc = clear_stats(sc, *(uint32_t *)data);
12353 break;
12354 case CHELSIO_T4_SCHED_CLASS:
12355 rc = t4_set_sched_class(sc, (struct t4_sched_params *)data);
12356 break;
12357 case CHELSIO_T4_SCHED_QUEUE:
12358 rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data);
12359 break;
12360 case CHELSIO_T4_GET_TRACER:
12361 rc = t4_get_tracer(sc, (struct t4_tracer *)data);
12362 break;
12363 case CHELSIO_T4_SET_TRACER:
12364 rc = t4_set_tracer(sc, (struct t4_tracer *)data);
12365 break;
12366 case CHELSIO_T4_LOAD_CFG:
12367 rc = load_cfg(sc, (struct t4_data *)data);
12368 break;
12369 case CHELSIO_T4_LOAD_BOOT:
12370 rc = load_boot(sc, (struct t4_bootrom *)data);
12371 break;
12372 case CHELSIO_T4_LOAD_BOOTCFG:
12373 rc = load_bootcfg(sc, (struct t4_data *)data);
12374 break;
12375 case CHELSIO_T4_CUDBG_DUMP:
12376 rc = cudbg_dump(sc, (struct t4_cudbg_dump *)data);
12377 break;
12378 case CHELSIO_T4_SET_OFLD_POLICY:
12379 rc = set_offload_policy(sc, (struct t4_offload_policy *)data);
12380 break;
12381 case CHELSIO_T4_HOLD_CLIP_ADDR:
12382 rc = hold_clip_addr(sc, (struct t4_clip_addr *)data);
12383 break;
12384 case CHELSIO_T4_RELEASE_CLIP_ADDR:
12385 rc = release_clip_addr(sc, (struct t4_clip_addr *)data);
12386 break;
12387 default:
12388 rc = ENOTTY;
12389 }
12390
12391 return (rc);
12392 }
12393
12394 #ifdef TCP_OFFLOAD
12395 static int
toe_capability(struct vi_info * vi,bool enable)12396 toe_capability(struct vi_info *vi, bool enable)
12397 {
12398 int rc;
12399 struct port_info *pi = vi->pi;
12400 struct adapter *sc = pi->adapter;
12401
12402 ASSERT_SYNCHRONIZED_OP(sc);
12403
12404 if (!is_offload(sc))
12405 return (ENODEV);
12406 if (hw_off_limits(sc))
12407 return (ENXIO);
12408
12409 if (enable) {
12410 #ifdef KERN_TLS
12411 if (sc->flags & KERN_TLS_ON && is_t6(sc)) {
12412 int i, j, n;
12413 struct port_info *p;
12414 struct vi_info *v;
12415
12416 /*
12417 * Reconfigure hardware for TOE if TXTLS is not enabled
12418 * on any ifnet.
12419 */
12420 n = 0;
12421 for_each_port(sc, i) {
12422 p = sc->port[i];
12423 for_each_vi(p, j, v) {
12424 if (if_getcapenable(v->ifp) & IFCAP_TXTLS) {
12425 CH_WARN(sc,
12426 "%s has NIC TLS enabled.\n",
12427 device_get_nameunit(v->dev));
12428 n++;
12429 }
12430 }
12431 }
12432 if (n > 0) {
12433 CH_WARN(sc, "Disable NIC TLS on all interfaces "
12434 "associated with this adapter before "
12435 "trying to enable TOE.\n");
12436 return (EAGAIN);
12437 }
12438 rc = t6_config_kern_tls(sc, false);
12439 if (rc)
12440 return (rc);
12441 }
12442 #endif
12443 if ((if_getcapenable(vi->ifp) & IFCAP_TOE) != 0) {
12444 /* TOE is already enabled. */
12445 return (0);
12446 }
12447
12448 /*
12449 * We need the port's queues around so that we're able to send
12450 * and receive CPLs to/from the TOE even if the ifnet for this
12451 * port has never been UP'd administratively.
12452 */
12453 if (!(vi->flags & VI_INIT_DONE) && ((rc = vi_init(vi)) != 0))
12454 return (rc);
12455 if (!(pi->vi[0].flags & VI_INIT_DONE) &&
12456 ((rc = vi_init(&pi->vi[0])) != 0))
12457 return (rc);
12458
12459 if (isset(&sc->offload_map, pi->port_id)) {
12460 /* TOE is enabled on another VI of this port. */
12461 pi->uld_vis++;
12462 return (0);
12463 }
12464
12465 if (!uld_active(sc, ULD_TOM)) {
12466 rc = t4_activate_uld(sc, ULD_TOM);
12467 if (rc == EAGAIN) {
12468 log(LOG_WARNING,
12469 "You must kldload t4_tom.ko before trying "
12470 "to enable TOE on a cxgbe interface.\n");
12471 }
12472 if (rc != 0)
12473 return (rc);
12474 KASSERT(sc->tom_softc != NULL,
12475 ("%s: TOM activated but softc NULL", __func__));
12476 KASSERT(uld_active(sc, ULD_TOM),
12477 ("%s: TOM activated but flag not set", __func__));
12478 }
12479
12480 /* Activate iWARP and iSCSI too, if the modules are loaded. */
12481 if (!uld_active(sc, ULD_IWARP))
12482 (void) t4_activate_uld(sc, ULD_IWARP);
12483 if (!uld_active(sc, ULD_ISCSI))
12484 (void) t4_activate_uld(sc, ULD_ISCSI);
12485
12486 pi->uld_vis++;
12487 setbit(&sc->offload_map, pi->port_id);
12488 } else {
12489 pi->uld_vis--;
12490
12491 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0)
12492 return (0);
12493
12494 KASSERT(uld_active(sc, ULD_TOM),
12495 ("%s: TOM never initialized?", __func__));
12496 clrbit(&sc->offload_map, pi->port_id);
12497 }
12498
12499 return (0);
12500 }
12501
12502 /*
12503 * Add an upper layer driver to the global list.
12504 */
12505 int
t4_register_uld(struct uld_info * ui)12506 t4_register_uld(struct uld_info *ui)
12507 {
12508 int rc = 0;
12509 struct uld_info *u;
12510
12511 sx_xlock(&t4_uld_list_lock);
12512 SLIST_FOREACH(u, &t4_uld_list, link) {
12513 if (u->uld_id == ui->uld_id) {
12514 rc = EEXIST;
12515 goto done;
12516 }
12517 }
12518
12519 SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
12520 ui->refcount = 0;
12521 done:
12522 sx_xunlock(&t4_uld_list_lock);
12523 return (rc);
12524 }
12525
12526 int
t4_unregister_uld(struct uld_info * ui)12527 t4_unregister_uld(struct uld_info *ui)
12528 {
12529 int rc = EINVAL;
12530 struct uld_info *u;
12531
12532 sx_xlock(&t4_uld_list_lock);
12533
12534 SLIST_FOREACH(u, &t4_uld_list, link) {
12535 if (u == ui) {
12536 if (ui->refcount > 0) {
12537 rc = EBUSY;
12538 goto done;
12539 }
12540
12541 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
12542 rc = 0;
12543 goto done;
12544 }
12545 }
12546 done:
12547 sx_xunlock(&t4_uld_list_lock);
12548 return (rc);
12549 }
12550
12551 int
t4_activate_uld(struct adapter * sc,int id)12552 t4_activate_uld(struct adapter *sc, int id)
12553 {
12554 int rc;
12555 struct uld_info *ui;
12556
12557 ASSERT_SYNCHRONIZED_OP(sc);
12558
12559 if (id < 0 || id > ULD_MAX)
12560 return (EINVAL);
12561 rc = EAGAIN; /* kldoad the module with this ULD and try again. */
12562
12563 sx_slock(&t4_uld_list_lock);
12564
12565 SLIST_FOREACH(ui, &t4_uld_list, link) {
12566 if (ui->uld_id == id) {
12567 if (!(sc->flags & FULL_INIT_DONE)) {
12568 rc = adapter_init(sc);
12569 if (rc != 0)
12570 break;
12571 }
12572
12573 rc = ui->activate(sc);
12574 if (rc == 0) {
12575 setbit(&sc->active_ulds, id);
12576 ui->refcount++;
12577 }
12578 break;
12579 }
12580 }
12581
12582 sx_sunlock(&t4_uld_list_lock);
12583
12584 return (rc);
12585 }
12586
12587 int
t4_deactivate_uld(struct adapter * sc,int id)12588 t4_deactivate_uld(struct adapter *sc, int id)
12589 {
12590 int rc;
12591 struct uld_info *ui;
12592
12593 ASSERT_SYNCHRONIZED_OP(sc);
12594
12595 if (id < 0 || id > ULD_MAX)
12596 return (EINVAL);
12597 rc = ENXIO;
12598
12599 sx_slock(&t4_uld_list_lock);
12600
12601 SLIST_FOREACH(ui, &t4_uld_list, link) {
12602 if (ui->uld_id == id) {
12603 rc = ui->deactivate(sc);
12604 if (rc == 0) {
12605 clrbit(&sc->active_ulds, id);
12606 ui->refcount--;
12607 }
12608 break;
12609 }
12610 }
12611
12612 sx_sunlock(&t4_uld_list_lock);
12613
12614 return (rc);
12615 }
12616
12617 static int
t4_deactivate_all_uld(struct adapter * sc)12618 t4_deactivate_all_uld(struct adapter *sc)
12619 {
12620 int rc;
12621 struct uld_info *ui;
12622
12623 rc = begin_synchronized_op(sc, NULL, SLEEP_OK, "t4detuld");
12624 if (rc != 0)
12625 return (ENXIO);
12626
12627 sx_slock(&t4_uld_list_lock);
12628
12629 SLIST_FOREACH(ui, &t4_uld_list, link) {
12630 if (isset(&sc->active_ulds, ui->uld_id)) {
12631 rc = ui->deactivate(sc);
12632 if (rc != 0)
12633 break;
12634 clrbit(&sc->active_ulds, ui->uld_id);
12635 ui->refcount--;
12636 }
12637 }
12638
12639 sx_sunlock(&t4_uld_list_lock);
12640 end_synchronized_op(sc, 0);
12641
12642 return (rc);
12643 }
12644
12645 static void
t4_async_event(struct adapter * sc)12646 t4_async_event(struct adapter *sc)
12647 {
12648 struct uld_info *ui;
12649
12650 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4async") != 0)
12651 return;
12652 sx_slock(&t4_uld_list_lock);
12653 SLIST_FOREACH(ui, &t4_uld_list, link) {
12654 if (ui->uld_id == ULD_IWARP) {
12655 ui->async_event(sc);
12656 break;
12657 }
12658 }
12659 sx_sunlock(&t4_uld_list_lock);
12660 end_synchronized_op(sc, 0);
12661 }
12662
12663 int
uld_active(struct adapter * sc,int uld_id)12664 uld_active(struct adapter *sc, int uld_id)
12665 {
12666
12667 MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
12668
12669 return (isset(&sc->active_ulds, uld_id));
12670 }
12671 #endif
12672
12673 #ifdef KERN_TLS
12674 static int
ktls_capability(struct adapter * sc,bool enable)12675 ktls_capability(struct adapter *sc, bool enable)
12676 {
12677 ASSERT_SYNCHRONIZED_OP(sc);
12678
12679 if (!is_ktls(sc))
12680 return (ENODEV);
12681 if (!is_t6(sc))
12682 return (0);
12683 if (hw_off_limits(sc))
12684 return (ENXIO);
12685
12686 if (enable) {
12687 if (sc->flags & KERN_TLS_ON)
12688 return (0); /* already on */
12689 if (sc->offload_map != 0) {
12690 CH_WARN(sc,
12691 "Disable TOE on all interfaces associated with "
12692 "this adapter before trying to enable NIC TLS.\n");
12693 return (EAGAIN);
12694 }
12695 return (t6_config_kern_tls(sc, true));
12696 } else {
12697 /*
12698 * Nothing to do for disable. If TOE is enabled sometime later
12699 * then toe_capability will reconfigure the hardware.
12700 */
12701 return (0);
12702 }
12703 }
12704 #endif
12705
12706 /*
12707 * t = ptr to tunable.
12708 * nc = number of CPUs.
12709 * c = compiled in default for that tunable.
12710 */
12711 static void
calculate_nqueues(int * t,int nc,const int c)12712 calculate_nqueues(int *t, int nc, const int c)
12713 {
12714 int nq;
12715
12716 if (*t > 0)
12717 return;
12718 nq = *t < 0 ? -*t : c;
12719 *t = min(nc, nq);
12720 }
12721
12722 /*
12723 * Come up with reasonable defaults for some of the tunables, provided they're
12724 * not set by the user (in which case we'll use the values as is).
12725 */
12726 static void
tweak_tunables(void)12727 tweak_tunables(void)
12728 {
12729 int nc = mp_ncpus; /* our snapshot of the number of CPUs */
12730
12731 if (t4_ntxq < 1) {
12732 #ifdef RSS
12733 t4_ntxq = rss_getnumbuckets();
12734 #else
12735 calculate_nqueues(&t4_ntxq, nc, NTXQ);
12736 #endif
12737 }
12738
12739 calculate_nqueues(&t4_ntxq_vi, nc, NTXQ_VI);
12740
12741 if (t4_nrxq < 1) {
12742 #ifdef RSS
12743 t4_nrxq = rss_getnumbuckets();
12744 #else
12745 calculate_nqueues(&t4_nrxq, nc, NRXQ);
12746 #endif
12747 }
12748
12749 calculate_nqueues(&t4_nrxq_vi, nc, NRXQ_VI);
12750
12751 #if defined(TCP_OFFLOAD) || defined(RATELIMIT)
12752 calculate_nqueues(&t4_nofldtxq, nc, NOFLDTXQ);
12753 calculate_nqueues(&t4_nofldtxq_vi, nc, NOFLDTXQ_VI);
12754 #endif
12755 #ifdef TCP_OFFLOAD
12756 calculate_nqueues(&t4_nofldrxq, nc, NOFLDRXQ);
12757 calculate_nqueues(&t4_nofldrxq_vi, nc, NOFLDRXQ_VI);
12758 #endif
12759
12760 #if defined(TCP_OFFLOAD) || defined(KERN_TLS)
12761 if (t4_toecaps_allowed == -1)
12762 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
12763 #else
12764 if (t4_toecaps_allowed == -1)
12765 t4_toecaps_allowed = 0;
12766 #endif
12767
12768 #ifdef TCP_OFFLOAD
12769 if (t4_rdmacaps_allowed == -1) {
12770 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP |
12771 FW_CAPS_CONFIG_RDMA_RDMAC;
12772 }
12773
12774 if (t4_iscsicaps_allowed == -1) {
12775 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU |
12776 FW_CAPS_CONFIG_ISCSI_TARGET_PDU |
12777 FW_CAPS_CONFIG_ISCSI_T10DIF;
12778 }
12779
12780 if (t4_tmr_idx_ofld < 0 || t4_tmr_idx_ofld >= SGE_NTIMERS)
12781 t4_tmr_idx_ofld = TMR_IDX_OFLD;
12782
12783 if (t4_pktc_idx_ofld < -1 || t4_pktc_idx_ofld >= SGE_NCOUNTERS)
12784 t4_pktc_idx_ofld = PKTC_IDX_OFLD;
12785 #else
12786 if (t4_rdmacaps_allowed == -1)
12787 t4_rdmacaps_allowed = 0;
12788
12789 if (t4_iscsicaps_allowed == -1)
12790 t4_iscsicaps_allowed = 0;
12791 #endif
12792
12793 #ifdef DEV_NETMAP
12794 calculate_nqueues(&t4_nnmtxq, nc, NNMTXQ);
12795 calculate_nqueues(&t4_nnmrxq, nc, NNMRXQ);
12796 calculate_nqueues(&t4_nnmtxq_vi, nc, NNMTXQ_VI);
12797 calculate_nqueues(&t4_nnmrxq_vi, nc, NNMRXQ_VI);
12798 #endif
12799
12800 if (t4_tmr_idx < 0 || t4_tmr_idx >= SGE_NTIMERS)
12801 t4_tmr_idx = TMR_IDX;
12802
12803 if (t4_pktc_idx < -1 || t4_pktc_idx >= SGE_NCOUNTERS)
12804 t4_pktc_idx = PKTC_IDX;
12805
12806 if (t4_qsize_txq < 128)
12807 t4_qsize_txq = 128;
12808
12809 if (t4_qsize_rxq < 128)
12810 t4_qsize_rxq = 128;
12811 while (t4_qsize_rxq & 7)
12812 t4_qsize_rxq++;
12813
12814 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
12815
12816 /*
12817 * Number of VIs to create per-port. The first VI is the "main" regular
12818 * VI for the port. The rest are additional virtual interfaces on the
12819 * same physical port. Note that the main VI does not have native
12820 * netmap support but the extra VIs do.
12821 *
12822 * Limit the number of VIs per port to the number of available
12823 * MAC addresses per port.
12824 */
12825 if (t4_num_vis < 1)
12826 t4_num_vis = 1;
12827 if (t4_num_vis > nitems(vi_mac_funcs)) {
12828 t4_num_vis = nitems(vi_mac_funcs);
12829 printf("cxgbe: number of VIs limited to %d\n", t4_num_vis);
12830 }
12831
12832 if (pcie_relaxed_ordering < 0 || pcie_relaxed_ordering > 2) {
12833 pcie_relaxed_ordering = 1;
12834 #if defined(__i386__) || defined(__amd64__)
12835 if (cpu_vendor_id == CPU_VENDOR_INTEL)
12836 pcie_relaxed_ordering = 0;
12837 #endif
12838 }
12839 }
12840
12841 #ifdef DDB
12842 static void
t4_dump_tcb(struct adapter * sc,int tid)12843 t4_dump_tcb(struct adapter *sc, int tid)
12844 {
12845 uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos;
12846
12847 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2);
12848 save = t4_read_reg(sc, reg);
12849 base = sc->memwin[2].mw_base;
12850
12851 /* Dump TCB for the tid */
12852 tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
12853 tcb_addr += tid * TCB_SIZE;
12854
12855 if (is_t4(sc)) {
12856 pf = 0;
12857 win_pos = tcb_addr & ~0xf; /* start must be 16B aligned */
12858 } else {
12859 pf = V_PFNUM(sc->pf);
12860 win_pos = tcb_addr & ~0x7f; /* start must be 128B aligned */
12861 }
12862 t4_write_reg(sc, reg, win_pos | pf);
12863 t4_read_reg(sc, reg);
12864
12865 off = tcb_addr - win_pos;
12866 for (i = 0; i < 4; i++) {
12867 uint32_t buf[8];
12868 for (j = 0; j < 8; j++, off += 4)
12869 buf[j] = htonl(t4_read_reg(sc, base + off));
12870
12871 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n",
12872 buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6],
12873 buf[7]);
12874 }
12875
12876 t4_write_reg(sc, reg, save);
12877 t4_read_reg(sc, reg);
12878 }
12879
12880 static void
t4_dump_devlog(struct adapter * sc)12881 t4_dump_devlog(struct adapter *sc)
12882 {
12883 struct devlog_params *dparams = &sc->params.devlog;
12884 struct fw_devlog_e e;
12885 int i, first, j, m, nentries, rc;
12886 uint64_t ftstamp = UINT64_MAX;
12887
12888 if (dparams->start == 0) {
12889 db_printf("devlog params not valid\n");
12890 return;
12891 }
12892
12893 nentries = dparams->size / sizeof(struct fw_devlog_e);
12894 m = fwmtype_to_hwmtype(dparams->memtype);
12895
12896 /* Find the first entry. */
12897 first = -1;
12898 for (i = 0; i < nentries && !db_pager_quit; i++) {
12899 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
12900 sizeof(e), (void *)&e);
12901 if (rc != 0)
12902 break;
12903
12904 if (e.timestamp == 0)
12905 break;
12906
12907 e.timestamp = be64toh(e.timestamp);
12908 if (e.timestamp < ftstamp) {
12909 ftstamp = e.timestamp;
12910 first = i;
12911 }
12912 }
12913
12914 if (first == -1)
12915 return;
12916
12917 i = first;
12918 do {
12919 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e),
12920 sizeof(e), (void *)&e);
12921 if (rc != 0)
12922 return;
12923
12924 if (e.timestamp == 0)
12925 return;
12926
12927 e.timestamp = be64toh(e.timestamp);
12928 e.seqno = be32toh(e.seqno);
12929 for (j = 0; j < 8; j++)
12930 e.params[j] = be32toh(e.params[j]);
12931
12932 db_printf("%10d %15ju %8s %8s ",
12933 e.seqno, e.timestamp,
12934 (e.level < nitems(devlog_level_strings) ?
12935 devlog_level_strings[e.level] : "UNKNOWN"),
12936 (e.facility < nitems(devlog_facility_strings) ?
12937 devlog_facility_strings[e.facility] : "UNKNOWN"));
12938 db_printf(e.fmt, e.params[0], e.params[1], e.params[2],
12939 e.params[3], e.params[4], e.params[5], e.params[6],
12940 e.params[7]);
12941
12942 if (++i == nentries)
12943 i = 0;
12944 } while (i != first && !db_pager_quit);
12945 }
12946
12947 static DB_DEFINE_TABLE(show, t4, show_t4);
12948
DB_TABLE_COMMAND_FLAGS(show_t4,devlog,db_show_devlog,CS_OWN)12949 DB_TABLE_COMMAND_FLAGS(show_t4, devlog, db_show_devlog, CS_OWN)
12950 {
12951 device_t dev;
12952 int t;
12953 bool valid;
12954
12955 valid = false;
12956 t = db_read_token();
12957 if (t == tIDENT) {
12958 dev = device_lookup_by_name(db_tok_string);
12959 valid = true;
12960 }
12961 db_skip_to_eol();
12962 if (!valid) {
12963 db_printf("usage: show t4 devlog <nexus>\n");
12964 return;
12965 }
12966
12967 if (dev == NULL) {
12968 db_printf("device not found\n");
12969 return;
12970 }
12971
12972 t4_dump_devlog(device_get_softc(dev));
12973 }
12974
DB_TABLE_COMMAND_FLAGS(show_t4,tcb,db_show_t4tcb,CS_OWN)12975 DB_TABLE_COMMAND_FLAGS(show_t4, tcb, db_show_t4tcb, CS_OWN)
12976 {
12977 device_t dev;
12978 int radix, tid, t;
12979 bool valid;
12980
12981 valid = false;
12982 radix = db_radix;
12983 db_radix = 10;
12984 t = db_read_token();
12985 if (t == tIDENT) {
12986 dev = device_lookup_by_name(db_tok_string);
12987 t = db_read_token();
12988 if (t == tNUMBER) {
12989 tid = db_tok_number;
12990 valid = true;
12991 }
12992 }
12993 db_radix = radix;
12994 db_skip_to_eol();
12995 if (!valid) {
12996 db_printf("usage: show t4 tcb <nexus> <tid>\n");
12997 return;
12998 }
12999
13000 if (dev == NULL) {
13001 db_printf("device not found\n");
13002 return;
13003 }
13004 if (tid < 0) {
13005 db_printf("invalid tid\n");
13006 return;
13007 }
13008
13009 t4_dump_tcb(device_get_softc(dev), tid);
13010 }
13011 #endif
13012
13013 static eventhandler_tag vxlan_start_evtag;
13014 static eventhandler_tag vxlan_stop_evtag;
13015
13016 struct vxlan_evargs {
13017 if_t ifp;
13018 uint16_t port;
13019 };
13020
13021 static void
enable_vxlan_rx(struct adapter * sc)13022 enable_vxlan_rx(struct adapter *sc)
13023 {
13024 int i, rc;
13025 struct port_info *pi;
13026 uint8_t match_all_mac[ETHER_ADDR_LEN] = {0};
13027
13028 ASSERT_SYNCHRONIZED_OP(sc);
13029
13030 t4_write_reg(sc, A_MPS_RX_VXLAN_TYPE, V_VXLAN(sc->vxlan_port) |
13031 F_VXLAN_EN);
13032 for_each_port(sc, i) {
13033 pi = sc->port[i];
13034 if (pi->vxlan_tcam_entry == true)
13035 continue;
13036 rc = t4_alloc_raw_mac_filt(sc, pi->vi[0].viid, match_all_mac,
13037 match_all_mac, sc->rawf_base + pi->port_id, 1, pi->port_id,
13038 true);
13039 if (rc < 0) {
13040 rc = -rc;
13041 CH_ERR(&pi->vi[0],
13042 "failed to add VXLAN TCAM entry: %d.\n", rc);
13043 } else {
13044 MPASS(rc == sc->rawf_base + pi->port_id);
13045 pi->vxlan_tcam_entry = true;
13046 }
13047 }
13048 }
13049
13050 static void
t4_vxlan_start(struct adapter * sc,void * arg)13051 t4_vxlan_start(struct adapter *sc, void *arg)
13052 {
13053 struct vxlan_evargs *v = arg;
13054
13055 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5)
13056 return;
13057 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vxst") != 0)
13058 return;
13059
13060 if (sc->vxlan_refcount == 0) {
13061 sc->vxlan_port = v->port;
13062 sc->vxlan_refcount = 1;
13063 if (!hw_off_limits(sc))
13064 enable_vxlan_rx(sc);
13065 } else if (sc->vxlan_port == v->port) {
13066 sc->vxlan_refcount++;
13067 } else {
13068 CH_ERR(sc, "VXLAN already configured on port %d; "
13069 "ignoring attempt to configure it on port %d\n",
13070 sc->vxlan_port, v->port);
13071 }
13072 end_synchronized_op(sc, 0);
13073 }
13074
13075 static void
t4_vxlan_stop(struct adapter * sc,void * arg)13076 t4_vxlan_stop(struct adapter *sc, void *arg)
13077 {
13078 struct vxlan_evargs *v = arg;
13079
13080 if (sc->nrawf == 0 || chip_id(sc) <= CHELSIO_T5)
13081 return;
13082 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4vxsp") != 0)
13083 return;
13084
13085 /*
13086 * VXLANs may have been configured before the driver was loaded so we
13087 * may see more stops than starts. This is not handled cleanly but at
13088 * least we keep the refcount sane.
13089 */
13090 if (sc->vxlan_port != v->port)
13091 goto done;
13092 if (sc->vxlan_refcount == 0) {
13093 CH_ERR(sc, "VXLAN operation on port %d was stopped earlier; "
13094 "ignoring attempt to stop it again.\n", sc->vxlan_port);
13095 } else if (--sc->vxlan_refcount == 0 && !hw_off_limits(sc))
13096 t4_set_reg_field(sc, A_MPS_RX_VXLAN_TYPE, F_VXLAN_EN, 0);
13097 done:
13098 end_synchronized_op(sc, 0);
13099 }
13100
13101 static void
t4_vxlan_start_handler(void * arg __unused,if_t ifp,sa_family_t family,u_int port)13102 t4_vxlan_start_handler(void *arg __unused, if_t ifp,
13103 sa_family_t family, u_int port)
13104 {
13105 struct vxlan_evargs v;
13106
13107 MPASS(family == AF_INET || family == AF_INET6);
13108 v.ifp = ifp;
13109 v.port = port;
13110
13111 t4_iterate(t4_vxlan_start, &v);
13112 }
13113
13114 static void
t4_vxlan_stop_handler(void * arg __unused,if_t ifp,sa_family_t family,u_int port)13115 t4_vxlan_stop_handler(void *arg __unused, if_t ifp, sa_family_t family,
13116 u_int port)
13117 {
13118 struct vxlan_evargs v;
13119
13120 MPASS(family == AF_INET || family == AF_INET6);
13121 v.ifp = ifp;
13122 v.port = port;
13123
13124 t4_iterate(t4_vxlan_stop, &v);
13125 }
13126
13127
13128 static struct sx mlu; /* mod load unload */
13129 SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload");
13130
13131 static int
mod_event(module_t mod,int cmd,void * arg)13132 mod_event(module_t mod, int cmd, void *arg)
13133 {
13134 int rc = 0;
13135 static int loaded = 0;
13136
13137 switch (cmd) {
13138 case MOD_LOAD:
13139 sx_xlock(&mlu);
13140 if (loaded++ == 0) {
13141 t4_sge_modload();
13142 t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
13143 t4_filter_rpl, CPL_COOKIE_FILTER);
13144 t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL,
13145 do_l2t_write_rpl, CPL_COOKIE_FILTER);
13146 t4_register_shared_cpl_handler(CPL_ACT_OPEN_RPL,
13147 t4_hashfilter_ao_rpl, CPL_COOKIE_HASHFILTER);
13148 t4_register_shared_cpl_handler(CPL_SET_TCB_RPL,
13149 t4_hashfilter_tcb_rpl, CPL_COOKIE_HASHFILTER);
13150 t4_register_shared_cpl_handler(CPL_ABORT_RPL_RSS,
13151 t4_del_hashfilter_rpl, CPL_COOKIE_HASHFILTER);
13152 t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt);
13153 t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt);
13154 t4_register_cpl_handler(CPL_SMT_WRITE_RPL,
13155 do_smt_write_rpl);
13156 sx_init(&t4_list_lock, "T4/T5 adapters");
13157 SLIST_INIT(&t4_list);
13158 callout_init(&fatal_callout, 1);
13159 #ifdef TCP_OFFLOAD
13160 sx_init(&t4_uld_list_lock, "T4/T5 ULDs");
13161 SLIST_INIT(&t4_uld_list);
13162 #endif
13163 #ifdef INET6
13164 t4_clip_modload();
13165 #endif
13166 #ifdef KERN_TLS
13167 t6_ktls_modload();
13168 #endif
13169 t4_tracer_modload();
13170 tweak_tunables();
13171 vxlan_start_evtag =
13172 EVENTHANDLER_REGISTER(vxlan_start,
13173 t4_vxlan_start_handler, NULL,
13174 EVENTHANDLER_PRI_ANY);
13175 vxlan_stop_evtag =
13176 EVENTHANDLER_REGISTER(vxlan_stop,
13177 t4_vxlan_stop_handler, NULL,
13178 EVENTHANDLER_PRI_ANY);
13179 reset_tq = taskqueue_create("t4_rst_tq", M_WAITOK,
13180 taskqueue_thread_enqueue, &reset_tq);
13181 taskqueue_start_threads(&reset_tq, 1, PI_SOFT,
13182 "t4_rst_thr");
13183 }
13184 sx_xunlock(&mlu);
13185 break;
13186
13187 case MOD_UNLOAD:
13188 sx_xlock(&mlu);
13189 if (--loaded == 0) {
13190 int tries;
13191
13192 taskqueue_free(reset_tq);
13193 sx_slock(&t4_list_lock);
13194 if (!SLIST_EMPTY(&t4_list)) {
13195 rc = EBUSY;
13196 sx_sunlock(&t4_list_lock);
13197 goto done_unload;
13198 }
13199 #ifdef TCP_OFFLOAD
13200 sx_slock(&t4_uld_list_lock);
13201 if (!SLIST_EMPTY(&t4_uld_list)) {
13202 rc = EBUSY;
13203 sx_sunlock(&t4_uld_list_lock);
13204 sx_sunlock(&t4_list_lock);
13205 goto done_unload;
13206 }
13207 #endif
13208 tries = 0;
13209 while (tries++ < 5 && t4_sge_extfree_refs() != 0) {
13210 uprintf("%ju clusters with custom free routine "
13211 "still is use.\n", t4_sge_extfree_refs());
13212 pause("t4unload", 2 * hz);
13213 }
13214 #ifdef TCP_OFFLOAD
13215 sx_sunlock(&t4_uld_list_lock);
13216 #endif
13217 sx_sunlock(&t4_list_lock);
13218
13219 if (t4_sge_extfree_refs() == 0) {
13220 EVENTHANDLER_DEREGISTER(vxlan_start,
13221 vxlan_start_evtag);
13222 EVENTHANDLER_DEREGISTER(vxlan_stop,
13223 vxlan_stop_evtag);
13224 t4_tracer_modunload();
13225 #ifdef KERN_TLS
13226 t6_ktls_modunload();
13227 #endif
13228 #ifdef INET6
13229 t4_clip_modunload();
13230 #endif
13231 #ifdef TCP_OFFLOAD
13232 sx_destroy(&t4_uld_list_lock);
13233 #endif
13234 sx_destroy(&t4_list_lock);
13235 t4_sge_modunload();
13236 loaded = 0;
13237 } else {
13238 rc = EBUSY;
13239 loaded++; /* undo earlier decrement */
13240 }
13241 }
13242 done_unload:
13243 sx_xunlock(&mlu);
13244 break;
13245 }
13246
13247 return (rc);
13248 }
13249
13250 DRIVER_MODULE(t4nex, pci, t4_driver, mod_event, 0);
13251 MODULE_VERSION(t4nex, 1);
13252 MODULE_DEPEND(t4nex, firmware, 1, 1, 1);
13253 #ifdef DEV_NETMAP
13254 MODULE_DEPEND(t4nex, netmap, 1, 1, 1);
13255 #endif /* DEV_NETMAP */
13256
13257 DRIVER_MODULE(t5nex, pci, t5_driver, mod_event, 0);
13258 MODULE_VERSION(t5nex, 1);
13259 MODULE_DEPEND(t5nex, firmware, 1, 1, 1);
13260 #ifdef DEV_NETMAP
13261 MODULE_DEPEND(t5nex, netmap, 1, 1, 1);
13262 #endif /* DEV_NETMAP */
13263
13264 DRIVER_MODULE(t6nex, pci, t6_driver, mod_event, 0);
13265 MODULE_VERSION(t6nex, 1);
13266 MODULE_DEPEND(t6nex, crypto, 1, 1, 1);
13267 MODULE_DEPEND(t6nex, firmware, 1, 1, 1);
13268 #ifdef DEV_NETMAP
13269 MODULE_DEPEND(t6nex, netmap, 1, 1, 1);
13270 #endif /* DEV_NETMAP */
13271
13272 DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, 0, 0);
13273 MODULE_VERSION(cxgbe, 1);
13274
13275 DRIVER_MODULE(cxl, t5nex, cxl_driver, 0, 0);
13276 MODULE_VERSION(cxl, 1);
13277
13278 DRIVER_MODULE(cc, t6nex, cc_driver, 0, 0);
13279 MODULE_VERSION(cc, 1);
13280
13281 DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, 0, 0);
13282 MODULE_VERSION(vcxgbe, 1);
13283
13284 DRIVER_MODULE(vcxl, cxl, vcxl_driver, 0, 0);
13285 MODULE_VERSION(vcxl, 1);
13286
13287 DRIVER_MODULE(vcc, cc, vcc_driver, 0, 0);
13288 MODULE_VERSION(vcc, 1);
13289