1 /**************************************************************************
2
3 Copyright (c) 2001-2003, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ***************************************************************************/
33
34 /* $OpenBSD: if_em.c,v 1.377 2024/05/24 06:02:53 jsg Exp $ */
35 /* $FreeBSD: if_em.c,v 1.46 2004/09/29 18:28:28 mlaier Exp $ */
36
37 #include <dev/pci/if_em.h>
38 #include <dev/pci/if_em_soc.h>
39
40 /*********************************************************************
41 * Driver version
42 *********************************************************************/
43
44 #define EM_DRIVER_VERSION "6.2.9"
45
46 /*********************************************************************
47 * PCI Device ID Table
48 *********************************************************************/
49 const struct pci_matchid em_devices[] = {
50 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_CPR_DPT },
51 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_SDS_DPT },
52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_CPR_SPT },
53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_SDS_SPT },
54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EM },
55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EM_LOM },
56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EP },
57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EP_LOM },
58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EP_LP },
59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541EI },
60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541EI_MOBILE },
61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541ER },
62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541ER_LOM },
63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541GI },
64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541GI_LF },
65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541GI_MOBILE },
66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82542 },
67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82543GC_COPPER },
68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82543GC_FIBER },
69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544EI_COPPER },
70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544EI_FIBER },
71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544GC_COPPER },
72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544GC_LOM },
73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545EM_COPPER },
74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545EM_FIBER },
75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545GM_COPPER },
76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545GM_FIBER },
77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545GM_SERDES },
78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546EB_COPPER },
79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546EB_FIBER },
80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546EB_QUAD_CPR },
81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_COPPER },
82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_FIBER },
83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_PCIE },
84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_QUAD_CPR },
85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_QUAD_CPR_K },
86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_SERDES },
87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_2 },
88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82547EI },
89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82547EI_MOBILE },
90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82547GI },
91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_AF },
92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_AT },
93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_COPPER },
94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_FIBER },
95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_QUAD_CPR },
96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_QUAD_CPR_LP },
97 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_QUAD_FBR },
98 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_SERDES },
99 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_SDS_DUAL },
100 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_SDS_QUAD },
101 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571PT_QUAD_CPR },
102 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI_COPPER },
103 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI_FIBER },
104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI_SERDES },
105 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI },
106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573E },
107 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573E_IAMT },
108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573E_PM },
109 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573L },
110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573L_PL_1 },
111 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573L_PL_2 },
112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573V_PM },
113 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82574L },
114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82574LA },
115 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575EB_COPPER },
116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575EB_SERDES },
117 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575GB_QUAD_CPR },
118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575GB_QP_PM },
119 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576 },
120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_FIBER },
121 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_SERDES },
122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_QUAD_COPPER },
123 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_QUAD_CU_ET2 },
124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_NS },
125 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_NS_SERDES },
126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_SERDES_QUAD },
127 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82577LC },
128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82577LM },
129 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82578DC },
130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82578DM },
131 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82579LM },
132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82579V },
133 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER },
134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER_OEM1 },
135 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER_IT },
136 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_FIBER },
137 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_SERDES },
138 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_SGMII },
139 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER_NF },
140 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_SERDES_NF },
141 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I211_COPPER },
142 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I217_LM },
143 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I217_V },
144 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_LM },
145 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_LM_2 },
146 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_LM_3 },
147 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_V },
148 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_V_2 },
149 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_V_3 },
150 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM },
151 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM2 },
152 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM3 },
153 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM4 },
154 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM5 },
155 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM6 },
156 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM7 },
157 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM8 },
158 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM9 },
159 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM10 },
160 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM11 },
161 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM12 },
162 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM13 },
163 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM14 },
164 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM15 },
165 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM16 },
166 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM17 },
167 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM18 },
168 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM19 },
169 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM20 },
170 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM21 },
171 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM22 },
172 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM23 },
173 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM24 },
174 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V },
175 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V2 },
176 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V4 },
177 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V5 },
178 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V6 },
179 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V7 },
180 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V8 },
181 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V9 },
182 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V10 },
183 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V11 },
184 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V12 },
185 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V13 },
186 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V14 },
187 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V15 },
188 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V16 },
189 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V17 },
190 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V18 },
191 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V19 },
192 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V20 },
193 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V21 },
194 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V22 },
195 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V23 },
196 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V24 },
197 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_COPPER },
198 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_FIBER },
199 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_SERDES },
200 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_SGMII },
201 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_COPPER_DUAL },
202 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_QUAD_FIBER },
203 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_SGMII },
204 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_SERDES },
205 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_BPLANE },
206 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_SFP },
207 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82583V },
208 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_COPPER },
209 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_FIBER },
210 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_SERDES },
211 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_SGMII },
212 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I354_BP_1GBPS },
213 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I354_BP_2_5GBPS },
214 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I354_SGMII },
215 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_82567V_3 },
216 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IFE },
217 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IFE_G },
218 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IFE_GT },
219 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_AMT },
220 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_C },
221 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_M },
222 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_M_AMT },
223 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_BM },
224 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IFE },
225 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IFE_G },
226 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IFE_GT },
227 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_AMT },
228 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_C },
229 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_M },
230 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_M_AMT },
231 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_M_V },
232 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_D_BM_LF },
233 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_D_BM_LM },
234 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_D_BM_V },
235 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_R_BM_LF },
236 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_R_BM_LM },
237 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_R_BM_V },
238 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_1 },
239 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_2 },
240 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_3 },
241 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_4 },
242 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_5 },
243 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_6 }
244 };
245
246 /*********************************************************************
247 * Function prototypes
248 *********************************************************************/
249 int em_probe(struct device *, void *, void *);
250 void em_attach(struct device *, struct device *, void *);
251 void em_defer_attach(struct device*);
252 int em_detach(struct device *, int);
253 int em_activate(struct device *, int);
254 int em_intr(void *);
255 int em_allocate_legacy(struct em_softc *);
256 void em_start(struct ifqueue *);
257 int em_ioctl(struct ifnet *, u_long, caddr_t);
258 void em_watchdog(struct ifnet *);
259 void em_init(void *);
260 void em_stop(void *, int);
261 void em_media_status(struct ifnet *, struct ifmediareq *);
262 int em_media_change(struct ifnet *);
263 uint64_t em_flowstatus(struct em_softc *);
264 void em_identify_hardware(struct em_softc *);
265 int em_allocate_pci_resources(struct em_softc *);
266 void em_free_pci_resources(struct em_softc *);
267 void em_local_timer(void *);
268 int em_hardware_init(struct em_softc *);
269 void em_setup_interface(struct em_softc *);
270 int em_setup_transmit_structures(struct em_softc *);
271 void em_initialize_transmit_unit(struct em_softc *);
272 int em_setup_receive_structures(struct em_softc *);
273 void em_initialize_receive_unit(struct em_softc *);
274 void em_enable_intr(struct em_softc *);
275 void em_disable_intr(struct em_softc *);
276 void em_free_transmit_structures(struct em_softc *);
277 void em_free_receive_structures(struct em_softc *);
278 void em_disable_aspm(struct em_softc *);
279 void em_txeof(struct em_queue *);
280 int em_allocate_receive_structures(struct em_softc *);
281 int em_allocate_transmit_structures(struct em_softc *);
282 int em_allocate_desc_rings(struct em_softc *);
283 int em_rxfill(struct em_queue *);
284 void em_rxrefill(void *);
285 void em_rxrefill_locked(struct em_queue *);
286 int em_rxeof(struct em_queue *);
287 void em_receive_checksum(struct em_softc *, struct em_rx_desc *,
288 struct mbuf *);
289 u_int em_transmit_checksum_setup(struct em_queue *, struct mbuf *, u_int,
290 u_int32_t *, u_int32_t *);
291 u_int em_tso_setup(struct em_queue *, struct mbuf *, u_int, u_int32_t *,
292 u_int32_t *);
293 u_int em_tx_ctx_setup(struct em_queue *, struct mbuf *, u_int, u_int32_t *,
294 u_int32_t *);
295 void em_iff(struct em_softc *);
296 void em_update_link_status(struct em_softc *);
297 int em_get_buf(struct em_queue *, int);
298 void em_enable_hw_vlans(struct em_softc *);
299 u_int em_encap(struct em_queue *, struct mbuf *);
300 void em_smartspeed(struct em_softc *);
301 int em_82547_fifo_workaround(struct em_softc *, int);
302 void em_82547_update_fifo_head(struct em_softc *, int);
303 int em_82547_tx_fifo_reset(struct em_softc *);
304 void em_82547_move_tail(void *arg);
305 void em_82547_move_tail_locked(struct em_softc *);
306 int em_dma_malloc(struct em_softc *, bus_size_t, struct em_dma_alloc *);
307 void em_dma_free(struct em_softc *, struct em_dma_alloc *);
308 u_int32_t em_fill_descriptors(u_int64_t address, u_int32_t length,
309 PDESC_ARRAY desc_array);
310 void em_flush_tx_ring(struct em_queue *);
311 void em_flush_rx_ring(struct em_queue *);
312 void em_flush_desc_rings(struct em_softc *);
313 int em_get_sffpage(struct em_softc *, struct if_sffpage *);
314
315 #ifndef SMALL_KERNEL
316 /* MSIX/Multiqueue functions */
317 int em_allocate_msix(struct em_softc *);
318 int em_setup_queues_msix(struct em_softc *);
319 int em_queue_intr_msix(void *);
320 int em_link_intr_msix(void *);
321 void em_enable_queue_intr_msix(struct em_queue *);
322 #else
323 #define em_allocate_msix(_sc) (-1)
324 #endif
325
326 #if NKSTAT > 0
327 void em_kstat_attach(struct em_softc *);
328 int em_kstat_read(struct kstat *);
329 void em_tbi_adjust_stats(struct em_softc *, uint32_t, uint8_t *);
330 #endif
331
332 /*********************************************************************
333 * OpenBSD Device Interface Entry Points
334 *********************************************************************/
335
336 const struct cfattach em_ca = {
337 sizeof(struct em_softc), em_probe, em_attach, em_detach,
338 em_activate
339 };
340
341 struct cfdriver em_cd = {
342 NULL, "em", DV_IFNET
343 };
344
345 static int em_smart_pwr_down = FALSE;
346 int em_enable_msix = 0;
347
348 /*********************************************************************
349 * Device identification routine
350 *
351 * em_probe determines if the driver should be loaded on
352 * adapter based on PCI vendor/device id of the adapter.
353 *
354 * return 0 on no match, positive on match
355 *********************************************************************/
356
357 int
em_probe(struct device * parent,void * match,void * aux)358 em_probe(struct device *parent, void *match, void *aux)
359 {
360 INIT_DEBUGOUT("em_probe: begin");
361
362 return (pci_matchbyid((struct pci_attach_args *)aux, em_devices,
363 nitems(em_devices)));
364 }
365
366 void
em_defer_attach(struct device * self)367 em_defer_attach(struct device *self)
368 {
369 struct em_softc *sc = (struct em_softc *)self;
370 struct pci_attach_args *pa = &sc->osdep.em_pa;
371 pci_chipset_tag_t pc = pa->pa_pc;
372 void *gcu;
373
374 INIT_DEBUGOUT("em_defer_attach: begin");
375
376 if ((gcu = em_lookup_gcu(self)) == 0) {
377 printf("%s: No GCU found, deferred attachment failed\n",
378 DEVNAME(sc));
379
380 if (sc->sc_intrhand)
381 pci_intr_disestablish(pc, sc->sc_intrhand);
382 sc->sc_intrhand = 0;
383
384 em_stop(sc, 1);
385
386 em_free_pci_resources(sc);
387
388 return;
389 }
390
391 sc->hw.gcu = gcu;
392
393 em_attach_miibus(self);
394
395 em_setup_interface(sc);
396
397 em_setup_link(&sc->hw);
398
399 em_update_link_status(sc);
400 }
401
402 /*********************************************************************
403 * Device initialization routine
404 *
405 * The attach entry point is called when the driver is being loaded.
406 * This routine identifies the type of hardware, allocates all resources
407 * and initializes the hardware.
408 *
409 *********************************************************************/
410
411 void
em_attach(struct device * parent,struct device * self,void * aux)412 em_attach(struct device *parent, struct device *self, void *aux)
413 {
414 struct pci_attach_args *pa = aux;
415 struct em_softc *sc;
416 int defer = 0;
417
418 INIT_DEBUGOUT("em_attach: begin");
419
420 sc = (struct em_softc *)self;
421 sc->sc_dmat = pa->pa_dmat;
422 sc->osdep.em_pa = *pa;
423
424 timeout_set(&sc->timer_handle, em_local_timer, sc);
425 timeout_set(&sc->tx_fifo_timer_handle, em_82547_move_tail, sc);
426
427 rw_init(&sc->sfflock, "emsff");
428
429 /* Determine hardware revision */
430 em_identify_hardware(sc);
431
432 /*
433 * Only use MSI on the newer PCIe parts, with the exception
434 * of 82571/82572 due to "Byte Enables 2 and 3 Are Not Set" errata
435 */
436 if (sc->hw.mac_type <= em_82572)
437 sc->osdep.em_pa.pa_flags &= ~PCI_FLAGS_MSI_ENABLED;
438
439 /* Parameters (to be read from user) */
440 if (sc->hw.mac_type >= em_82544) {
441 sc->sc_tx_slots = EM_MAX_TXD;
442 sc->sc_rx_slots = EM_MAX_RXD;
443 } else {
444 sc->sc_tx_slots = EM_MAX_TXD_82543;
445 sc->sc_rx_slots = EM_MAX_RXD_82543;
446 }
447 sc->tx_int_delay = EM_TIDV;
448 sc->tx_abs_int_delay = EM_TADV;
449 sc->rx_int_delay = EM_RDTR;
450 sc->rx_abs_int_delay = EM_RADV;
451 sc->hw.autoneg = DO_AUTO_NEG;
452 sc->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
453 sc->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
454 sc->hw.tbi_compatibility_en = TRUE;
455 sc->sc_rx_buffer_len = EM_RXBUFFER_2048;
456
457 sc->hw.phy_init_script = 1;
458 sc->hw.phy_reset_disable = FALSE;
459
460 #ifndef EM_MASTER_SLAVE
461 sc->hw.master_slave = em_ms_hw_default;
462 #else
463 sc->hw.master_slave = EM_MASTER_SLAVE;
464 #endif
465
466 /*
467 * This controls when hardware reports transmit completion
468 * status.
469 */
470 sc->hw.report_tx_early = 1;
471
472 if (em_allocate_pci_resources(sc))
473 goto err_pci;
474
475 /* Initialize eeprom parameters */
476 em_init_eeprom_params(&sc->hw);
477
478 /*
479 * Set the max frame size assuming standard Ethernet
480 * sized frames.
481 */
482 switch (sc->hw.mac_type) {
483 case em_82573:
484 {
485 uint16_t eeprom_data = 0;
486
487 /*
488 * 82573 only supports Jumbo frames
489 * if ASPM is disabled.
490 */
491 em_read_eeprom(&sc->hw, EEPROM_INIT_3GIO_3,
492 1, &eeprom_data);
493 if (eeprom_data & EEPROM_WORD1A_ASPM_MASK) {
494 sc->hw.max_frame_size = ETHER_MAX_LEN;
495 break;
496 }
497 /* Allow Jumbo frames */
498 /* FALLTHROUGH */
499 }
500 case em_82571:
501 case em_82572:
502 case em_82574:
503 case em_82575:
504 case em_82576:
505 case em_82580:
506 case em_i210:
507 case em_i350:
508 case em_ich9lan:
509 case em_ich10lan:
510 case em_pch2lan:
511 case em_pch_lpt:
512 case em_pch_spt:
513 case em_pch_cnp:
514 case em_pch_tgp:
515 case em_pch_adp:
516 case em_80003es2lan:
517 /* 9K Jumbo Frame size */
518 sc->hw.max_frame_size = 9234;
519 break;
520 case em_pchlan:
521 sc->hw.max_frame_size = 4096;
522 break;
523 case em_82542_rev2_0:
524 case em_82542_rev2_1:
525 case em_ich8lan:
526 /* Adapters that do not support Jumbo frames */
527 sc->hw.max_frame_size = ETHER_MAX_LEN;
528 break;
529 default:
530 sc->hw.max_frame_size =
531 MAX_JUMBO_FRAME_SIZE;
532 }
533
534 sc->hw.min_frame_size =
535 ETHER_MIN_LEN + ETHER_CRC_LEN;
536
537 if (em_allocate_desc_rings(sc) != 0) {
538 printf("%s: Unable to allocate descriptor ring memory\n",
539 DEVNAME(sc));
540 goto err_pci;
541 }
542
543 /* Initialize the hardware */
544 if ((defer = em_hardware_init(sc))) {
545 if (defer == EAGAIN)
546 config_defer(self, em_defer_attach);
547 else {
548 printf("%s: Unable to initialize the hardware\n",
549 DEVNAME(sc));
550 goto err_pci;
551 }
552 }
553
554 if (sc->hw.mac_type == em_80003es2lan || sc->hw.mac_type == em_82575 ||
555 sc->hw.mac_type == em_82576 ||
556 sc->hw.mac_type == em_82580 || sc->hw.mac_type == em_i210 ||
557 sc->hw.mac_type == em_i350) {
558 uint32_t reg = EM_READ_REG(&sc->hw, E1000_STATUS);
559 sc->hw.bus_func = (reg & E1000_STATUS_FUNC_MASK) >>
560 E1000_STATUS_FUNC_SHIFT;
561
562 switch (sc->hw.bus_func) {
563 case 0:
564 sc->hw.swfw = E1000_SWFW_PHY0_SM;
565 break;
566 case 1:
567 sc->hw.swfw = E1000_SWFW_PHY1_SM;
568 break;
569 case 2:
570 sc->hw.swfw = E1000_SWFW_PHY2_SM;
571 break;
572 case 3:
573 sc->hw.swfw = E1000_SWFW_PHY3_SM;
574 break;
575 }
576 } else {
577 sc->hw.bus_func = 0;
578 }
579
580 /* Copy the permanent MAC address out of the EEPROM */
581 if (em_read_mac_addr(&sc->hw) < 0) {
582 printf("%s: EEPROM read error while reading mac address\n",
583 DEVNAME(sc));
584 goto err_pci;
585 }
586
587 bcopy(sc->hw.mac_addr, sc->sc_ac.ac_enaddr, ETHER_ADDR_LEN);
588
589 /* Setup OS specific network interface */
590 if (!defer)
591 em_setup_interface(sc);
592
593 /* Initialize statistics */
594 em_clear_hw_cntrs(&sc->hw);
595 #if NKSTAT > 0
596 em_kstat_attach(sc);
597 #endif
598 sc->hw.get_link_status = 1;
599 if (!defer)
600 em_update_link_status(sc);
601
602 #ifdef EM_DEBUG
603 printf(", mac %#x phy %#x", sc->hw.mac_type, sc->hw.phy_type);
604 #endif
605 printf(", address %s\n", ether_sprintf(sc->sc_ac.ac_enaddr));
606
607 /* Indicate SOL/IDER usage */
608 if (em_check_phy_reset_block(&sc->hw))
609 printf("%s: PHY reset is blocked due to SOL/IDER session.\n",
610 DEVNAME(sc));
611
612 /* Identify 82544 on PCI-X */
613 em_get_bus_info(&sc->hw);
614 if (sc->hw.bus_type == em_bus_type_pcix &&
615 sc->hw.mac_type == em_82544)
616 sc->pcix_82544 = TRUE;
617 else
618 sc->pcix_82544 = FALSE;
619
620 sc->hw.icp_xxxx_is_link_up = FALSE;
621
622 INIT_DEBUGOUT("em_attach: end");
623 return;
624
625 err_pci:
626 em_free_pci_resources(sc);
627 }
628
629 /*********************************************************************
630 * Transmit entry point
631 *
632 * em_start is called by the stack to initiate a transmit.
633 * The driver will remain in this routine as long as there are
634 * packets to transmit and transmit resources are available.
635 * In case resources are not available stack is notified and
636 * the packet is requeued.
637 **********************************************************************/
638
639 void
em_start(struct ifqueue * ifq)640 em_start(struct ifqueue *ifq)
641 {
642 struct ifnet *ifp = ifq->ifq_if;
643 struct em_softc *sc = ifp->if_softc;
644 u_int head, free, used;
645 struct mbuf *m;
646 int post = 0;
647 struct em_queue *que = sc->queues; /* Use only first queue. */
648
649 if (!sc->link_active) {
650 ifq_purge(ifq);
651 return;
652 }
653
654 /* calculate free space */
655 head = que->tx.sc_tx_desc_head;
656 free = que->tx.sc_tx_desc_tail;
657 if (free <= head)
658 free += sc->sc_tx_slots;
659 free -= head;
660
661 if (sc->hw.mac_type != em_82547) {
662 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
663 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
664 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
665 }
666
667 for (;;) {
668 /* use 2 because cksum setup can use an extra slot */
669 if (EM_MAX_SCATTER + 2 > free) {
670 ifq_set_oactive(ifq);
671 break;
672 }
673
674 m = ifq_dequeue(ifq);
675 if (m == NULL)
676 break;
677
678 used = em_encap(que, m);
679 if (used == 0) {
680 m_freem(m);
681 continue;
682 }
683
684 KASSERT(used <= free);
685
686 free -= used;
687
688 #if NBPFILTER > 0
689 /* Send a copy of the frame to the BPF listener */
690 if (ifp->if_bpf)
691 bpf_mtap_ether(ifp->if_bpf, m, BPF_DIRECTION_OUT);
692 #endif
693
694 /* Set timeout in case hardware has problems transmitting */
695 ifp->if_timer = EM_TX_TIMEOUT;
696
697 if (sc->hw.mac_type == em_82547) {
698 int len = m->m_pkthdr.len;
699
700 if (sc->link_duplex == HALF_DUPLEX)
701 em_82547_move_tail_locked(sc);
702 else {
703 E1000_WRITE_REG(&sc->hw, TDT(que->me),
704 que->tx.sc_tx_desc_head);
705 em_82547_update_fifo_head(sc, len);
706 }
707 }
708
709 post = 1;
710 }
711
712 if (sc->hw.mac_type != em_82547) {
713 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
714 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
715 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
716 /*
717 * Advance the Transmit Descriptor Tail (Tdt),
718 * this tells the E1000 that this frame is
719 * available to transmit.
720 */
721 if (post)
722 E1000_WRITE_REG(&sc->hw, TDT(que->me),
723 que->tx.sc_tx_desc_head);
724 }
725 }
726
727 /*********************************************************************
728 * Ioctl entry point
729 *
730 * em_ioctl is called when the user wants to configure the
731 * interface.
732 *
733 * return 0 on success, positive on failure
734 **********************************************************************/
735
736 int
em_ioctl(struct ifnet * ifp,u_long command,caddr_t data)737 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
738 {
739 int error = 0;
740 struct ifreq *ifr = (struct ifreq *) data;
741 struct em_softc *sc = ifp->if_softc;
742 int s;
743
744 s = splnet();
745
746 switch (command) {
747 case SIOCSIFADDR:
748 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFADDR (Set Interface "
749 "Addr)");
750 if (!(ifp->if_flags & IFF_UP)) {
751 ifp->if_flags |= IFF_UP;
752 em_init(sc);
753 }
754 break;
755
756 case SIOCSIFFLAGS:
757 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
758 if (ifp->if_flags & IFF_UP) {
759 if (ifp->if_flags & IFF_RUNNING)
760 error = ENETRESET;
761 else
762 em_init(sc);
763 } else {
764 if (ifp->if_flags & IFF_RUNNING)
765 em_stop(sc, 0);
766 }
767 break;
768
769 case SIOCSIFMEDIA:
770 /* Check SOL/IDER usage */
771 if (em_check_phy_reset_block(&sc->hw)) {
772 printf("%s: Media change is blocked due to SOL/IDER session.\n",
773 DEVNAME(sc));
774 break;
775 }
776 case SIOCGIFMEDIA:
777 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
778 error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
779 break;
780
781 case SIOCGIFRXR:
782 error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
783 NULL, EM_MCLBYTES, &sc->queues->rx.sc_rx_ring);
784 break;
785
786 case SIOCGIFSFFPAGE:
787 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR);
788 if (error != 0)
789 break;
790
791 error = em_get_sffpage(sc, (struct if_sffpage *)data);
792 rw_exit(&sc->sfflock);
793 break;
794
795 default:
796 error = ether_ioctl(ifp, &sc->sc_ac, command, data);
797 }
798
799 if (error == ENETRESET) {
800 if (ifp->if_flags & IFF_RUNNING) {
801 em_disable_intr(sc);
802 em_iff(sc);
803 if (sc->hw.mac_type == em_82542_rev2_0)
804 em_initialize_receive_unit(sc);
805 em_enable_intr(sc);
806 }
807 error = 0;
808 }
809
810 splx(s);
811 return (error);
812 }
813
814 /*********************************************************************
815 * Watchdog entry point
816 *
817 * This routine is called whenever hardware quits transmitting.
818 *
819 **********************************************************************/
820
821 void
em_watchdog(struct ifnet * ifp)822 em_watchdog(struct ifnet *ifp)
823 {
824 struct em_softc *sc = ifp->if_softc;
825 struct em_queue *que = sc->queues; /* Use only first queue. */
826
827
828 /* If we are in this routine because of pause frames, then
829 * don't reset the hardware.
830 */
831 if (E1000_READ_REG(&sc->hw, STATUS) & E1000_STATUS_TXOFF) {
832 ifp->if_timer = EM_TX_TIMEOUT;
833 return;
834 }
835 printf("%s: watchdog: head %u tail %u TDH %u TDT %u\n",
836 DEVNAME(sc),
837 que->tx.sc_tx_desc_head, que->tx.sc_tx_desc_tail,
838 E1000_READ_REG(&sc->hw, TDH(que->me)),
839 E1000_READ_REG(&sc->hw, TDT(que->me)));
840
841 em_init(sc);
842
843 sc->watchdog_events++;
844 }
845
846 /*********************************************************************
847 * Init entry point
848 *
849 * This routine is used in two ways. It is used by the stack as
850 * init entry point in network interface structure. It is also used
851 * by the driver as a hw/sw initialization routine to get to a
852 * consistent state.
853 *
854 **********************************************************************/
855
856 void
em_init(void * arg)857 em_init(void *arg)
858 {
859 struct em_softc *sc = arg;
860 struct ifnet *ifp = &sc->sc_ac.ac_if;
861 uint32_t pba;
862 int s;
863
864 s = splnet();
865
866 INIT_DEBUGOUT("em_init: begin");
867
868 em_stop(sc, 0);
869
870 /*
871 * Packet Buffer Allocation (PBA)
872 * Writing PBA sets the receive portion of the buffer
873 * the remainder is used for the transmit buffer.
874 *
875 * Devices before the 82547 had a Packet Buffer of 64K.
876 * Default allocation: PBA=48K for Rx, leaving 16K for Tx.
877 * After the 82547 the buffer was reduced to 40K.
878 * Default allocation: PBA=30K for Rx, leaving 10K for Tx.
879 * Note: default does not leave enough room for Jumbo Frame >10k.
880 */
881 switch (sc->hw.mac_type) {
882 case em_82547:
883 case em_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
884 if (sc->hw.max_frame_size > EM_RXBUFFER_8192)
885 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
886 else
887 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
888 sc->tx_fifo_head = 0;
889 sc->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
890 sc->tx_fifo_size = (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
891 break;
892 case em_82571:
893 case em_82572: /* Total Packet Buffer on these is 48k */
894 case em_82575:
895 case em_82576:
896 case em_82580:
897 case em_80003es2lan:
898 case em_i350:
899 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
900 break;
901 case em_i210:
902 pba = E1000_PBA_34K;
903 break;
904 case em_82573: /* 82573: Total Packet Buffer is 32K */
905 /* Jumbo frames not supported */
906 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
907 break;
908 case em_82574: /* Total Packet Buffer is 40k */
909 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
910 break;
911 case em_ich8lan:
912 pba = E1000_PBA_8K;
913 break;
914 case em_ich9lan:
915 case em_ich10lan:
916 /* Boost Receive side for jumbo frames */
917 if (sc->hw.max_frame_size > EM_RXBUFFER_4096)
918 pba = E1000_PBA_14K;
919 else
920 pba = E1000_PBA_10K;
921 break;
922 case em_pchlan:
923 case em_pch2lan:
924 case em_pch_lpt:
925 case em_pch_spt:
926 case em_pch_cnp:
927 case em_pch_tgp:
928 case em_pch_adp:
929 pba = E1000_PBA_26K;
930 break;
931 default:
932 /* Devices before 82547 had a Packet Buffer of 64K. */
933 if (sc->hw.max_frame_size > EM_RXBUFFER_8192)
934 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
935 else
936 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
937 }
938 INIT_DEBUGOUT1("em_init: pba=%dK",pba);
939 E1000_WRITE_REG(&sc->hw, PBA, pba);
940
941 /* Get the latest mac address, User can use a LAA */
942 bcopy(sc->sc_ac.ac_enaddr, sc->hw.mac_addr, ETHER_ADDR_LEN);
943
944 /* Initialize the hardware */
945 if (em_hardware_init(sc)) {
946 printf("%s: Unable to initialize the hardware\n",
947 DEVNAME(sc));
948 splx(s);
949 return;
950 }
951 em_update_link_status(sc);
952
953 E1000_WRITE_REG(&sc->hw, VET, ETHERTYPE_VLAN);
954 if (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING)
955 em_enable_hw_vlans(sc);
956
957 /* Prepare transmit descriptors and buffers */
958 if (em_setup_transmit_structures(sc)) {
959 printf("%s: Could not setup transmit structures\n",
960 DEVNAME(sc));
961 em_stop(sc, 0);
962 splx(s);
963 return;
964 }
965 em_initialize_transmit_unit(sc);
966
967 /* Prepare receive descriptors and buffers */
968 if (em_setup_receive_structures(sc)) {
969 printf("%s: Could not setup receive structures\n",
970 DEVNAME(sc));
971 em_stop(sc, 0);
972 splx(s);
973 return;
974 }
975 em_initialize_receive_unit(sc);
976
977 #ifndef SMALL_KERNEL
978 if (sc->msix) {
979 if (em_setup_queues_msix(sc)) {
980 printf("%s: Can't setup msix queues\n", DEVNAME(sc));
981 splx(s);
982 return;
983 }
984 }
985 #endif
986
987 /* Program promiscuous mode and multicast filters. */
988 em_iff(sc);
989
990 ifp->if_flags |= IFF_RUNNING;
991 ifq_clr_oactive(&ifp->if_snd);
992
993 timeout_add_sec(&sc->timer_handle, 1);
994 em_clear_hw_cntrs(&sc->hw);
995 em_enable_intr(sc);
996
997 /* Don't reset the phy next time init gets called */
998 sc->hw.phy_reset_disable = TRUE;
999
1000 splx(s);
1001 }
1002
1003 /*********************************************************************
1004 *
1005 * Interrupt Service routine
1006 *
1007 **********************************************************************/
1008 int
em_intr(void * arg)1009 em_intr(void *arg)
1010 {
1011 struct em_softc *sc = arg;
1012 struct em_queue *que = sc->queues; /* single queue */
1013 struct ifnet *ifp = &sc->sc_ac.ac_if;
1014 u_int32_t reg_icr, test_icr;
1015
1016 test_icr = reg_icr = E1000_READ_REG(&sc->hw, ICR);
1017 if (sc->hw.mac_type >= em_82571)
1018 test_icr = (reg_icr & E1000_ICR_INT_ASSERTED);
1019 if (!test_icr)
1020 return (0);
1021
1022 if (ifp->if_flags & IFF_RUNNING) {
1023 em_txeof(que);
1024 if (em_rxeof(que))
1025 em_rxrefill_locked(que);
1026 }
1027
1028 /* Link status change */
1029 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1030 KERNEL_LOCK();
1031 sc->hw.get_link_status = 1;
1032 em_check_for_link(&sc->hw);
1033 em_update_link_status(sc);
1034 KERNEL_UNLOCK();
1035 }
1036
1037 return (1);
1038 }
1039
1040 /*********************************************************************
1041 *
1042 * Media Ioctl callback
1043 *
1044 * This routine is called whenever the user queries the status of
1045 * the interface using ifconfig.
1046 *
1047 **********************************************************************/
1048 void
em_media_status(struct ifnet * ifp,struct ifmediareq * ifmr)1049 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1050 {
1051 struct em_softc *sc = ifp->if_softc;
1052 uint64_t fiber_type = IFM_1000_SX;
1053 u_int16_t gsr;
1054
1055 INIT_DEBUGOUT("em_media_status: begin");
1056
1057 em_check_for_link(&sc->hw);
1058 em_update_link_status(sc);
1059
1060 ifmr->ifm_status = IFM_AVALID;
1061 ifmr->ifm_active = IFM_ETHER;
1062
1063 if (!sc->link_active) {
1064 ifmr->ifm_active |= IFM_NONE;
1065 return;
1066 }
1067
1068 ifmr->ifm_status |= IFM_ACTIVE;
1069
1070 if (sc->hw.media_type == em_media_type_fiber ||
1071 sc->hw.media_type == em_media_type_internal_serdes) {
1072 if (sc->hw.mac_type == em_82545)
1073 fiber_type = IFM_1000_LX;
1074 ifmr->ifm_active |= fiber_type | IFM_FDX;
1075 } else {
1076 switch (sc->link_speed) {
1077 case 10:
1078 ifmr->ifm_active |= IFM_10_T;
1079 break;
1080 case 100:
1081 ifmr->ifm_active |= IFM_100_TX;
1082 break;
1083 case 1000:
1084 ifmr->ifm_active |= IFM_1000_T;
1085 break;
1086 }
1087
1088 if (sc->link_duplex == FULL_DUPLEX)
1089 ifmr->ifm_active |= em_flowstatus(sc) | IFM_FDX;
1090 else
1091 ifmr->ifm_active |= IFM_HDX;
1092
1093 if (IFM_SUBTYPE(ifmr->ifm_active) == IFM_1000_T) {
1094 em_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &gsr);
1095 if (gsr & SR_1000T_MS_CONFIG_RES)
1096 ifmr->ifm_active |= IFM_ETH_MASTER;
1097 }
1098 }
1099 }
1100
1101 /*********************************************************************
1102 *
1103 * Media Ioctl callback
1104 *
1105 * This routine is called when the user changes speed/duplex using
1106 * media/mediopt option with ifconfig.
1107 *
1108 **********************************************************************/
1109 int
em_media_change(struct ifnet * ifp)1110 em_media_change(struct ifnet *ifp)
1111 {
1112 struct em_softc *sc = ifp->if_softc;
1113 struct ifmedia *ifm = &sc->media;
1114
1115 INIT_DEBUGOUT("em_media_change: begin");
1116
1117 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1118 return (EINVAL);
1119
1120 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1121 case IFM_AUTO:
1122 sc->hw.autoneg = DO_AUTO_NEG;
1123 sc->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1124 break;
1125 case IFM_1000_LX:
1126 case IFM_1000_SX:
1127 case IFM_1000_T:
1128 sc->hw.autoneg = DO_AUTO_NEG;
1129 sc->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1130 break;
1131 case IFM_100_TX:
1132 sc->hw.autoneg = FALSE;
1133 sc->hw.autoneg_advertised = 0;
1134 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1135 sc->hw.forced_speed_duplex = em_100_full;
1136 else
1137 sc->hw.forced_speed_duplex = em_100_half;
1138 break;
1139 case IFM_10_T:
1140 sc->hw.autoneg = FALSE;
1141 sc->hw.autoneg_advertised = 0;
1142 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1143 sc->hw.forced_speed_duplex = em_10_full;
1144 else
1145 sc->hw.forced_speed_duplex = em_10_half;
1146 break;
1147 default:
1148 printf("%s: Unsupported media type\n", DEVNAME(sc));
1149 }
1150
1151 /*
1152 * As the speed/duplex settings may have changed we need to
1153 * reset the PHY.
1154 */
1155 sc->hw.phy_reset_disable = FALSE;
1156
1157 em_init(sc);
1158
1159 return (0);
1160 }
1161
1162 uint64_t
em_flowstatus(struct em_softc * sc)1163 em_flowstatus(struct em_softc *sc)
1164 {
1165 u_int16_t ar, lpar;
1166
1167 if (sc->hw.media_type == em_media_type_fiber ||
1168 sc->hw.media_type == em_media_type_internal_serdes)
1169 return (0);
1170
1171 em_read_phy_reg(&sc->hw, PHY_AUTONEG_ADV, &ar);
1172 em_read_phy_reg(&sc->hw, PHY_LP_ABILITY, &lpar);
1173
1174 if ((ar & NWAY_AR_PAUSE) && (lpar & NWAY_LPAR_PAUSE))
1175 return (IFM_FLOW|IFM_ETH_TXPAUSE|IFM_ETH_RXPAUSE);
1176 else if (!(ar & NWAY_AR_PAUSE) && (ar & NWAY_AR_ASM_DIR) &&
1177 (lpar & NWAY_LPAR_PAUSE) && (lpar & NWAY_LPAR_ASM_DIR))
1178 return (IFM_FLOW|IFM_ETH_TXPAUSE);
1179 else if ((ar & NWAY_AR_PAUSE) && (ar & NWAY_AR_ASM_DIR) &&
1180 !(lpar & NWAY_LPAR_PAUSE) && (lpar & NWAY_LPAR_ASM_DIR))
1181 return (IFM_FLOW|IFM_ETH_RXPAUSE);
1182
1183 return (0);
1184 }
1185
1186 /*********************************************************************
1187 *
1188 * This routine maps the mbufs to tx descriptors.
1189 *
1190 * return 0 on failure, positive on success
1191 **********************************************************************/
1192 u_int
em_encap(struct em_queue * que,struct mbuf * m)1193 em_encap(struct em_queue *que, struct mbuf *m)
1194 {
1195 struct em_softc *sc = que->sc;
1196 struct em_packet *pkt;
1197 struct em_tx_desc *desc;
1198 bus_dmamap_t map;
1199 u_int32_t txd_upper, txd_lower;
1200 u_int head, last, used = 0;
1201 int i, j;
1202
1203 /* For 82544 Workaround */
1204 DESC_ARRAY desc_array;
1205 u_int32_t array_elements;
1206
1207 /* get a dmamap for this packet from the next free slot */
1208 head = que->tx.sc_tx_desc_head;
1209 pkt = &que->tx.sc_tx_pkts_ring[head];
1210 map = pkt->pkt_map;
1211
1212 switch (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, BUS_DMA_NOWAIT)) {
1213 case 0:
1214 break;
1215 case EFBIG:
1216 if (m_defrag(m, M_DONTWAIT) == 0 &&
1217 bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
1218 BUS_DMA_NOWAIT) == 0)
1219 break;
1220
1221 /* FALLTHROUGH */
1222 default:
1223 sc->no_tx_dma_setup++;
1224 return (0);
1225 }
1226
1227 bus_dmamap_sync(sc->sc_dmat, map,
1228 0, map->dm_mapsize,
1229 BUS_DMASYNC_PREWRITE);
1230
1231 if (sc->hw.mac_type == em_82547) {
1232 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
1233 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
1234 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1235 }
1236
1237 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210) {
1238 if (ISSET(m->m_pkthdr.csum_flags, M_TCP_TSO)) {
1239 used += em_tso_setup(que, m, head, &txd_upper,
1240 &txd_lower);
1241 if (!used)
1242 return (used);
1243 } else {
1244 used += em_tx_ctx_setup(que, m, head, &txd_upper,
1245 &txd_lower);
1246 }
1247 } else if (sc->hw.mac_type >= em_82543) {
1248 used += em_transmit_checksum_setup(que, m, head,
1249 &txd_upper, &txd_lower);
1250 } else {
1251 txd_upper = txd_lower = 0;
1252 }
1253
1254 head += used;
1255 if (head >= sc->sc_tx_slots)
1256 head -= sc->sc_tx_slots;
1257
1258 for (i = 0; i < map->dm_nsegs; i++) {
1259 /* If sc is 82544 and on PCI-X bus */
1260 if (sc->pcix_82544) {
1261 /*
1262 * Check the Address and Length combination and
1263 * split the data accordingly
1264 */
1265 array_elements = em_fill_descriptors(
1266 map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len,
1267 &desc_array);
1268 for (j = 0; j < array_elements; j++) {
1269 desc = &que->tx.sc_tx_desc_ring[head];
1270
1271 desc->buffer_addr = htole64(
1272 desc_array.descriptor[j].address);
1273 desc->lower.data = htole32(
1274 (que->tx.sc_txd_cmd | txd_lower |
1275 (u_int16_t)desc_array.descriptor[j].length));
1276 desc->upper.data = htole32(txd_upper);
1277
1278 last = head;
1279 if (++head == sc->sc_tx_slots)
1280 head = 0;
1281
1282 used++;
1283 }
1284 } else {
1285 desc = &que->tx.sc_tx_desc_ring[head];
1286
1287 desc->buffer_addr = htole64(map->dm_segs[i].ds_addr);
1288 desc->lower.data = htole32(que->tx.sc_txd_cmd |
1289 txd_lower | map->dm_segs[i].ds_len);
1290 desc->upper.data = htole32(txd_upper);
1291
1292 last = head;
1293 if (++head == sc->sc_tx_slots)
1294 head = 0;
1295
1296 used++;
1297 }
1298 }
1299
1300 #if NVLAN > 0
1301 /* Find out if we are in VLAN mode */
1302 if (m->m_flags & M_VLANTAG && (sc->hw.mac_type < em_82575 ||
1303 sc->hw.mac_type > em_i210)) {
1304 /* Set the VLAN id */
1305 desc->upper.fields.special = htole16(m->m_pkthdr.ether_vtag);
1306
1307 /* Tell hardware to add tag */
1308 desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1309 }
1310 #endif
1311
1312 /* mark the packet with the mbuf and last desc slot */
1313 pkt->pkt_m = m;
1314 pkt->pkt_eop = last;
1315
1316 que->tx.sc_tx_desc_head = head;
1317
1318 /*
1319 * Last Descriptor of Packet
1320 * needs End Of Packet (EOP)
1321 * and Report Status (RS)
1322 */
1323 desc->lower.data |= htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1324
1325 if (sc->hw.mac_type == em_82547) {
1326 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
1327 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
1328 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1329 }
1330
1331 return (used);
1332 }
1333
1334 /*********************************************************************
1335 *
1336 * 82547 workaround to avoid controller hang in half-duplex environment.
1337 * The workaround is to avoid queuing a large packet that would span
1338 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1339 * in this case. We do that only when FIFO is quiescent.
1340 *
1341 **********************************************************************/
1342 void
em_82547_move_tail_locked(struct em_softc * sc)1343 em_82547_move_tail_locked(struct em_softc *sc)
1344 {
1345 uint16_t hw_tdt;
1346 uint16_t sw_tdt;
1347 struct em_tx_desc *tx_desc;
1348 uint16_t length = 0;
1349 boolean_t eop = 0;
1350 struct em_queue *que = sc->queues; /* single queue chip */
1351
1352 hw_tdt = E1000_READ_REG(&sc->hw, TDT(que->me));
1353 sw_tdt = que->tx.sc_tx_desc_head;
1354
1355 while (hw_tdt != sw_tdt) {
1356 tx_desc = &que->tx.sc_tx_desc_ring[hw_tdt];
1357 length += tx_desc->lower.flags.length;
1358 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1359 if (++hw_tdt == sc->sc_tx_slots)
1360 hw_tdt = 0;
1361
1362 if (eop) {
1363 if (em_82547_fifo_workaround(sc, length)) {
1364 sc->tx_fifo_wrk_cnt++;
1365 timeout_add(&sc->tx_fifo_timer_handle, 1);
1366 break;
1367 }
1368 E1000_WRITE_REG(&sc->hw, TDT(que->me), hw_tdt);
1369 em_82547_update_fifo_head(sc, length);
1370 length = 0;
1371 }
1372 }
1373 }
1374
1375 void
em_82547_move_tail(void * arg)1376 em_82547_move_tail(void *arg)
1377 {
1378 struct em_softc *sc = arg;
1379 int s;
1380
1381 s = splnet();
1382 em_82547_move_tail_locked(sc);
1383 splx(s);
1384 }
1385
1386 int
em_82547_fifo_workaround(struct em_softc * sc,int len)1387 em_82547_fifo_workaround(struct em_softc *sc, int len)
1388 {
1389 int fifo_space, fifo_pkt_len;
1390
1391 fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1392
1393 if (sc->link_duplex == HALF_DUPLEX) {
1394 fifo_space = sc->tx_fifo_size - sc->tx_fifo_head;
1395
1396 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1397 if (em_82547_tx_fifo_reset(sc))
1398 return (0);
1399 else
1400 return (1);
1401 }
1402 }
1403
1404 return (0);
1405 }
1406
1407 void
em_82547_update_fifo_head(struct em_softc * sc,int len)1408 em_82547_update_fifo_head(struct em_softc *sc, int len)
1409 {
1410 int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1411
1412 /* tx_fifo_head is always 16 byte aligned */
1413 sc->tx_fifo_head += fifo_pkt_len;
1414 if (sc->tx_fifo_head >= sc->tx_fifo_size)
1415 sc->tx_fifo_head -= sc->tx_fifo_size;
1416 }
1417
1418 int
em_82547_tx_fifo_reset(struct em_softc * sc)1419 em_82547_tx_fifo_reset(struct em_softc *sc)
1420 {
1421 uint32_t tctl;
1422 struct em_queue *que = sc->queues; /* single queue chip */
1423
1424 if ((E1000_READ_REG(&sc->hw, TDT(que->me)) ==
1425 E1000_READ_REG(&sc->hw, TDH(que->me))) &&
1426 (E1000_READ_REG(&sc->hw, TDFT) ==
1427 E1000_READ_REG(&sc->hw, TDFH)) &&
1428 (E1000_READ_REG(&sc->hw, TDFTS) ==
1429 E1000_READ_REG(&sc->hw, TDFHS)) &&
1430 (E1000_READ_REG(&sc->hw, TDFPC) == 0)) {
1431
1432 /* Disable TX unit */
1433 tctl = E1000_READ_REG(&sc->hw, TCTL);
1434 E1000_WRITE_REG(&sc->hw, TCTL, tctl & ~E1000_TCTL_EN);
1435
1436 /* Reset FIFO pointers */
1437 E1000_WRITE_REG(&sc->hw, TDFT, sc->tx_head_addr);
1438 E1000_WRITE_REG(&sc->hw, TDFH, sc->tx_head_addr);
1439 E1000_WRITE_REG(&sc->hw, TDFTS, sc->tx_head_addr);
1440 E1000_WRITE_REG(&sc->hw, TDFHS, sc->tx_head_addr);
1441
1442 /* Re-enable TX unit */
1443 E1000_WRITE_REG(&sc->hw, TCTL, tctl);
1444 E1000_WRITE_FLUSH(&sc->hw);
1445
1446 sc->tx_fifo_head = 0;
1447 sc->tx_fifo_reset_cnt++;
1448
1449 return (TRUE);
1450 } else
1451 return (FALSE);
1452 }
1453
1454 void
em_iff(struct em_softc * sc)1455 em_iff(struct em_softc *sc)
1456 {
1457 struct ifnet *ifp = &sc->sc_ac.ac_if;
1458 struct arpcom *ac = &sc->sc_ac;
1459 u_int32_t reg_rctl = 0;
1460 u_int8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1461 struct ether_multi *enm;
1462 struct ether_multistep step;
1463 int i = 0;
1464
1465 IOCTL_DEBUGOUT("em_iff: begin");
1466
1467 if (sc->hw.mac_type == em_82542_rev2_0) {
1468 reg_rctl = E1000_READ_REG(&sc->hw, RCTL);
1469 if (sc->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1470 em_pci_clear_mwi(&sc->hw);
1471 reg_rctl |= E1000_RCTL_RST;
1472 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
1473 msec_delay(5);
1474 }
1475
1476 reg_rctl = E1000_READ_REG(&sc->hw, RCTL);
1477 reg_rctl &= ~(E1000_RCTL_MPE | E1000_RCTL_UPE);
1478 ifp->if_flags &= ~IFF_ALLMULTI;
1479
1480 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 ||
1481 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) {
1482 ifp->if_flags |= IFF_ALLMULTI;
1483 reg_rctl |= E1000_RCTL_MPE;
1484 if (ifp->if_flags & IFF_PROMISC)
1485 reg_rctl |= E1000_RCTL_UPE;
1486 } else {
1487 ETHER_FIRST_MULTI(step, ac, enm);
1488 while (enm != NULL) {
1489 bcopy(enm->enm_addrlo, mta + i, ETH_LENGTH_OF_ADDRESS);
1490 i += ETH_LENGTH_OF_ADDRESS;
1491
1492 ETHER_NEXT_MULTI(step, enm);
1493 }
1494
1495 em_mc_addr_list_update(&sc->hw, mta, ac->ac_multicnt, 0);
1496 }
1497
1498 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
1499
1500 if (sc->hw.mac_type == em_82542_rev2_0) {
1501 reg_rctl = E1000_READ_REG(&sc->hw, RCTL);
1502 reg_rctl &= ~E1000_RCTL_RST;
1503 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
1504 msec_delay(5);
1505 if (sc->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1506 em_pci_set_mwi(&sc->hw);
1507 }
1508 }
1509
1510 /*********************************************************************
1511 * Timer routine
1512 *
1513 * This routine checks for link status and updates statistics.
1514 *
1515 **********************************************************************/
1516
1517 void
em_local_timer(void * arg)1518 em_local_timer(void *arg)
1519 {
1520 struct em_softc *sc = arg;
1521 int s;
1522
1523 timeout_add_sec(&sc->timer_handle, 1);
1524
1525 s = splnet();
1526 em_smartspeed(sc);
1527 splx(s);
1528
1529 #if NKSTAT > 0
1530 if (sc->kstat != NULL && mtx_enter_try(&sc->kstat_mtx)) {
1531 em_kstat_read(sc->kstat);
1532 mtx_leave(&sc->kstat_mtx);
1533 }
1534 #endif
1535 }
1536
1537 void
em_update_link_status(struct em_softc * sc)1538 em_update_link_status(struct em_softc *sc)
1539 {
1540 struct ifnet *ifp = &sc->sc_ac.ac_if;
1541 u_char link_state;
1542
1543 if (E1000_READ_REG(&sc->hw, STATUS) & E1000_STATUS_LU) {
1544 if (sc->link_active == 0) {
1545 em_get_speed_and_duplex(&sc->hw,
1546 &sc->link_speed,
1547 &sc->link_duplex);
1548 /* Check if we may set SPEED_MODE bit on PCI-E */
1549 if ((sc->link_speed == SPEED_1000) &&
1550 ((sc->hw.mac_type == em_82571) ||
1551 (sc->hw.mac_type == em_82572) ||
1552 (sc->hw.mac_type == em_82575) ||
1553 (sc->hw.mac_type == em_82576) ||
1554 (sc->hw.mac_type == em_82580))) {
1555 int tarc0;
1556
1557 tarc0 = E1000_READ_REG(&sc->hw, TARC0);
1558 tarc0 |= SPEED_MODE_BIT;
1559 E1000_WRITE_REG(&sc->hw, TARC0, tarc0);
1560 }
1561 sc->link_active = 1;
1562 sc->smartspeed = 0;
1563 ifp->if_baudrate = IF_Mbps(sc->link_speed);
1564 }
1565 link_state = (sc->link_duplex == FULL_DUPLEX) ?
1566 LINK_STATE_FULL_DUPLEX : LINK_STATE_HALF_DUPLEX;
1567 } else {
1568 if (sc->link_active == 1) {
1569 ifp->if_baudrate = sc->link_speed = 0;
1570 sc->link_duplex = 0;
1571 sc->link_active = 0;
1572 }
1573 link_state = LINK_STATE_DOWN;
1574 }
1575 if (ifp->if_link_state != link_state) {
1576 ifp->if_link_state = link_state;
1577 if_link_state_change(ifp);
1578 }
1579
1580 /* Disable TSO for 10/100 speeds to avoid some hardware issues */
1581 switch (sc->link_speed) {
1582 case SPEED_10:
1583 case SPEED_100:
1584 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210) {
1585 ifp->if_capabilities &= ~IFCAP_TSOv4;
1586 ifp->if_capabilities &= ~IFCAP_TSOv6;
1587 }
1588 break;
1589 case SPEED_1000:
1590 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210)
1591 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
1592 break;
1593 }
1594 }
1595
1596 /*********************************************************************
1597 *
1598 * This routine disables all traffic on the adapter by issuing a
1599 * global reset on the MAC and deallocates TX/RX buffers.
1600 *
1601 **********************************************************************/
1602
1603 void
em_stop(void * arg,int softonly)1604 em_stop(void *arg, int softonly)
1605 {
1606 struct em_softc *sc = arg;
1607 struct em_queue *que = sc->queues; /* Use only first queue. */
1608 struct ifnet *ifp = &sc->sc_ac.ac_if;
1609
1610 /* Tell the stack that the interface is no longer active */
1611 ifp->if_flags &= ~IFF_RUNNING;
1612
1613 INIT_DEBUGOUT("em_stop: begin");
1614
1615 timeout_del(&que->rx_refill);
1616 timeout_del(&sc->timer_handle);
1617 timeout_del(&sc->tx_fifo_timer_handle);
1618
1619 if (!softonly)
1620 em_disable_intr(sc);
1621 if (sc->hw.mac_type >= em_pch_spt)
1622 em_flush_desc_rings(sc);
1623 if (!softonly)
1624 em_reset_hw(&sc->hw);
1625
1626 intr_barrier(sc->sc_intrhand);
1627 ifq_barrier(&ifp->if_snd);
1628
1629 KASSERT((ifp->if_flags & IFF_RUNNING) == 0);
1630
1631 ifq_clr_oactive(&ifp->if_snd);
1632 ifp->if_timer = 0;
1633
1634 em_free_transmit_structures(sc);
1635 em_free_receive_structures(sc);
1636 }
1637
1638 /*********************************************************************
1639 *
1640 * Determine hardware revision.
1641 *
1642 **********************************************************************/
1643 void
em_identify_hardware(struct em_softc * sc)1644 em_identify_hardware(struct em_softc *sc)
1645 {
1646 u_int32_t reg;
1647 struct pci_attach_args *pa = &sc->osdep.em_pa;
1648
1649 /* Make sure our PCI config space has the necessary stuff set */
1650 sc->hw.pci_cmd_word = pci_conf_read(pa->pa_pc, pa->pa_tag,
1651 PCI_COMMAND_STATUS_REG);
1652
1653 /* Save off the information about this board */
1654 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id);
1655 sc->hw.device_id = PCI_PRODUCT(pa->pa_id);
1656
1657 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG);
1658 sc->hw.revision_id = PCI_REVISION(reg);
1659
1660 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
1661 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg);
1662 sc->hw.subsystem_id = PCI_PRODUCT(reg);
1663
1664 /* Identify the MAC */
1665 if (em_set_mac_type(&sc->hw))
1666 printf("%s: Unknown MAC Type\n", DEVNAME(sc));
1667
1668 if (sc->hw.mac_type == em_pchlan)
1669 sc->hw.revision_id = PCI_PRODUCT(pa->pa_id) & 0x0f;
1670
1671 if (sc->hw.mac_type == em_82541 ||
1672 sc->hw.mac_type == em_82541_rev_2 ||
1673 sc->hw.mac_type == em_82547 ||
1674 sc->hw.mac_type == em_82547_rev_2)
1675 sc->hw.phy_init_script = TRUE;
1676 }
1677
1678 void
em_legacy_irq_quirk_spt(struct em_softc * sc)1679 em_legacy_irq_quirk_spt(struct em_softc *sc)
1680 {
1681 uint32_t reg;
1682
1683 /* Legacy interrupt: SPT needs a quirk. */
1684 if (sc->hw.mac_type != em_pch_spt && sc->hw.mac_type != em_pch_cnp &&
1685 sc->hw.mac_type != em_pch_tgp && sc->hw.mac_type != em_pch_adp)
1686 return;
1687 if (sc->legacy_irq == 0)
1688 return;
1689
1690 reg = EM_READ_REG(&sc->hw, E1000_FEXTNVM7);
1691 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE;
1692 EM_WRITE_REG(&sc->hw, E1000_FEXTNVM7, reg);
1693
1694 reg = EM_READ_REG(&sc->hw, E1000_FEXTNVM9);
1695 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS |
1696 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS;
1697 EM_WRITE_REG(&sc->hw, E1000_FEXTNVM9, reg);
1698 }
1699
1700 int
em_allocate_pci_resources(struct em_softc * sc)1701 em_allocate_pci_resources(struct em_softc *sc)
1702 {
1703 int val, rid;
1704 struct pci_attach_args *pa = &sc->osdep.em_pa;
1705 struct em_queue *que = NULL;
1706
1707 val = pci_conf_read(pa->pa_pc, pa->pa_tag, EM_MMBA);
1708 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) {
1709 printf(": mmba is not mem space\n");
1710 return (ENXIO);
1711 }
1712 if (pci_mapreg_map(pa, EM_MMBA, PCI_MAPREG_MEM_TYPE(val), 0,
1713 &sc->osdep.mem_bus_space_tag, &sc->osdep.mem_bus_space_handle,
1714 &sc->osdep.em_membase, &sc->osdep.em_memsize, 0)) {
1715 printf(": cannot find mem space\n");
1716 return (ENXIO);
1717 }
1718
1719 switch (sc->hw.mac_type) {
1720 case em_82544:
1721 case em_82540:
1722 case em_82545:
1723 case em_82546:
1724 case em_82541:
1725 case em_82541_rev_2:
1726 /* Figure out where our I/O BAR is ? */
1727 for (rid = PCI_MAPREG_START; rid < PCI_MAPREG_END;) {
1728 val = pci_conf_read(pa->pa_pc, pa->pa_tag, rid);
1729 if (PCI_MAPREG_TYPE(val) == PCI_MAPREG_TYPE_IO) {
1730 sc->io_rid = rid;
1731 break;
1732 }
1733 rid += 4;
1734 if (PCI_MAPREG_MEM_TYPE(val) ==
1735 PCI_MAPREG_MEM_TYPE_64BIT)
1736 rid += 4; /* skip high bits, too */
1737 }
1738
1739 if (pci_mapreg_map(pa, rid, PCI_MAPREG_TYPE_IO, 0,
1740 &sc->osdep.io_bus_space_tag, &sc->osdep.io_bus_space_handle,
1741 &sc->osdep.em_iobase, &sc->osdep.em_iosize, 0)) {
1742 printf(": cannot find i/o space\n");
1743 return (ENXIO);
1744 }
1745
1746 sc->hw.io_base = 0;
1747 break;
1748 default:
1749 break;
1750 }
1751
1752 sc->osdep.em_flashoffset = 0;
1753 /* for ICH8 and family we need to find the flash memory */
1754 if (sc->hw.mac_type >= em_pch_spt) {
1755 sc->osdep.flash_bus_space_tag = sc->osdep.mem_bus_space_tag;
1756 sc->osdep.flash_bus_space_handle = sc->osdep.mem_bus_space_handle;
1757 sc->osdep.em_flashbase = 0;
1758 sc->osdep.em_flashsize = 0;
1759 sc->osdep.em_flashoffset = 0xe000;
1760 } else if (IS_ICH8(sc->hw.mac_type)) {
1761 val = pci_conf_read(pa->pa_pc, pa->pa_tag, EM_FLASH);
1762 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) {
1763 printf(": flash is not mem space\n");
1764 return (ENXIO);
1765 }
1766
1767 if (pci_mapreg_map(pa, EM_FLASH, PCI_MAPREG_MEM_TYPE(val), 0,
1768 &sc->osdep.flash_bus_space_tag, &sc->osdep.flash_bus_space_handle,
1769 &sc->osdep.em_flashbase, &sc->osdep.em_flashsize, 0)) {
1770 printf(": cannot find mem space\n");
1771 return (ENXIO);
1772 }
1773 }
1774
1775 sc->osdep.dev = (struct device *)sc;
1776 sc->hw.back = &sc->osdep;
1777
1778 /* Only one queue for the moment. */
1779 que = malloc(sizeof(struct em_queue), M_DEVBUF, M_NOWAIT | M_ZERO);
1780 if (que == NULL) {
1781 printf(": unable to allocate queue memory\n");
1782 return (ENOMEM);
1783 }
1784 que->me = 0;
1785 que->sc = sc;
1786 timeout_set(&que->rx_refill, em_rxrefill, que);
1787
1788 sc->queues = que;
1789 sc->num_queues = 1;
1790 sc->msix = 0;
1791 sc->legacy_irq = 0;
1792 if (em_allocate_msix(sc) && em_allocate_legacy(sc))
1793 return (ENXIO);
1794
1795 /*
1796 * the ICP_xxxx device has multiple, duplicate register sets for
1797 * use when it is being used as a network processor. Disable those
1798 * registers here, as they are not necessary in this context and
1799 * can confuse the system
1800 */
1801 if(sc->hw.mac_type == em_icp_xxxx) {
1802 int offset;
1803 pcireg_t val;
1804
1805 if (!pci_get_capability(sc->osdep.em_pa.pa_pc,
1806 sc->osdep.em_pa.pa_tag, PCI_CAP_ID_ST, &offset, &val)) {
1807 return (0);
1808 }
1809 offset += PCI_ST_SMIA_OFFSET;
1810 pci_conf_write(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
1811 offset, 0x06);
1812 E1000_WRITE_REG(&sc->hw, IMC1, ~0x0);
1813 E1000_WRITE_REG(&sc->hw, IMC2, ~0x0);
1814 }
1815 return (0);
1816 }
1817
1818 void
em_free_pci_resources(struct em_softc * sc)1819 em_free_pci_resources(struct em_softc *sc)
1820 {
1821 struct pci_attach_args *pa = &sc->osdep.em_pa;
1822 pci_chipset_tag_t pc = pa->pa_pc;
1823 struct em_queue *que = NULL;
1824 if (sc->sc_intrhand)
1825 pci_intr_disestablish(pc, sc->sc_intrhand);
1826 sc->sc_intrhand = 0;
1827
1828 if (sc->osdep.em_flashbase)
1829 bus_space_unmap(sc->osdep.flash_bus_space_tag, sc->osdep.flash_bus_space_handle,
1830 sc->osdep.em_flashsize);
1831 sc->osdep.em_flashbase = 0;
1832
1833 if (sc->osdep.em_iobase)
1834 bus_space_unmap(sc->osdep.io_bus_space_tag, sc->osdep.io_bus_space_handle,
1835 sc->osdep.em_iosize);
1836 sc->osdep.em_iobase = 0;
1837
1838 if (sc->osdep.em_membase)
1839 bus_space_unmap(sc->osdep.mem_bus_space_tag, sc->osdep.mem_bus_space_handle,
1840 sc->osdep.em_memsize);
1841 sc->osdep.em_membase = 0;
1842
1843 FOREACH_QUEUE(sc, que) {
1844 if (que->rx.sc_rx_desc_ring != NULL) {
1845 que->rx.sc_rx_desc_ring = NULL;
1846 em_dma_free(sc, &que->rx.sc_rx_dma);
1847 }
1848 if (que->tx.sc_tx_desc_ring != NULL) {
1849 que->tx.sc_tx_desc_ring = NULL;
1850 em_dma_free(sc, &que->tx.sc_tx_dma);
1851 }
1852 if (que->tag)
1853 pci_intr_disestablish(pc, que->tag);
1854 que->tag = NULL;
1855 que->eims = 0;
1856 que->me = 0;
1857 que->sc = NULL;
1858 }
1859 sc->legacy_irq = 0;
1860 sc->msix_linkvec = 0;
1861 sc->msix_queuesmask = 0;
1862 if (sc->queues)
1863 free(sc->queues, M_DEVBUF,
1864 sc->num_queues * sizeof(struct em_queue));
1865 sc->num_queues = 0;
1866 sc->queues = NULL;
1867 }
1868
1869 /*********************************************************************
1870 *
1871 * Initialize the hardware to a configuration as specified by the
1872 * em_softc structure. The controller is reset, the EEPROM is
1873 * verified, the MAC address is set, then the shared initialization
1874 * routines are called.
1875 *
1876 **********************************************************************/
1877 int
em_hardware_init(struct em_softc * sc)1878 em_hardware_init(struct em_softc *sc)
1879 {
1880 uint32_t ret_val;
1881 u_int16_t rx_buffer_size;
1882
1883 INIT_DEBUGOUT("em_hardware_init: begin");
1884 if (sc->hw.mac_type >= em_pch_spt)
1885 em_flush_desc_rings(sc);
1886 /* Issue a global reset */
1887 em_reset_hw(&sc->hw);
1888
1889 /* When hardware is reset, fifo_head is also reset */
1890 sc->tx_fifo_head = 0;
1891
1892 /* Make sure we have a good EEPROM before we read from it */
1893 if (em_get_flash_presence_i210(&sc->hw) &&
1894 em_validate_eeprom_checksum(&sc->hw) < 0) {
1895 /*
1896 * Some PCIe parts fail the first check due to
1897 * the link being in sleep state, call it again,
1898 * if it fails a second time its a real issue.
1899 */
1900 if (em_validate_eeprom_checksum(&sc->hw) < 0) {
1901 printf("%s: The EEPROM Checksum Is Not Valid\n",
1902 DEVNAME(sc));
1903 return (EIO);
1904 }
1905 }
1906
1907 if (em_get_flash_presence_i210(&sc->hw) &&
1908 em_read_part_num(&sc->hw, &(sc->part_num)) < 0) {
1909 printf("%s: EEPROM read error while reading part number\n",
1910 DEVNAME(sc));
1911 return (EIO);
1912 }
1913
1914 /* Set up smart power down as default off on newer adapters */
1915 if (!em_smart_pwr_down &&
1916 (sc->hw.mac_type == em_82571 ||
1917 sc->hw.mac_type == em_82572 ||
1918 sc->hw.mac_type == em_82575 ||
1919 sc->hw.mac_type == em_82576 ||
1920 sc->hw.mac_type == em_82580 ||
1921 sc->hw.mac_type == em_i210 ||
1922 sc->hw.mac_type == em_i350 )) {
1923 uint16_t phy_tmp = 0;
1924
1925 /* Speed up time to link by disabling smart power down */
1926 em_read_phy_reg(&sc->hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
1927 phy_tmp &= ~IGP02E1000_PM_SPD;
1928 em_write_phy_reg(&sc->hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
1929 }
1930
1931 em_legacy_irq_quirk_spt(sc);
1932
1933 /*
1934 * These parameters control the automatic generation (Tx) and
1935 * response (Rx) to Ethernet PAUSE frames.
1936 * - High water mark should allow for at least two frames to be
1937 * received after sending an XOFF.
1938 * - Low water mark works best when it is very near the high water mark.
1939 * This allows the receiver to restart by sending XON when it has
1940 * drained a bit. Here we use an arbitrary value of 1500 which will
1941 * restart after one full frame is pulled from the buffer. There
1942 * could be several smaller frames in the buffer and if so they will
1943 * not trigger the XON until their total number reduces the buffer
1944 * by 1500.
1945 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1946 */
1947 rx_buffer_size = ((E1000_READ_REG(&sc->hw, PBA) & 0xffff) << 10 );
1948
1949 sc->hw.fc_high_water = rx_buffer_size -
1950 EM_ROUNDUP(sc->hw.max_frame_size, 1024);
1951 sc->hw.fc_low_water = sc->hw.fc_high_water - 1500;
1952 if (sc->hw.mac_type == em_80003es2lan)
1953 sc->hw.fc_pause_time = 0xFFFF;
1954 else
1955 sc->hw.fc_pause_time = 1000;
1956 sc->hw.fc_send_xon = TRUE;
1957 sc->hw.fc = E1000_FC_FULL;
1958
1959 em_disable_aspm(sc);
1960
1961 if ((ret_val = em_init_hw(sc)) != 0) {
1962 if (ret_val == E1000_DEFER_INIT) {
1963 INIT_DEBUGOUT("\nHardware Initialization Deferred ");
1964 return (EAGAIN);
1965 }
1966 printf("\n%s: Hardware Initialization Failed: %d\n",
1967 DEVNAME(sc), ret_val);
1968 return (EIO);
1969 }
1970
1971 em_check_for_link(&sc->hw);
1972
1973 return (0);
1974 }
1975
1976 /*********************************************************************
1977 *
1978 * Setup networking device structure and register an interface.
1979 *
1980 **********************************************************************/
1981 void
em_setup_interface(struct em_softc * sc)1982 em_setup_interface(struct em_softc *sc)
1983 {
1984 struct ifnet *ifp;
1985 uint64_t fiber_type = IFM_1000_SX;
1986
1987 INIT_DEBUGOUT("em_setup_interface: begin");
1988
1989 ifp = &sc->sc_ac.ac_if;
1990 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
1991 ifp->if_softc = sc;
1992 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1993 ifp->if_xflags = IFXF_MPSAFE;
1994 ifp->if_ioctl = em_ioctl;
1995 ifp->if_qstart = em_start;
1996 ifp->if_watchdog = em_watchdog;
1997 ifp->if_hardmtu =
1998 sc->hw.max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN;
1999 ifq_init_maxlen(&ifp->if_snd, sc->sc_tx_slots - 1);
2000
2001 ifp->if_capabilities = IFCAP_VLAN_MTU;
2002
2003 #if NVLAN > 0
2004 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
2005 #endif
2006
2007 if (sc->hw.mac_type >= em_82543) {
2008 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4;
2009 }
2010 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210) {
2011 ifp->if_capabilities |= IFCAP_CSUM_IPv4;
2012 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6;
2013 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
2014 }
2015
2016 /*
2017 * Specify the media types supported by this adapter and register
2018 * callbacks to update media and link information
2019 */
2020 ifmedia_init(&sc->media, IFM_IMASK, em_media_change,
2021 em_media_status);
2022 if (sc->hw.media_type == em_media_type_fiber ||
2023 sc->hw.media_type == em_media_type_internal_serdes) {
2024 if (sc->hw.mac_type == em_82545)
2025 fiber_type = IFM_1000_LX;
2026 ifmedia_add(&sc->media, IFM_ETHER | fiber_type | IFM_FDX,
2027 0, NULL);
2028 ifmedia_add(&sc->media, IFM_ETHER | fiber_type,
2029 0, NULL);
2030 } else {
2031 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL);
2032 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2033 0, NULL);
2034 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX,
2035 0, NULL);
2036 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2037 0, NULL);
2038 if (sc->hw.phy_type != em_phy_ife) {
2039 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
2040 0, NULL);
2041 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2042 }
2043 }
2044 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2045 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2046
2047 if_attach(ifp);
2048 ether_ifattach(ifp);
2049 em_enable_intr(sc);
2050 }
2051
2052 int
em_detach(struct device * self,int flags)2053 em_detach(struct device *self, int flags)
2054 {
2055 struct em_softc *sc = (struct em_softc *)self;
2056 struct ifnet *ifp = &sc->sc_ac.ac_if;
2057 struct pci_attach_args *pa = &sc->osdep.em_pa;
2058 pci_chipset_tag_t pc = pa->pa_pc;
2059
2060 if (sc->sc_intrhand)
2061 pci_intr_disestablish(pc, sc->sc_intrhand);
2062 sc->sc_intrhand = 0;
2063
2064 em_stop(sc, 1);
2065
2066 em_free_pci_resources(sc);
2067
2068 ether_ifdetach(ifp);
2069 if_detach(ifp);
2070
2071 return (0);
2072 }
2073
2074 int
em_activate(struct device * self,int act)2075 em_activate(struct device *self, int act)
2076 {
2077 struct em_softc *sc = (struct em_softc *)self;
2078 struct ifnet *ifp = &sc->sc_ac.ac_if;
2079 int rv = 0;
2080
2081 switch (act) {
2082 case DVACT_SUSPEND:
2083 if (ifp->if_flags & IFF_RUNNING)
2084 em_stop(sc, 0);
2085 /* We have no children atm, but we will soon */
2086 rv = config_activate_children(self, act);
2087 break;
2088 case DVACT_RESUME:
2089 if (ifp->if_flags & IFF_UP)
2090 em_init(sc);
2091 break;
2092 default:
2093 rv = config_activate_children(self, act);
2094 break;
2095 }
2096 return (rv);
2097 }
2098
2099 /*********************************************************************
2100 *
2101 * Workaround for SmartSpeed on 82541 and 82547 controllers
2102 *
2103 **********************************************************************/
2104 void
em_smartspeed(struct em_softc * sc)2105 em_smartspeed(struct em_softc *sc)
2106 {
2107 uint16_t phy_tmp;
2108
2109 if (sc->link_active || (sc->hw.phy_type != em_phy_igp) ||
2110 !sc->hw.autoneg || !(sc->hw.autoneg_advertised & ADVERTISE_1000_FULL))
2111 return;
2112
2113 if (sc->smartspeed == 0) {
2114 /* If Master/Slave config fault is asserted twice,
2115 * we assume back-to-back */
2116 em_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp);
2117 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2118 return;
2119 em_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp);
2120 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2121 em_read_phy_reg(&sc->hw, PHY_1000T_CTRL,
2122 &phy_tmp);
2123 if (phy_tmp & CR_1000T_MS_ENABLE) {
2124 phy_tmp &= ~CR_1000T_MS_ENABLE;
2125 em_write_phy_reg(&sc->hw,
2126 PHY_1000T_CTRL, phy_tmp);
2127 sc->smartspeed++;
2128 if (sc->hw.autoneg &&
2129 !em_phy_setup_autoneg(&sc->hw) &&
2130 !em_read_phy_reg(&sc->hw, PHY_CTRL,
2131 &phy_tmp)) {
2132 phy_tmp |= (MII_CR_AUTO_NEG_EN |
2133 MII_CR_RESTART_AUTO_NEG);
2134 em_write_phy_reg(&sc->hw,
2135 PHY_CTRL, phy_tmp);
2136 }
2137 }
2138 }
2139 return;
2140 } else if (sc->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2141 /* If still no link, perhaps using 2/3 pair cable */
2142 em_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp);
2143 phy_tmp |= CR_1000T_MS_ENABLE;
2144 em_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp);
2145 if (sc->hw.autoneg &&
2146 !em_phy_setup_autoneg(&sc->hw) &&
2147 !em_read_phy_reg(&sc->hw, PHY_CTRL, &phy_tmp)) {
2148 phy_tmp |= (MII_CR_AUTO_NEG_EN |
2149 MII_CR_RESTART_AUTO_NEG);
2150 em_write_phy_reg(&sc->hw, PHY_CTRL, phy_tmp);
2151 }
2152 }
2153 /* Restart process after EM_SMARTSPEED_MAX iterations */
2154 if (sc->smartspeed++ == EM_SMARTSPEED_MAX)
2155 sc->smartspeed = 0;
2156 }
2157
2158 /*
2159 * Manage DMA'able memory.
2160 */
2161 int
em_dma_malloc(struct em_softc * sc,bus_size_t size,struct em_dma_alloc * dma)2162 em_dma_malloc(struct em_softc *sc, bus_size_t size, struct em_dma_alloc *dma)
2163 {
2164 int r;
2165
2166 r = bus_dmamap_create(sc->sc_dmat, size, 1,
2167 size, 0, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map);
2168 if (r != 0)
2169 return (r);
2170
2171 r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &dma->dma_seg,
2172 1, &dma->dma_nseg, BUS_DMA_WAITOK | BUS_DMA_ZERO);
2173 if (r != 0)
2174 goto destroy;
2175
2176 r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, size,
2177 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT);
2178 if (r != 0)
2179 goto free;
2180
2181 r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, size,
2182 NULL, BUS_DMA_WAITOK);
2183 if (r != 0)
2184 goto unmap;
2185
2186 dma->dma_size = size;
2187 return (0);
2188
2189 unmap:
2190 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
2191 free:
2192 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2193 destroy:
2194 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2195
2196 return (r);
2197 }
2198
2199 void
em_dma_free(struct em_softc * sc,struct em_dma_alloc * dma)2200 em_dma_free(struct em_softc *sc, struct em_dma_alloc *dma)
2201 {
2202 bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
2203 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
2204 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2205 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2206 }
2207
2208 /*********************************************************************
2209 *
2210 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2211 * the information needed to transmit a packet on the wire.
2212 *
2213 **********************************************************************/
2214 int
em_allocate_transmit_structures(struct em_softc * sc)2215 em_allocate_transmit_structures(struct em_softc *sc)
2216 {
2217 struct em_queue *que;
2218
2219 FOREACH_QUEUE(sc, que) {
2220 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2221 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2222 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2223
2224 que->tx.sc_tx_pkts_ring = mallocarray(sc->sc_tx_slots,
2225 sizeof(*que->tx.sc_tx_pkts_ring), M_DEVBUF, M_NOWAIT | M_ZERO);
2226 if (que->tx.sc_tx_pkts_ring == NULL) {
2227 printf("%s: Unable to allocate tx_buffer memory\n",
2228 DEVNAME(sc));
2229 return (ENOMEM);
2230 }
2231 }
2232
2233 return (0);
2234 }
2235
2236 /*********************************************************************
2237 *
2238 * Allocate and initialize transmit structures.
2239 *
2240 **********************************************************************/
2241 int
em_setup_transmit_structures(struct em_softc * sc)2242 em_setup_transmit_structures(struct em_softc *sc)
2243 {
2244 struct em_queue *que;
2245 struct em_packet *pkt;
2246 int error, i;
2247
2248 if ((error = em_allocate_transmit_structures(sc)) != 0)
2249 goto fail;
2250
2251 FOREACH_QUEUE(sc, que) {
2252 bzero((void *) que->tx.sc_tx_desc_ring,
2253 (sizeof(struct em_tx_desc)) * sc->sc_tx_slots);
2254
2255 for (i = 0; i < sc->sc_tx_slots; i++) {
2256 pkt = &que->tx.sc_tx_pkts_ring[i];
2257 error = bus_dmamap_create(sc->sc_dmat, EM_TSO_SIZE,
2258 EM_MAX_SCATTER / (sc->pcix_82544 ? 2 : 1),
2259 EM_TSO_SEG_SIZE, 0, BUS_DMA_NOWAIT, &pkt->pkt_map);
2260 if (error != 0) {
2261 printf("%s: Unable to create TX DMA map\n",
2262 DEVNAME(sc));
2263 goto fail;
2264 }
2265 }
2266
2267 que->tx.sc_tx_desc_head = 0;
2268 que->tx.sc_tx_desc_tail = 0;
2269
2270 /* Set checksum context */
2271 que->tx.active_checksum_context = OFFLOAD_NONE;
2272 }
2273
2274 return (0);
2275
2276 fail:
2277 em_free_transmit_structures(sc);
2278 return (error);
2279 }
2280
2281 /*********************************************************************
2282 *
2283 * Enable transmit unit.
2284 *
2285 **********************************************************************/
2286 void
em_initialize_transmit_unit(struct em_softc * sc)2287 em_initialize_transmit_unit(struct em_softc *sc)
2288 {
2289 u_int32_t reg_tctl, reg_tipg = 0;
2290 u_int64_t bus_addr;
2291 struct em_queue *que;
2292
2293 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
2294
2295 FOREACH_QUEUE(sc, que) {
2296 /* Setup the Base and Length of the Tx Descriptor Ring */
2297 bus_addr = que->tx.sc_tx_dma.dma_map->dm_segs[0].ds_addr;
2298 E1000_WRITE_REG(&sc->hw, TDLEN(que->me),
2299 sc->sc_tx_slots *
2300 sizeof(struct em_tx_desc));
2301 E1000_WRITE_REG(&sc->hw, TDBAH(que->me), (u_int32_t)(bus_addr >> 32));
2302 E1000_WRITE_REG(&sc->hw, TDBAL(que->me), (u_int32_t)bus_addr);
2303
2304 /* Setup the HW Tx Head and Tail descriptor pointers */
2305 E1000_WRITE_REG(&sc->hw, TDT(que->me), 0);
2306 E1000_WRITE_REG(&sc->hw, TDH(que->me), 0);
2307
2308 HW_DEBUGOUT2("Base = %x, Length = %x\n",
2309 E1000_READ_REG(&sc->hw, TDBAL(que->me)),
2310 E1000_READ_REG(&sc->hw, TDLEN(que->me)));
2311
2312 /* Set the default values for the Tx Inter Packet Gap timer */
2313 switch (sc->hw.mac_type) {
2314 case em_82542_rev2_0:
2315 case em_82542_rev2_1:
2316 reg_tipg = DEFAULT_82542_TIPG_IPGT;
2317 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2318 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2319 break;
2320 case em_80003es2lan:
2321 reg_tipg = DEFAULT_82543_TIPG_IPGR1;
2322 reg_tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2323 break;
2324 default:
2325 if (sc->hw.media_type == em_media_type_fiber ||
2326 sc->hw.media_type == em_media_type_internal_serdes)
2327 reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2328 else
2329 reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2330 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2331 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2332 }
2333
2334
2335 E1000_WRITE_REG(&sc->hw, TIPG, reg_tipg);
2336 E1000_WRITE_REG(&sc->hw, TIDV, sc->tx_int_delay);
2337 if (sc->hw.mac_type >= em_82540)
2338 E1000_WRITE_REG(&sc->hw, TADV, sc->tx_abs_int_delay);
2339
2340 /* Setup Transmit Descriptor Base Settings */
2341 que->tx.sc_txd_cmd = E1000_TXD_CMD_IFCS;
2342
2343 if (sc->hw.mac_type == em_82575 || sc->hw.mac_type == em_82580 ||
2344 sc->hw.mac_type == em_82576 ||
2345 sc->hw.mac_type == em_i210 || sc->hw.mac_type == em_i350) {
2346 /* 82575/6 need to enable the TX queue and lack the IDE bit */
2347 reg_tctl = E1000_READ_REG(&sc->hw, TXDCTL(que->me));
2348 reg_tctl |= E1000_TXDCTL_QUEUE_ENABLE;
2349 E1000_WRITE_REG(&sc->hw, TXDCTL(que->me), reg_tctl);
2350 } else if (sc->tx_int_delay > 0)
2351 que->tx.sc_txd_cmd |= E1000_TXD_CMD_IDE;
2352 }
2353
2354 /* Program the Transmit Control Register */
2355 reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
2356 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2357 if (sc->hw.mac_type >= em_82571)
2358 reg_tctl |= E1000_TCTL_MULR;
2359 if (sc->link_duplex == FULL_DUPLEX)
2360 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2361 else
2362 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2363 /* This write will effectively turn on the transmit unit */
2364 E1000_WRITE_REG(&sc->hw, TCTL, reg_tctl);
2365
2366 /* SPT Si errata workaround to avoid data corruption */
2367
2368 if (sc->hw.mac_type == em_pch_spt) {
2369 uint32_t reg_val;
2370
2371 reg_val = EM_READ_REG(&sc->hw, E1000_IOSFPC);
2372 reg_val |= E1000_RCTL_RDMTS_HEX;
2373 EM_WRITE_REG(&sc->hw, E1000_IOSFPC, reg_val);
2374
2375 reg_val = E1000_READ_REG(&sc->hw, TARC0);
2376 /* i218-i219 Specification Update 1.5.4.5 */
2377 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ;
2378 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ;
2379 E1000_WRITE_REG(&sc->hw, TARC0, reg_val);
2380 }
2381 }
2382
2383 /*********************************************************************
2384 *
2385 * Free all transmit related data structures.
2386 *
2387 **********************************************************************/
2388 void
em_free_transmit_structures(struct em_softc * sc)2389 em_free_transmit_structures(struct em_softc *sc)
2390 {
2391 struct em_queue *que;
2392 struct em_packet *pkt;
2393 int i;
2394
2395 INIT_DEBUGOUT("free_transmit_structures: begin");
2396
2397 FOREACH_QUEUE(sc, que) {
2398 if (que->tx.sc_tx_pkts_ring != NULL) {
2399 for (i = 0; i < sc->sc_tx_slots; i++) {
2400 pkt = &que->tx.sc_tx_pkts_ring[i];
2401
2402 if (pkt->pkt_m != NULL) {
2403 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
2404 0, pkt->pkt_map->dm_mapsize,
2405 BUS_DMASYNC_POSTWRITE);
2406 bus_dmamap_unload(sc->sc_dmat,
2407 pkt->pkt_map);
2408
2409 m_freem(pkt->pkt_m);
2410 pkt->pkt_m = NULL;
2411 }
2412
2413 if (pkt->pkt_map != NULL) {
2414 bus_dmamap_destroy(sc->sc_dmat,
2415 pkt->pkt_map);
2416 pkt->pkt_map = NULL;
2417 }
2418 }
2419
2420 free(que->tx.sc_tx_pkts_ring, M_DEVBUF,
2421 sc->sc_tx_slots * sizeof(*que->tx.sc_tx_pkts_ring));
2422 que->tx.sc_tx_pkts_ring = NULL;
2423 }
2424
2425 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2426 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2427 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2428 }
2429 }
2430
2431 u_int
em_tso_setup(struct em_queue * que,struct mbuf * mp,u_int head,u_int32_t * olinfo_status,u_int32_t * cmd_type_len)2432 em_tso_setup(struct em_queue *que, struct mbuf *mp, u_int head,
2433 u_int32_t *olinfo_status, u_int32_t *cmd_type_len)
2434 {
2435 struct ether_extracted ext;
2436 struct e1000_adv_tx_context_desc *TD;
2437 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
2438
2439 *olinfo_status = 0;
2440 *cmd_type_len = 0;
2441 TD = (struct e1000_adv_tx_context_desc *)&que->tx.sc_tx_desc_ring[head];
2442
2443 #if NVLAN > 0
2444 if (ISSET(mp->m_flags, M_VLANTAG)) {
2445 uint32_t vtag = mp->m_pkthdr.ether_vtag;
2446 vlan_macip_lens |= vtag << E1000_ADVTXD_VLAN_SHIFT;
2447 *cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2448 }
2449 #endif
2450
2451 ether_extract_headers(mp, &ext);
2452 if (ext.tcp == NULL || mp->m_pkthdr.ph_mss == 0)
2453 goto out;
2454
2455 vlan_macip_lens |= (sizeof(*ext.eh) << E1000_ADVTXD_MACLEN_SHIFT);
2456
2457 if (ext.ip4) {
2458 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
2459 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2460 #ifdef INET6
2461 } else if (ext.ip6) {
2462 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
2463 #endif
2464 } else {
2465 goto out;
2466 }
2467
2468 *cmd_type_len |= E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS;
2469 *cmd_type_len |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DCMD_TSE;
2470 *olinfo_status |= ext.paylen << E1000_ADVTXD_PAYLEN_SHIFT;
2471 vlan_macip_lens |= ext.iphlen;
2472 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
2473
2474 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
2475 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2476
2477 mss_l4len_idx |= mp->m_pkthdr.ph_mss << E1000_ADVTXD_MSS_SHIFT;
2478 mss_l4len_idx |= ext.tcphlen << E1000_ADVTXD_L4LEN_SHIFT;
2479 /* 82575 needs the queue index added */
2480 if (que->sc->hw.mac_type == em_82575)
2481 mss_l4len_idx |= (que->me & 0xff) << 4;
2482
2483 htolem32(&TD->vlan_macip_lens, vlan_macip_lens);
2484 htolem32(&TD->type_tucmd_mlhl, type_tucmd_mlhl);
2485 htolem32(&TD->u.seqnum_seed, 0);
2486 htolem32(&TD->mss_l4len_idx, mss_l4len_idx);
2487
2488 tcpstat_add(tcps_outpkttso, (ext.paylen + mp->m_pkthdr.ph_mss - 1) /
2489 mp->m_pkthdr.ph_mss);
2490
2491 return 1;
2492
2493 out:
2494 tcpstat_inc(tcps_outbadtso);
2495 return 0;
2496 }
2497
2498 u_int
em_tx_ctx_setup(struct em_queue * que,struct mbuf * mp,u_int head,u_int32_t * olinfo_status,u_int32_t * cmd_type_len)2499 em_tx_ctx_setup(struct em_queue *que, struct mbuf *mp, u_int head,
2500 u_int32_t *olinfo_status, u_int32_t *cmd_type_len)
2501 {
2502 struct ether_extracted ext;
2503 struct e1000_adv_tx_context_desc *TD;
2504 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
2505 int off = 0;
2506
2507 *olinfo_status = 0;
2508 *cmd_type_len = 0;
2509 TD = (struct e1000_adv_tx_context_desc *)&que->tx.sc_tx_desc_ring[head];
2510
2511 #if NVLAN > 0
2512 if (ISSET(mp->m_flags, M_VLANTAG)) {
2513 uint32_t vtag = mp->m_pkthdr.ether_vtag;
2514 vlan_macip_lens |= vtag << E1000_ADVTXD_VLAN_SHIFT;
2515 *cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2516 off = 1;
2517 }
2518 #endif
2519
2520 ether_extract_headers(mp, &ext);
2521
2522 vlan_macip_lens |= (sizeof(*ext.eh) << E1000_ADVTXD_MACLEN_SHIFT);
2523
2524 if (ext.ip4) {
2525 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
2526 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) {
2527 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2528 off = 1;
2529 }
2530 #ifdef INET6
2531 } else if (ext.ip6) {
2532 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
2533 #endif
2534 }
2535
2536 *cmd_type_len |= E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS;
2537 *cmd_type_len |= E1000_ADVTXD_DCMD_DEXT;
2538 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
2539 vlan_macip_lens |= ext.iphlen;
2540 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
2541
2542 if (ext.tcp) {
2543 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
2544 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) {
2545 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2546 off = 1;
2547 }
2548 } else if (ext.udp) {
2549 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
2550 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) {
2551 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2552 off = 1;
2553 }
2554 }
2555
2556 if (!off)
2557 return (0);
2558
2559 /* 82575 needs the queue index added */
2560 if (que->sc->hw.mac_type == em_82575)
2561 mss_l4len_idx |= (que->me & 0xff) << 4;
2562
2563 htolem32(&TD->vlan_macip_lens, vlan_macip_lens);
2564 htolem32(&TD->type_tucmd_mlhl, type_tucmd_mlhl);
2565 htolem32(&TD->u.seqnum_seed, 0);
2566 htolem32(&TD->mss_l4len_idx, mss_l4len_idx);
2567
2568 return (1);
2569 }
2570
2571 /*********************************************************************
2572 *
2573 * The offload context needs to be set when we transfer the first
2574 * packet of a particular protocol (TCP/UDP). We change the
2575 * context only if the protocol type changes.
2576 *
2577 **********************************************************************/
2578 u_int
em_transmit_checksum_setup(struct em_queue * que,struct mbuf * mp,u_int head,u_int32_t * txd_upper,u_int32_t * txd_lower)2579 em_transmit_checksum_setup(struct em_queue *que, struct mbuf *mp, u_int head,
2580 u_int32_t *txd_upper, u_int32_t *txd_lower)
2581 {
2582 struct em_context_desc *TXD;
2583
2584 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) {
2585 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2586 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2587 if (que->tx.active_checksum_context == OFFLOAD_TCP_IP)
2588 return (0);
2589 else
2590 que->tx.active_checksum_context = OFFLOAD_TCP_IP;
2591 } else if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) {
2592 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2593 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2594 if (que->tx.active_checksum_context == OFFLOAD_UDP_IP)
2595 return (0);
2596 else
2597 que->tx.active_checksum_context = OFFLOAD_UDP_IP;
2598 } else {
2599 *txd_upper = 0;
2600 *txd_lower = 0;
2601 return (0);
2602 }
2603
2604 /* If we reach this point, the checksum offload context
2605 * needs to be reset.
2606 */
2607 TXD = (struct em_context_desc *)&que->tx.sc_tx_desc_ring[head];
2608
2609 TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2610 TXD->lower_setup.ip_fields.ipcso =
2611 ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2612 TXD->lower_setup.ip_fields.ipcse =
2613 htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2614
2615 TXD->upper_setup.tcp_fields.tucss =
2616 ETHER_HDR_LEN + sizeof(struct ip);
2617 TXD->upper_setup.tcp_fields.tucse = htole16(0);
2618
2619 if (que->tx.active_checksum_context == OFFLOAD_TCP_IP) {
2620 TXD->upper_setup.tcp_fields.tucso =
2621 ETHER_HDR_LEN + sizeof(struct ip) +
2622 offsetof(struct tcphdr, th_sum);
2623 } else if (que->tx.active_checksum_context == OFFLOAD_UDP_IP) {
2624 TXD->upper_setup.tcp_fields.tucso =
2625 ETHER_HDR_LEN + sizeof(struct ip) +
2626 offsetof(struct udphdr, uh_sum);
2627 }
2628
2629 TXD->tcp_seg_setup.data = htole32(0);
2630 TXD->cmd_and_length = htole32(que->tx.sc_txd_cmd | E1000_TXD_CMD_DEXT);
2631
2632 return (1);
2633 }
2634
2635 /**********************************************************************
2636 *
2637 * Examine each tx_buffer in the used queue. If the hardware is done
2638 * processing the packet then free associated resources. The
2639 * tx_buffer is put back on the free queue.
2640 *
2641 **********************************************************************/
2642 void
em_txeof(struct em_queue * que)2643 em_txeof(struct em_queue *que)
2644 {
2645 struct em_softc *sc = que->sc;
2646 struct ifnet *ifp = &sc->sc_ac.ac_if;
2647 struct em_packet *pkt;
2648 struct em_tx_desc *desc;
2649 u_int head, tail;
2650 u_int free = 0;
2651
2652 head = que->tx.sc_tx_desc_head;
2653 tail = que->tx.sc_tx_desc_tail;
2654
2655 if (head == tail)
2656 return;
2657
2658 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2659 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2660 BUS_DMASYNC_POSTREAD);
2661
2662 do {
2663 pkt = &que->tx.sc_tx_pkts_ring[tail];
2664 desc = &que->tx.sc_tx_desc_ring[pkt->pkt_eop];
2665
2666 if (!ISSET(desc->upper.fields.status, E1000_TXD_STAT_DD))
2667 break;
2668
2669 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
2670 0, pkt->pkt_map->dm_mapsize,
2671 BUS_DMASYNC_POSTWRITE);
2672 bus_dmamap_unload(sc->sc_dmat, pkt->pkt_map);
2673
2674 KASSERT(pkt->pkt_m != NULL);
2675
2676 m_freem(pkt->pkt_m);
2677 pkt->pkt_m = NULL;
2678
2679 tail = pkt->pkt_eop;
2680
2681 if (++tail == sc->sc_tx_slots)
2682 tail = 0;
2683
2684 free++;
2685 } while (tail != head);
2686
2687 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2688 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2689 BUS_DMASYNC_PREREAD);
2690
2691 if (free == 0)
2692 return;
2693
2694 que->tx.sc_tx_desc_tail = tail;
2695
2696 if (ifq_is_oactive(&ifp->if_snd))
2697 ifq_restart(&ifp->if_snd);
2698 else if (tail == head)
2699 ifp->if_timer = 0;
2700 }
2701
2702 /*********************************************************************
2703 *
2704 * Get a buffer from system mbuf buffer pool.
2705 *
2706 **********************************************************************/
2707 int
em_get_buf(struct em_queue * que,int i)2708 em_get_buf(struct em_queue *que, int i)
2709 {
2710 struct em_softc *sc = que->sc;
2711 struct mbuf *m;
2712 struct em_packet *pkt;
2713 struct em_rx_desc *desc;
2714 int error;
2715
2716 pkt = &que->rx.sc_rx_pkts_ring[i];
2717 desc = &que->rx.sc_rx_desc_ring[i];
2718
2719 KASSERT(pkt->pkt_m == NULL);
2720
2721 m = MCLGETL(NULL, M_DONTWAIT, EM_MCLBYTES);
2722 if (m == NULL) {
2723 sc->mbuf_cluster_failed++;
2724 return (ENOBUFS);
2725 }
2726 m->m_len = m->m_pkthdr.len = EM_MCLBYTES;
2727 m_adj(m, ETHER_ALIGN);
2728
2729 error = bus_dmamap_load_mbuf(sc->sc_dmat, pkt->pkt_map,
2730 m, BUS_DMA_NOWAIT);
2731 if (error) {
2732 m_freem(m);
2733 return (error);
2734 }
2735
2736 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
2737 0, pkt->pkt_map->dm_mapsize,
2738 BUS_DMASYNC_PREREAD);
2739 pkt->pkt_m = m;
2740
2741 memset(desc, 0, sizeof(*desc));
2742 htolem64(&desc->buffer_addr, pkt->pkt_map->dm_segs[0].ds_addr);
2743
2744 return (0);
2745 }
2746
2747 /*********************************************************************
2748 *
2749 * Allocate memory for rx_buffer structures. Since we use one
2750 * rx_buffer per received packet, the maximum number of rx_buffer's
2751 * that we'll need is equal to the number of receive descriptors
2752 * that we've allocated.
2753 *
2754 **********************************************************************/
2755 int
em_allocate_receive_structures(struct em_softc * sc)2756 em_allocate_receive_structures(struct em_softc *sc)
2757 {
2758 struct em_queue *que;
2759 struct em_packet *pkt;
2760 int i;
2761 int error;
2762
2763 FOREACH_QUEUE(sc, que) {
2764 que->rx.sc_rx_pkts_ring = mallocarray(sc->sc_rx_slots,
2765 sizeof(*que->rx.sc_rx_pkts_ring),
2766 M_DEVBUF, M_NOWAIT | M_ZERO);
2767 if (que->rx.sc_rx_pkts_ring == NULL) {
2768 printf("%s: Unable to allocate rx_buffer memory\n",
2769 DEVNAME(sc));
2770 return (ENOMEM);
2771 }
2772
2773 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
2774 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
2775 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2776
2777 for (i = 0; i < sc->sc_rx_slots; i++) {
2778 pkt = &que->rx.sc_rx_pkts_ring[i];
2779
2780 error = bus_dmamap_create(sc->sc_dmat, EM_MCLBYTES, 1,
2781 EM_MCLBYTES, 0, BUS_DMA_NOWAIT, &pkt->pkt_map);
2782 if (error != 0) {
2783 printf("%s: em_allocate_receive_structures: "
2784 "bus_dmamap_create failed; error %u\n",
2785 DEVNAME(sc), error);
2786 goto fail;
2787 }
2788
2789 pkt->pkt_m = NULL;
2790 }
2791 }
2792
2793 return (0);
2794
2795 fail:
2796 em_free_receive_structures(sc);
2797 return (error);
2798 }
2799
2800 /*********************************************************************
2801 *
2802 * Allocate and initialize receive structures.
2803 *
2804 **********************************************************************/
2805 int
em_setup_receive_structures(struct em_softc * sc)2806 em_setup_receive_structures(struct em_softc *sc)
2807 {
2808 struct ifnet *ifp = &sc->sc_ac.ac_if;
2809 struct em_queue *que;
2810 u_int lwm;
2811
2812 if (em_allocate_receive_structures(sc))
2813 return (ENOMEM);
2814
2815 FOREACH_QUEUE(sc, que) {
2816 memset(que->rx.sc_rx_desc_ring, 0,
2817 sc->sc_rx_slots * sizeof(*que->rx.sc_rx_desc_ring));
2818
2819 /* Setup our descriptor pointers */
2820 que->rx.sc_rx_desc_tail = 0;
2821 que->rx.sc_rx_desc_head = sc->sc_rx_slots - 1;
2822
2823 lwm = max(4, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1));
2824 if_rxr_init(&que->rx.sc_rx_ring, lwm, sc->sc_rx_slots);
2825
2826 if (em_rxfill(que) == 0) {
2827 printf("%s: unable to fill any rx descriptors\n",
2828 DEVNAME(sc));
2829 return (ENOMEM);
2830 }
2831 }
2832
2833 return (0);
2834 }
2835
2836 /*********************************************************************
2837 *
2838 * Enable receive unit.
2839 *
2840 **********************************************************************/
2841 void
em_initialize_receive_unit(struct em_softc * sc)2842 em_initialize_receive_unit(struct em_softc *sc)
2843 {
2844 struct em_queue *que;
2845 u_int32_t reg_rctl;
2846 u_int32_t reg_rxcsum;
2847 u_int32_t reg_srrctl;
2848 u_int64_t bus_addr;
2849
2850 INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2851
2852 /* Make sure receives are disabled while setting up the descriptor ring */
2853 E1000_WRITE_REG(&sc->hw, RCTL, 0);
2854
2855 /* Set the Receive Delay Timer Register */
2856 E1000_WRITE_REG(&sc->hw, RDTR,
2857 sc->rx_int_delay | E1000_RDT_FPDB);
2858
2859 if (sc->hw.mac_type >= em_82540) {
2860 if (sc->rx_int_delay)
2861 E1000_WRITE_REG(&sc->hw, RADV, sc->rx_abs_int_delay);
2862
2863 /* Set the interrupt throttling rate. Value is calculated
2864 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) */
2865 E1000_WRITE_REG(&sc->hw, ITR, DEFAULT_ITR);
2866 }
2867
2868 /* Setup the Receive Control Register */
2869 reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2870 E1000_RCTL_RDMTS_HALF |
2871 (sc->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2872
2873 if (sc->hw.tbi_compatibility_on == TRUE)
2874 reg_rctl |= E1000_RCTL_SBP;
2875
2876 /*
2877 * The i350 has a bug where it always strips the CRC whether
2878 * asked to or not. So ask for stripped CRC here and
2879 * cope in rxeof
2880 */
2881 if (sc->hw.mac_type == em_i210 || sc->hw.mac_type == em_i350)
2882 reg_rctl |= E1000_RCTL_SECRC;
2883
2884 switch (sc->sc_rx_buffer_len) {
2885 default:
2886 case EM_RXBUFFER_2048:
2887 reg_rctl |= E1000_RCTL_SZ_2048;
2888 break;
2889 case EM_RXBUFFER_4096:
2890 reg_rctl |= E1000_RCTL_SZ_4096|E1000_RCTL_BSEX|E1000_RCTL_LPE;
2891 break;
2892 case EM_RXBUFFER_8192:
2893 reg_rctl |= E1000_RCTL_SZ_8192|E1000_RCTL_BSEX|E1000_RCTL_LPE;
2894 break;
2895 case EM_RXBUFFER_16384:
2896 reg_rctl |= E1000_RCTL_SZ_16384|E1000_RCTL_BSEX|E1000_RCTL_LPE;
2897 break;
2898 }
2899
2900 if (sc->hw.max_frame_size != ETHER_MAX_LEN)
2901 reg_rctl |= E1000_RCTL_LPE;
2902
2903 /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2904 if (sc->hw.mac_type >= em_82543) {
2905 reg_rxcsum = E1000_READ_REG(&sc->hw, RXCSUM);
2906 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2907 E1000_WRITE_REG(&sc->hw, RXCSUM, reg_rxcsum);
2908 }
2909
2910 /*
2911 * XXX TEMPORARY WORKAROUND: on some systems with 82573
2912 * long latencies are observed, like Lenovo X60.
2913 */
2914 if (sc->hw.mac_type == em_82573)
2915 E1000_WRITE_REG(&sc->hw, RDTR, 0x20);
2916
2917 FOREACH_QUEUE(sc, que) {
2918 if (sc->num_queues > 1) {
2919 /*
2920 * Disable Drop Enable for every queue, default has
2921 * it enabled for queues > 0
2922 */
2923 reg_srrctl = E1000_READ_REG(&sc->hw, SRRCTL(que->me));
2924 reg_srrctl &= ~E1000_SRRCTL_DROP_EN;
2925 E1000_WRITE_REG(&sc->hw, SRRCTL(que->me), reg_srrctl);
2926 }
2927
2928 /* Setup the Base and Length of the Rx Descriptor Ring */
2929 bus_addr = que->rx.sc_rx_dma.dma_map->dm_segs[0].ds_addr;
2930 E1000_WRITE_REG(&sc->hw, RDLEN(que->me),
2931 sc->sc_rx_slots * sizeof(*que->rx.sc_rx_desc_ring));
2932 E1000_WRITE_REG(&sc->hw, RDBAH(que->me), (u_int32_t)(bus_addr >> 32));
2933 E1000_WRITE_REG(&sc->hw, RDBAL(que->me), (u_int32_t)bus_addr);
2934
2935 if (sc->hw.mac_type == em_82575 || sc->hw.mac_type == em_82580 ||
2936 sc->hw.mac_type == em_82576 ||
2937 sc->hw.mac_type == em_i210 || sc->hw.mac_type == em_i350) {
2938 /* 82575/6 need to enable the RX queue */
2939 uint32_t reg;
2940 reg = E1000_READ_REG(&sc->hw, RXDCTL(que->me));
2941 reg |= E1000_RXDCTL_QUEUE_ENABLE;
2942 E1000_WRITE_REG(&sc->hw, RXDCTL(que->me), reg);
2943 }
2944 }
2945
2946 /* Enable Receives */
2947 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
2948
2949 /* Setup the HW Rx Head and Tail Descriptor Pointers */
2950 FOREACH_QUEUE(sc, que) {
2951 E1000_WRITE_REG(&sc->hw, RDH(que->me), 0);
2952 E1000_WRITE_REG(&sc->hw, RDT(que->me), que->rx.sc_rx_desc_head);
2953 }
2954 }
2955
2956 /*********************************************************************
2957 *
2958 * Free receive related data structures.
2959 *
2960 **********************************************************************/
2961 void
em_free_receive_structures(struct em_softc * sc)2962 em_free_receive_structures(struct em_softc *sc)
2963 {
2964 struct em_queue *que;
2965 struct em_packet *pkt;
2966 int i;
2967
2968 INIT_DEBUGOUT("free_receive_structures: begin");
2969
2970 FOREACH_QUEUE(sc, que) {
2971 if_rxr_init(&que->rx.sc_rx_ring, 0, 0);
2972
2973 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
2974 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
2975 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2976
2977 if (que->rx.sc_rx_pkts_ring != NULL) {
2978 for (i = 0; i < sc->sc_rx_slots; i++) {
2979 pkt = &que->rx.sc_rx_pkts_ring[i];
2980 if (pkt->pkt_m != NULL) {
2981 bus_dmamap_sync(sc->sc_dmat,
2982 pkt->pkt_map,
2983 0, pkt->pkt_map->dm_mapsize,
2984 BUS_DMASYNC_POSTREAD);
2985 bus_dmamap_unload(sc->sc_dmat,
2986 pkt->pkt_map);
2987 m_freem(pkt->pkt_m);
2988 pkt->pkt_m = NULL;
2989 }
2990 bus_dmamap_destroy(sc->sc_dmat, pkt->pkt_map);
2991 }
2992
2993 free(que->rx.sc_rx_pkts_ring, M_DEVBUF,
2994 sc->sc_rx_slots * sizeof(*que->rx.sc_rx_pkts_ring));
2995 que->rx.sc_rx_pkts_ring = NULL;
2996 }
2997
2998 if (que->rx.fmp != NULL) {
2999 m_freem(que->rx.fmp);
3000 que->rx.fmp = NULL;
3001 que->rx.lmp = NULL;
3002 }
3003 }
3004 }
3005
3006 int
em_rxfill(struct em_queue * que)3007 em_rxfill(struct em_queue *que)
3008 {
3009 struct em_softc *sc = que->sc;
3010 u_int slots;
3011 int post = 0;
3012 int i;
3013
3014 i = que->rx.sc_rx_desc_head;
3015
3016 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3017 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3018 BUS_DMASYNC_POSTWRITE);
3019
3020 for (slots = if_rxr_get(&que->rx.sc_rx_ring, sc->sc_rx_slots);
3021 slots > 0; slots--) {
3022 if (++i == sc->sc_rx_slots)
3023 i = 0;
3024
3025 if (em_get_buf(que, i) != 0)
3026 break;
3027
3028 que->rx.sc_rx_desc_head = i;
3029 post = 1;
3030 }
3031
3032 if_rxr_put(&que->rx.sc_rx_ring, slots);
3033
3034 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3035 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3036 BUS_DMASYNC_PREWRITE);
3037
3038 return (post);
3039 }
3040
3041 void
em_rxrefill(void * arg)3042 em_rxrefill(void *arg)
3043 {
3044 struct em_queue *que = arg;
3045 int s;
3046
3047 s = splnet();
3048 em_rxrefill_locked(que);
3049 splx(s);
3050 }
3051
3052 void
em_rxrefill_locked(struct em_queue * que)3053 em_rxrefill_locked(struct em_queue *que)
3054 {
3055 struct em_softc *sc = que->sc;
3056
3057 if (em_rxfill(que))
3058 E1000_WRITE_REG(&sc->hw, RDT(que->me), que->rx.sc_rx_desc_head);
3059 else if (if_rxr_needrefill(&que->rx.sc_rx_ring))
3060 timeout_add(&que->rx_refill, 1);
3061 }
3062
3063 /*********************************************************************
3064 *
3065 * This routine executes in interrupt context. It replenishes
3066 * the mbufs in the descriptor and sends data which has been
3067 * dma'ed into host memory to upper layer.
3068 *
3069 *********************************************************************/
3070 int
em_rxeof(struct em_queue * que)3071 em_rxeof(struct em_queue *que)
3072 {
3073 struct em_softc *sc = que->sc;
3074 struct ifnet *ifp = &sc->sc_ac.ac_if;
3075 struct mbuf_list ml = MBUF_LIST_INITIALIZER();
3076 struct mbuf *m;
3077 u_int8_t accept_frame = 0;
3078 u_int8_t eop = 0;
3079 u_int16_t len, desc_len, prev_len_adj;
3080 int i, rv = 0;
3081
3082 /* Pointer to the receive descriptor being examined. */
3083 struct em_rx_desc *desc;
3084 struct em_packet *pkt;
3085 u_int8_t status;
3086
3087 if (if_rxr_inuse(&que->rx.sc_rx_ring) == 0)
3088 return (0);
3089
3090 i = que->rx.sc_rx_desc_tail;
3091
3092 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3093 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3094 BUS_DMASYNC_POSTREAD);
3095
3096 do {
3097 m = NULL;
3098
3099 pkt = &que->rx.sc_rx_pkts_ring[i];
3100 desc = &que->rx.sc_rx_desc_ring[i];
3101
3102 status = desc->status;
3103 if (!ISSET(status, E1000_RXD_STAT_DD))
3104 break;
3105
3106 /* pull the mbuf off the ring */
3107 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
3108 0, pkt->pkt_map->dm_mapsize,
3109 BUS_DMASYNC_POSTREAD);
3110 bus_dmamap_unload(sc->sc_dmat, pkt->pkt_map);
3111 m = pkt->pkt_m;
3112 pkt->pkt_m = NULL;
3113
3114 KASSERT(m != NULL);
3115
3116 if_rxr_put(&que->rx.sc_rx_ring, 1);
3117 rv = 1;
3118
3119 accept_frame = 1;
3120 prev_len_adj = 0;
3121 desc_len = letoh16(desc->length);
3122
3123 if (status & E1000_RXD_STAT_EOP) {
3124 eop = 1;
3125 if (desc_len < ETHER_CRC_LEN) {
3126 len = 0;
3127 prev_len_adj = ETHER_CRC_LEN - desc_len;
3128 } else if (sc->hw.mac_type == em_i210 ||
3129 sc->hw.mac_type == em_i350)
3130 len = desc_len;
3131 else
3132 len = desc_len - ETHER_CRC_LEN;
3133 } else {
3134 eop = 0;
3135 len = desc_len;
3136 }
3137
3138 if (desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3139 u_int8_t last_byte;
3140 u_int32_t pkt_len = desc_len;
3141
3142 if (que->rx.fmp != NULL)
3143 pkt_len += que->rx.fmp->m_pkthdr.len;
3144
3145 last_byte = *(mtod(m, caddr_t) + desc_len - 1);
3146 if (TBI_ACCEPT(&sc->hw, status, desc->errors,
3147 pkt_len, last_byte)) {
3148 #if NKSTAT > 0
3149 em_tbi_adjust_stats(sc,
3150 pkt_len, sc->hw.mac_addr);
3151 #endif
3152 if (len > 0)
3153 len--;
3154 } else
3155 accept_frame = 0;
3156 }
3157
3158 if (accept_frame) {
3159 /* Assign correct length to the current fragment */
3160 m->m_len = len;
3161
3162 if (que->rx.fmp == NULL) {
3163 m->m_pkthdr.len = m->m_len;
3164 que->rx.fmp = m; /* Store the first mbuf */
3165 que->rx.lmp = m;
3166 } else {
3167 /* Chain mbuf's together */
3168 m->m_flags &= ~M_PKTHDR;
3169 /*
3170 * Adjust length of previous mbuf in chain if
3171 * we received less than 4 bytes in the last
3172 * descriptor.
3173 */
3174 if (prev_len_adj > 0) {
3175 que->rx.lmp->m_len -= prev_len_adj;
3176 que->rx.fmp->m_pkthdr.len -= prev_len_adj;
3177 }
3178 que->rx.lmp->m_next = m;
3179 que->rx.lmp = m;
3180 que->rx.fmp->m_pkthdr.len += m->m_len;
3181 }
3182
3183 if (eop) {
3184 m = que->rx.fmp;
3185
3186 em_receive_checksum(sc, desc, m);
3187 #if NVLAN > 0
3188 if (desc->status & E1000_RXD_STAT_VP) {
3189 m->m_pkthdr.ether_vtag =
3190 letoh16(desc->special);
3191 m->m_flags |= M_VLANTAG;
3192 }
3193 #endif
3194 ml_enqueue(&ml, m);
3195
3196 que->rx.fmp = NULL;
3197 que->rx.lmp = NULL;
3198 }
3199 } else {
3200 que->rx.dropped_pkts++;
3201
3202 if (que->rx.fmp != NULL) {
3203 m_freem(que->rx.fmp);
3204 que->rx.fmp = NULL;
3205 que->rx.lmp = NULL;
3206 }
3207
3208 m_freem(m);
3209 }
3210
3211 /* Advance our pointers to the next descriptor. */
3212 if (++i == sc->sc_rx_slots)
3213 i = 0;
3214 } while (if_rxr_inuse(&que->rx.sc_rx_ring) > 0);
3215
3216 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3217 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3218 BUS_DMASYNC_PREREAD);
3219
3220 que->rx.sc_rx_desc_tail = i;
3221
3222 if (ifiq_input(&ifp->if_rcv, &ml))
3223 if_rxr_livelocked(&que->rx.sc_rx_ring);
3224
3225 return (rv);
3226 }
3227
3228 /*********************************************************************
3229 *
3230 * Verify that the hardware indicated that the checksum is valid.
3231 * Inform the stack about the status of checksum so that stack
3232 * doesn't spend time verifying the checksum.
3233 *
3234 *********************************************************************/
3235 void
em_receive_checksum(struct em_softc * sc,struct em_rx_desc * rx_desc,struct mbuf * mp)3236 em_receive_checksum(struct em_softc *sc, struct em_rx_desc *rx_desc,
3237 struct mbuf *mp)
3238 {
3239 /* 82543 or newer only */
3240 if ((sc->hw.mac_type < em_82543) ||
3241 /* Ignore Checksum bit is set */
3242 (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3243 mp->m_pkthdr.csum_flags = 0;
3244 return;
3245 }
3246
3247 if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3248 /* Did it pass? */
3249 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3250 /* IP Checksum Good */
3251 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK;
3252
3253 } else
3254 mp->m_pkthdr.csum_flags = 0;
3255 }
3256
3257 if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3258 /* Did it pass? */
3259 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE))
3260 mp->m_pkthdr.csum_flags |=
3261 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK;
3262 }
3263 }
3264
3265 /*
3266 * This turns on the hardware offload of the VLAN
3267 * tag insertion and strip
3268 */
3269 void
em_enable_hw_vlans(struct em_softc * sc)3270 em_enable_hw_vlans(struct em_softc *sc)
3271 {
3272 uint32_t ctrl;
3273
3274 ctrl = E1000_READ_REG(&sc->hw, CTRL);
3275 ctrl |= E1000_CTRL_VME;
3276 E1000_WRITE_REG(&sc->hw, CTRL, ctrl);
3277 }
3278
3279 void
em_enable_intr(struct em_softc * sc)3280 em_enable_intr(struct em_softc *sc)
3281 {
3282 uint32_t mask;
3283
3284 if (sc->msix) {
3285 mask = sc->msix_queuesmask | sc->msix_linkmask;
3286 E1000_WRITE_REG(&sc->hw, EIAC, mask);
3287 E1000_WRITE_REG(&sc->hw, EIAM, mask);
3288 E1000_WRITE_REG(&sc->hw, EIMS, mask);
3289 E1000_WRITE_REG(&sc->hw, IMS, E1000_IMS_LSC);
3290 } else
3291 E1000_WRITE_REG(&sc->hw, IMS, (IMS_ENABLE_MASK));
3292 }
3293
3294 void
em_disable_intr(struct em_softc * sc)3295 em_disable_intr(struct em_softc *sc)
3296 {
3297 /*
3298 * The first version of 82542 had an errata where when link
3299 * was forced it would stay up even if the cable was disconnected
3300 * Sequence errors were used to detect the disconnect and then
3301 * the driver would unforce the link. This code is in the ISR.
3302 * For this to work correctly the Sequence error interrupt had
3303 * to be enabled all the time.
3304 */
3305 if (sc->msix) {
3306 E1000_WRITE_REG(&sc->hw, EIMC, ~0);
3307 E1000_WRITE_REG(&sc->hw, EIAC, 0);
3308 } else if (sc->hw.mac_type == em_82542_rev2_0)
3309 E1000_WRITE_REG(&sc->hw, IMC, (0xffffffff & ~E1000_IMC_RXSEQ));
3310 else
3311 E1000_WRITE_REG(&sc->hw, IMC, 0xffffffff);
3312 }
3313
3314 void
em_write_pci_cfg(struct em_hw * hw,uint32_t reg,uint16_t * value)3315 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3316 {
3317 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3318 pcireg_t val;
3319
3320 val = pci_conf_read(pa->pa_pc, pa->pa_tag, reg & ~0x3);
3321 if (reg & 0x2) {
3322 val &= 0x0000ffff;
3323 val |= (*value << 16);
3324 } else {
3325 val &= 0xffff0000;
3326 val |= *value;
3327 }
3328 pci_conf_write(pa->pa_pc, pa->pa_tag, reg & ~0x3, val);
3329 }
3330
3331 void
em_read_pci_cfg(struct em_hw * hw,uint32_t reg,uint16_t * value)3332 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3333 {
3334 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3335 pcireg_t val;
3336
3337 val = pci_conf_read(pa->pa_pc, pa->pa_tag, reg & ~0x3);
3338 if (reg & 0x2)
3339 *value = (val >> 16) & 0xffff;
3340 else
3341 *value = val & 0xffff;
3342 }
3343
3344 void
em_pci_set_mwi(struct em_hw * hw)3345 em_pci_set_mwi(struct em_hw *hw)
3346 {
3347 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3348
3349 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
3350 (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE));
3351 }
3352
3353 void
em_pci_clear_mwi(struct em_hw * hw)3354 em_pci_clear_mwi(struct em_hw *hw)
3355 {
3356 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3357
3358 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
3359 (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE));
3360 }
3361
3362 /*
3363 * We may eventually really do this, but its unnecessary
3364 * for now so we just return unsupported.
3365 */
3366 int32_t
em_read_pcie_cap_reg(struct em_hw * hw,uint32_t reg,uint16_t * value)3367 em_read_pcie_cap_reg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3368 {
3369 return -E1000_NOT_IMPLEMENTED;
3370 }
3371
3372 /*********************************************************************
3373 * 82544 Coexistence issue workaround.
3374 * There are 2 issues.
3375 * 1. Transmit Hang issue.
3376 * To detect this issue, following equation can be used...
3377 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3378 * If SUM[3:0] is in between 1 to 4, we will have this issue.
3379 *
3380 * 2. DAC issue.
3381 * To detect this issue, following equation can be used...
3382 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3383 * If SUM[3:0] is in between 9 to c, we will have this issue.
3384 *
3385 *
3386 * WORKAROUND:
3387 * Make sure we do not have ending address as 1,2,3,4(Hang) or 9,a,b,c (DAC)
3388 *
3389 *** *********************************************************************/
3390 u_int32_t
em_fill_descriptors(u_int64_t address,u_int32_t length,PDESC_ARRAY desc_array)3391 em_fill_descriptors(u_int64_t address, u_int32_t length,
3392 PDESC_ARRAY desc_array)
3393 {
3394 /* Since issue is sensitive to length and address.*/
3395 /* Let us first check the address...*/
3396 u_int32_t safe_terminator;
3397 if (length <= 4) {
3398 desc_array->descriptor[0].address = address;
3399 desc_array->descriptor[0].length = length;
3400 desc_array->elements = 1;
3401 return desc_array->elements;
3402 }
3403 safe_terminator = (u_int32_t)((((u_int32_t)address & 0x7) + (length & 0xF)) & 0xF);
3404 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3405 if (safe_terminator == 0 ||
3406 (safe_terminator > 4 &&
3407 safe_terminator < 9) ||
3408 (safe_terminator > 0xC &&
3409 safe_terminator <= 0xF)) {
3410 desc_array->descriptor[0].address = address;
3411 desc_array->descriptor[0].length = length;
3412 desc_array->elements = 1;
3413 return desc_array->elements;
3414 }
3415
3416 desc_array->descriptor[0].address = address;
3417 desc_array->descriptor[0].length = length - 4;
3418 desc_array->descriptor[1].address = address + (length - 4);
3419 desc_array->descriptor[1].length = 4;
3420 desc_array->elements = 2;
3421 return desc_array->elements;
3422 }
3423
3424 /*
3425 * Disable the L0S and L1 LINK states.
3426 */
3427 void
em_disable_aspm(struct em_softc * sc)3428 em_disable_aspm(struct em_softc *sc)
3429 {
3430 int offset;
3431 pcireg_t val;
3432
3433 switch (sc->hw.mac_type) {
3434 case em_82571:
3435 case em_82572:
3436 case em_82573:
3437 case em_82574:
3438 break;
3439 default:
3440 return;
3441 }
3442
3443 if (!pci_get_capability(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
3444 PCI_CAP_PCIEXPRESS, &offset, NULL))
3445 return;
3446
3447 /* Disable PCIe Active State Power Management (ASPM). */
3448 val = pci_conf_read(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
3449 offset + PCI_PCIE_LCSR);
3450
3451 switch (sc->hw.mac_type) {
3452 case em_82571:
3453 case em_82572:
3454 val &= ~PCI_PCIE_LCSR_ASPM_L1;
3455 break;
3456 case em_82573:
3457 case em_82574:
3458 val &= ~(PCI_PCIE_LCSR_ASPM_L0S |
3459 PCI_PCIE_LCSR_ASPM_L1);
3460 break;
3461 default:
3462 break;
3463 }
3464
3465 pci_conf_write(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
3466 offset + PCI_PCIE_LCSR, val);
3467 }
3468
3469 /*
3470 * em_flush_tx_ring - remove all descriptors from the tx_ring
3471 *
3472 * We want to clear all pending descriptors from the TX ring.
3473 * zeroing happens when the HW reads the regs. We assign the ring itself as
3474 * the data of the next descriptor. We don't care about the data we are about
3475 * to reset the HW.
3476 */
3477 void
em_flush_tx_ring(struct em_queue * que)3478 em_flush_tx_ring(struct em_queue *que)
3479 {
3480 struct em_softc *sc = que->sc;
3481 uint32_t tctl, txd_lower = E1000_TXD_CMD_IFCS;
3482 uint16_t size = 512;
3483 struct em_tx_desc *txd;
3484
3485 KASSERT(que->tx.sc_tx_desc_ring != NULL);
3486
3487 tctl = EM_READ_REG(&sc->hw, E1000_TCTL);
3488 EM_WRITE_REG(&sc->hw, E1000_TCTL, tctl | E1000_TCTL_EN);
3489
3490 KASSERT(EM_READ_REG(&sc->hw, E1000_TDT(que->me)) == que->tx.sc_tx_desc_head);
3491
3492 txd = &que->tx.sc_tx_desc_ring[que->tx.sc_tx_desc_head];
3493 txd->buffer_addr = que->tx.sc_tx_dma.dma_map->dm_segs[0].ds_addr;
3494 txd->lower.data = htole32(txd_lower | size);
3495 txd->upper.data = 0;
3496
3497 /* flush descriptors to memory before notifying the HW */
3498 bus_space_barrier(sc->osdep.mem_bus_space_tag,
3499 sc->osdep.mem_bus_space_handle, 0, 0, BUS_SPACE_BARRIER_WRITE);
3500
3501 if (++que->tx.sc_tx_desc_head == sc->sc_tx_slots)
3502 que->tx.sc_tx_desc_head = 0;
3503
3504 EM_WRITE_REG(&sc->hw, E1000_TDT(que->me), que->tx.sc_tx_desc_head);
3505 bus_space_barrier(sc->osdep.mem_bus_space_tag, sc->osdep.mem_bus_space_handle,
3506 0, 0, BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE);
3507 usec_delay(250);
3508 }
3509
3510 /*
3511 * em_flush_rx_ring - remove all descriptors from the rx_ring
3512 *
3513 * Mark all descriptors in the RX ring as consumed and disable the rx ring
3514 */
3515 void
em_flush_rx_ring(struct em_queue * que)3516 em_flush_rx_ring(struct em_queue *que)
3517 {
3518 uint32_t rctl, rxdctl;
3519 struct em_softc *sc = que->sc;
3520
3521 rctl = EM_READ_REG(&sc->hw, E1000_RCTL);
3522 EM_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3523 E1000_WRITE_FLUSH(&sc->hw);
3524 usec_delay(150);
3525
3526 rxdctl = EM_READ_REG(&sc->hw, E1000_RXDCTL(que->me));
3527 /* zero the lower 14 bits (prefetch and host thresholds) */
3528 rxdctl &= 0xffffc000;
3529 /*
3530 * update thresholds: prefetch threshold to 31, host threshold to 1
3531 * and make sure the granularity is "descriptors" and not "cache lines"
3532 */
3533 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC);
3534 EM_WRITE_REG(&sc->hw, E1000_RXDCTL(que->me), rxdctl);
3535
3536 /* momentarily enable the RX ring for the changes to take effect */
3537 EM_WRITE_REG(&sc->hw, E1000_RCTL, rctl | E1000_RCTL_EN);
3538 E1000_WRITE_FLUSH(&sc->hw);
3539 usec_delay(150);
3540 EM_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3541 }
3542
3543 /*
3544 * em_flush_desc_rings - remove all descriptors from the descriptor rings
3545 *
3546 * In i219, the descriptor rings must be emptied before resetting the HW
3547 * or before changing the device state to D3 during runtime (runtime PM).
3548 *
3549 * Failure to do this will cause the HW to enter a unit hang state which can
3550 * only be released by PCI reset on the device
3551 *
3552 */
3553 void
em_flush_desc_rings(struct em_softc * sc)3554 em_flush_desc_rings(struct em_softc *sc)
3555 {
3556 struct em_queue *que = sc->queues; /* Use only first queue. */
3557 struct pci_attach_args *pa = &sc->osdep.em_pa;
3558 uint32_t fextnvm11, tdlen;
3559 uint16_t hang_state;
3560
3561 /* First, disable MULR fix in FEXTNVM11 */
3562 fextnvm11 = EM_READ_REG(&sc->hw, E1000_FEXTNVM11);
3563 fextnvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;
3564 EM_WRITE_REG(&sc->hw, E1000_FEXTNVM11, fextnvm11);
3565
3566 /* do nothing if we're not in faulty state, or if the queue is empty */
3567 tdlen = EM_READ_REG(&sc->hw, E1000_TDLEN(que->me));
3568 hang_state = pci_conf_read(pa->pa_pc, pa->pa_tag, PCICFG_DESC_RING_STATUS);
3569 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen)
3570 return;
3571 em_flush_tx_ring(que);
3572
3573 /* recheck, maybe the fault is caused by the rx ring */
3574 hang_state = pci_conf_read(pa->pa_pc, pa->pa_tag, PCICFG_DESC_RING_STATUS);
3575 if (hang_state & FLUSH_DESC_REQUIRED)
3576 em_flush_rx_ring(que);
3577 }
3578
3579 int
em_allocate_legacy(struct em_softc * sc)3580 em_allocate_legacy(struct em_softc *sc)
3581 {
3582 pci_intr_handle_t ih;
3583 const char *intrstr = NULL;
3584 struct pci_attach_args *pa = &sc->osdep.em_pa;
3585 pci_chipset_tag_t pc = pa->pa_pc;
3586
3587 if (pci_intr_map_msi(pa, &ih)) {
3588 if (pci_intr_map(pa, &ih)) {
3589 printf(": couldn't map interrupt\n");
3590 return (ENXIO);
3591 }
3592 sc->legacy_irq = 1;
3593 }
3594
3595 intrstr = pci_intr_string(pc, ih);
3596 sc->sc_intrhand = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
3597 em_intr, sc, DEVNAME(sc));
3598 if (sc->sc_intrhand == NULL) {
3599 printf(": couldn't establish interrupt");
3600 if (intrstr != NULL)
3601 printf(" at %s", intrstr);
3602 printf("\n");
3603 return (ENXIO);
3604 }
3605 printf(": %s", intrstr);
3606
3607 return (0);
3608 }
3609
3610 #if NKSTAT > 0
3611 /* this is used to look up the array of kstats quickly */
3612 enum em_stat {
3613 em_stat_crcerrs,
3614 em_stat_algnerrc,
3615 em_stat_symerrs,
3616 em_stat_rxerrc,
3617 em_stat_mpc,
3618 em_stat_scc,
3619 em_stat_ecol,
3620 em_stat_mcc,
3621 em_stat_latecol,
3622 em_stat_colc,
3623 em_stat_dc,
3624 em_stat_tncrs,
3625 em_stat_sec,
3626 em_stat_cexterr,
3627 em_stat_rlec,
3628 em_stat_xonrxc,
3629 em_stat_xontxc,
3630 em_stat_xoffrxc,
3631 em_stat_xofftxc,
3632 em_stat_fcruc,
3633 em_stat_prc64,
3634 em_stat_prc127,
3635 em_stat_prc255,
3636 em_stat_prc511,
3637 em_stat_prc1023,
3638 em_stat_prc1522,
3639 em_stat_gprc,
3640 em_stat_bprc,
3641 em_stat_mprc,
3642 em_stat_gptc,
3643 em_stat_gorc,
3644 em_stat_gotc,
3645 em_stat_rnbc,
3646 em_stat_ruc,
3647 em_stat_rfc,
3648 em_stat_roc,
3649 em_stat_rjc,
3650 em_stat_mgtprc,
3651 em_stat_mgtpdc,
3652 em_stat_mgtptc,
3653 em_stat_tor,
3654 em_stat_tot,
3655 em_stat_tpr,
3656 em_stat_tpt,
3657 em_stat_ptc64,
3658 em_stat_ptc127,
3659 em_stat_ptc255,
3660 em_stat_ptc511,
3661 em_stat_ptc1023,
3662 em_stat_ptc1522,
3663 em_stat_mptc,
3664 em_stat_bptc,
3665 #if 0
3666 em_stat_tsctc,
3667 em_stat_tsctf,
3668 #endif
3669
3670 em_stat_count,
3671 };
3672
3673 struct em_counter {
3674 const char *name;
3675 enum kstat_kv_unit unit;
3676 uint32_t reg;
3677 };
3678
3679 static const struct em_counter em_counters[em_stat_count] = {
3680 [em_stat_crcerrs] =
3681 { "rx crc errs", KSTAT_KV_U_PACKETS, E1000_CRCERRS },
3682 [em_stat_algnerrc] = /* >= em_82543 */
3683 { "rx align errs", KSTAT_KV_U_PACKETS, 0 },
3684 [em_stat_symerrs] = /* >= em_82543 */
3685 { "rx align errs", KSTAT_KV_U_PACKETS, 0 },
3686 [em_stat_rxerrc] =
3687 { "rx errs", KSTAT_KV_U_PACKETS, E1000_RXERRC },
3688 [em_stat_mpc] =
3689 { "rx missed", KSTAT_KV_U_PACKETS, E1000_MPC },
3690 [em_stat_scc] =
3691 { "tx single coll", KSTAT_KV_U_PACKETS, E1000_SCC },
3692 [em_stat_ecol] =
3693 { "tx excess coll", KSTAT_KV_U_PACKETS, E1000_ECOL },
3694 [em_stat_mcc] =
3695 { "tx multi coll", KSTAT_KV_U_PACKETS, E1000_MCC },
3696 [em_stat_latecol] =
3697 { "tx late coll", KSTAT_KV_U_PACKETS, E1000_LATECOL },
3698 [em_stat_colc] =
3699 { "tx coll", KSTAT_KV_U_NONE, E1000_COLC },
3700 [em_stat_dc] =
3701 { "tx defers", KSTAT_KV_U_NONE, E1000_DC },
3702 [em_stat_tncrs] = /* >= em_82543 */
3703 { "tx no CRS", KSTAT_KV_U_PACKETS, 0 },
3704 [em_stat_sec] =
3705 { "seq errs", KSTAT_KV_U_NONE, E1000_SEC },
3706 [em_stat_cexterr] = /* >= em_82543 */
3707 { "carr ext errs", KSTAT_KV_U_PACKETS, 0 },
3708 [em_stat_rlec] =
3709 { "rx len errs", KSTAT_KV_U_PACKETS, E1000_RLEC },
3710 [em_stat_xonrxc] =
3711 { "rx xon", KSTAT_KV_U_PACKETS, E1000_XONRXC },
3712 [em_stat_xontxc] =
3713 { "tx xon", KSTAT_KV_U_PACKETS, E1000_XONTXC },
3714 [em_stat_xoffrxc] =
3715 { "rx xoff", KSTAT_KV_U_PACKETS, E1000_XOFFRXC },
3716 [em_stat_xofftxc] =
3717 { "tx xoff", KSTAT_KV_U_PACKETS, E1000_XOFFTXC },
3718 [em_stat_fcruc] =
3719 { "FC unsupported", KSTAT_KV_U_PACKETS, E1000_FCRUC },
3720 [em_stat_prc64] =
3721 { "rx 64B", KSTAT_KV_U_PACKETS, E1000_PRC64 },
3722 [em_stat_prc127] =
3723 { "rx 65-127B", KSTAT_KV_U_PACKETS, E1000_PRC127 },
3724 [em_stat_prc255] =
3725 { "rx 128-255B", KSTAT_KV_U_PACKETS, E1000_PRC255 },
3726 [em_stat_prc511] =
3727 { "rx 256-511B", KSTAT_KV_U_PACKETS, E1000_PRC511 },
3728 [em_stat_prc1023] =
3729 { "rx 512-1023B", KSTAT_KV_U_PACKETS, E1000_PRC1023 },
3730 [em_stat_prc1522] =
3731 { "rx 1024-maxB", KSTAT_KV_U_PACKETS, E1000_PRC1522 },
3732 [em_stat_gprc] =
3733 { "rx good", KSTAT_KV_U_PACKETS, E1000_GPRC },
3734 [em_stat_bprc] =
3735 { "rx bcast", KSTAT_KV_U_PACKETS, E1000_BPRC },
3736 [em_stat_mprc] =
3737 { "rx mcast", KSTAT_KV_U_PACKETS, E1000_MPRC },
3738 [em_stat_gptc] =
3739 { "tx good", KSTAT_KV_U_PACKETS, E1000_GPTC },
3740 [em_stat_gorc] = /* 64bit */
3741 { "rx good", KSTAT_KV_U_BYTES, 0 },
3742 [em_stat_gotc] = /* 64bit */
3743 { "tx good", KSTAT_KV_U_BYTES, 0 },
3744 [em_stat_rnbc] =
3745 { "rx no buffers", KSTAT_KV_U_PACKETS, E1000_RNBC },
3746 [em_stat_ruc] =
3747 { "rx undersize", KSTAT_KV_U_PACKETS, E1000_RUC },
3748 [em_stat_rfc] =
3749 { "rx fragments", KSTAT_KV_U_PACKETS, E1000_RFC },
3750 [em_stat_roc] =
3751 { "rx oversize", KSTAT_KV_U_PACKETS, E1000_ROC },
3752 [em_stat_rjc] =
3753 { "rx jabbers", KSTAT_KV_U_PACKETS, E1000_RJC },
3754 [em_stat_mgtprc] =
3755 { "rx mgmt", KSTAT_KV_U_PACKETS, E1000_MGTPRC },
3756 [em_stat_mgtpdc] =
3757 { "rx mgmt drops", KSTAT_KV_U_PACKETS, E1000_MGTPDC },
3758 [em_stat_mgtptc] =
3759 { "tx mgmt", KSTAT_KV_U_PACKETS, E1000_MGTPTC },
3760 [em_stat_tor] = /* 64bit */
3761 { "rx total", KSTAT_KV_U_BYTES, 0 },
3762 [em_stat_tot] = /* 64bit */
3763 { "tx total", KSTAT_KV_U_BYTES, 0 },
3764 [em_stat_tpr] =
3765 { "rx total", KSTAT_KV_U_PACKETS, E1000_TPR },
3766 [em_stat_tpt] =
3767 { "tx total", KSTAT_KV_U_PACKETS, E1000_TPT },
3768 [em_stat_ptc64] =
3769 { "tx 64B", KSTAT_KV_U_PACKETS, E1000_PTC64 },
3770 [em_stat_ptc127] =
3771 { "tx 65-127B", KSTAT_KV_U_PACKETS, E1000_PTC127 },
3772 [em_stat_ptc255] =
3773 { "tx 128-255B", KSTAT_KV_U_PACKETS, E1000_PTC255 },
3774 [em_stat_ptc511] =
3775 { "tx 256-511B", KSTAT_KV_U_PACKETS, E1000_PTC511 },
3776 [em_stat_ptc1023] =
3777 { "tx 512-1023B", KSTAT_KV_U_PACKETS, E1000_PTC1023 },
3778 [em_stat_ptc1522] =
3779 { "tx 1024-maxB", KSTAT_KV_U_PACKETS, E1000_PTC1522 },
3780 [em_stat_mptc] =
3781 { "tx mcast", KSTAT_KV_U_PACKETS, E1000_MPTC },
3782 [em_stat_bptc] =
3783 { "tx bcast", KSTAT_KV_U_PACKETS, E1000_BPTC },
3784 };
3785
3786 /**********************************************************************
3787 *
3788 * Update the board statistics counters.
3789 *
3790 **********************************************************************/
3791 int
em_kstat_read(struct kstat * ks)3792 em_kstat_read(struct kstat *ks)
3793 {
3794 struct em_softc *sc = ks->ks_softc;
3795 struct em_hw *hw = &sc->hw;
3796 struct kstat_kv *kvs = ks->ks_data;
3797 uint32_t lo, hi;
3798 unsigned int i;
3799
3800 for (i = 0; i < nitems(em_counters); i++) {
3801 const struct em_counter *c = &em_counters[i];
3802 if (c->reg == 0)
3803 continue;
3804
3805 kstat_kv_u64(&kvs[i]) += EM_READ_REG(hw,
3806 E1000_REG_TR(hw, c->reg)); /* wtf */
3807 }
3808
3809 /* Handle the exceptions. */
3810
3811 if (sc->hw.mac_type >= em_82543) {
3812 kstat_kv_u64(&kvs[em_stat_algnerrc]) +=
3813 E1000_READ_REG(hw, ALGNERRC);
3814 kstat_kv_u64(&kvs[em_stat_rxerrc]) +=
3815 E1000_READ_REG(hw, RXERRC);
3816 kstat_kv_u64(&kvs[em_stat_cexterr]) +=
3817 E1000_READ_REG(hw, CEXTERR);
3818 kstat_kv_u64(&kvs[em_stat_tncrs]) +=
3819 E1000_READ_REG(hw, TNCRS);
3820 #if 0
3821 sc->stats.tsctc +=
3822 E1000_READ_REG(hw, TSCTC);
3823 sc->stats.tsctfc +=
3824 E1000_READ_REG(hw, TSCTFC);
3825 #endif
3826 }
3827
3828 /* For the 64-bit byte counters the low dword must be read first. */
3829 /* Both registers clear on the read of the high dword */
3830
3831 lo = E1000_READ_REG(hw, GORCL);
3832 hi = E1000_READ_REG(hw, GORCH);
3833 kstat_kv_u64(&kvs[em_stat_gorc]) +=
3834 ((uint64_t)hi << 32) | (uint64_t)lo;
3835
3836 lo = E1000_READ_REG(hw, GOTCL);
3837 hi = E1000_READ_REG(hw, GOTCH);
3838 kstat_kv_u64(&kvs[em_stat_gotc]) +=
3839 ((uint64_t)hi << 32) | (uint64_t)lo;
3840
3841 lo = E1000_READ_REG(hw, TORL);
3842 hi = E1000_READ_REG(hw, TORH);
3843 kstat_kv_u64(&kvs[em_stat_tor]) +=
3844 ((uint64_t)hi << 32) | (uint64_t)lo;
3845
3846 lo = E1000_READ_REG(hw, TOTL);
3847 hi = E1000_READ_REG(hw, TOTH);
3848 kstat_kv_u64(&kvs[em_stat_tot]) +=
3849 ((uint64_t)hi << 32) | (uint64_t)lo;
3850
3851 getnanouptime(&ks->ks_updated);
3852
3853 return (0);
3854 }
3855
3856 void
em_kstat_attach(struct em_softc * sc)3857 em_kstat_attach(struct em_softc *sc)
3858 {
3859 struct kstat *ks;
3860 struct kstat_kv *kvs;
3861 unsigned int i;
3862
3863 mtx_init(&sc->kstat_mtx, IPL_SOFTCLOCK);
3864
3865 ks = kstat_create(DEVNAME(sc), 0, "em-stats", 0,
3866 KSTAT_T_KV, 0);
3867 if (ks == NULL)
3868 return;
3869
3870 kvs = mallocarray(nitems(em_counters), sizeof(*kvs),
3871 M_DEVBUF, M_WAITOK|M_ZERO);
3872 for (i = 0; i < nitems(em_counters); i++) {
3873 const struct em_counter *c = &em_counters[i];
3874 kstat_kv_unit_init(&kvs[i], c->name,
3875 KSTAT_KV_T_COUNTER64, c->unit);
3876 }
3877
3878 ks->ks_softc = sc;
3879 ks->ks_data = kvs;
3880 ks->ks_datalen = nitems(em_counters) * sizeof(*kvs);
3881 ks->ks_read = em_kstat_read;
3882 kstat_set_mutex(ks, &sc->kstat_mtx);
3883
3884 kstat_install(ks);
3885 }
3886
3887 /******************************************************************************
3888 * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
3889 *****************************************************************************/
3890 void
em_tbi_adjust_stats(struct em_softc * sc,uint32_t frame_len,uint8_t * mac_addr)3891 em_tbi_adjust_stats(struct em_softc *sc, uint32_t frame_len, uint8_t *mac_addr)
3892 {
3893 struct em_hw *hw = &sc->hw;
3894 struct kstat *ks = sc->kstat;
3895 struct kstat_kv *kvs;
3896
3897 if (ks == NULL)
3898 return;
3899
3900 /* First adjust the frame length. */
3901 frame_len--;
3902
3903 mtx_enter(&sc->kstat_mtx);
3904 kvs = ks->ks_data;
3905
3906 /*
3907 * We need to adjust the statistics counters, since the hardware
3908 * counters overcount this packet as a CRC error and undercount the
3909 * packet as a good packet
3910 */
3911
3912 /* This packet should not be counted as a CRC error. */
3913 kstat_kv_u64(&kvs[em_stat_crcerrs])--;
3914 /* This packet does count as a Good Packet Received. */
3915 kstat_kv_u64(&kvs[em_stat_gprc])++;
3916
3917 /* Adjust the Good Octets received counters */
3918 kstat_kv_u64(&kvs[em_stat_gorc]) += frame_len;
3919
3920 /*
3921 * Is this a broadcast or multicast? Check broadcast first, since
3922 * the test for a multicast frame will test positive on a broadcast
3923 * frame.
3924 */
3925 if (ETHER_IS_BROADCAST(mac_addr)) {
3926 /* Broadcast packet */
3927 kstat_kv_u64(&kvs[em_stat_bprc])++;
3928 } else if (ETHER_IS_MULTICAST(mac_addr)) {
3929 /* Multicast packet */
3930 kstat_kv_u64(&kvs[em_stat_mprc])++;
3931 }
3932
3933 if (frame_len == hw->max_frame_size) {
3934 /*
3935 * In this case, the hardware has overcounted the number of
3936 * oversize frames.
3937 */
3938 kstat_kv_u64(&kvs[em_stat_roc])--;
3939 }
3940
3941 /*
3942 * Adjust the bin counters when the extra byte put the frame in the
3943 * wrong bin. Remember that the frame_len was adjusted above.
3944 */
3945 if (frame_len == 64) {
3946 kstat_kv_u64(&kvs[em_stat_prc64])++;
3947 kstat_kv_u64(&kvs[em_stat_prc127])--;
3948 } else if (frame_len == 127) {
3949 kstat_kv_u64(&kvs[em_stat_prc127])++;
3950 kstat_kv_u64(&kvs[em_stat_prc255])--;
3951 } else if (frame_len == 255) {
3952 kstat_kv_u64(&kvs[em_stat_prc255])++;
3953 kstat_kv_u64(&kvs[em_stat_prc511])--;
3954 } else if (frame_len == 511) {
3955 kstat_kv_u64(&kvs[em_stat_prc511])++;
3956 kstat_kv_u64(&kvs[em_stat_prc1023])--;
3957 } else if (frame_len == 1023) {
3958 kstat_kv_u64(&kvs[em_stat_prc1023])++;
3959 kstat_kv_u64(&kvs[em_stat_prc1522])--;
3960 } else if (frame_len == 1522) {
3961 kstat_kv_u64(&kvs[em_stat_prc1522])++;
3962 }
3963
3964 mtx_leave(&sc->kstat_mtx);
3965 }
3966 #endif /* NKSTAT > 0 */
3967
3968 #ifndef SMALL_KERNEL
3969 int
em_allocate_msix(struct em_softc * sc)3970 em_allocate_msix(struct em_softc *sc)
3971 {
3972 pci_intr_handle_t ih;
3973 const char *intrstr = NULL;
3974 struct pci_attach_args *pa = &sc->osdep.em_pa;
3975 pci_chipset_tag_t pc = pa->pa_pc;
3976 struct em_queue *que = sc->queues; /* Use only first queue. */
3977 int vec;
3978
3979 if (!em_enable_msix)
3980 return (ENODEV);
3981
3982 switch (sc->hw.mac_type) {
3983 case em_82576:
3984 case em_82580:
3985 case em_i350:
3986 case em_i210:
3987 break;
3988 default:
3989 return (ENODEV);
3990 }
3991
3992 vec = 0;
3993 if (pci_intr_map_msix(pa, vec, &ih))
3994 return (ENODEV);
3995 sc->msix = 1;
3996
3997 que->me = vec;
3998 que->eims = 1 << vec;
3999 snprintf(que->name, sizeof(que->name), "%s:%d", DEVNAME(sc), vec);
4000
4001 intrstr = pci_intr_string(pc, ih);
4002 que->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
4003 em_queue_intr_msix, que, que->name);
4004 if (que->tag == NULL) {
4005 printf(": couldn't establish interrupt");
4006 if (intrstr != NULL)
4007 printf(" at %s", intrstr);
4008 printf("\n");
4009 return (ENXIO);
4010 }
4011
4012 /* Setup linkvector, use last queue vector + 1 */
4013 vec++;
4014 sc->msix_linkvec = vec;
4015 if (pci_intr_map_msix(pa, sc->msix_linkvec, &ih)) {
4016 printf(": couldn't map link vector\n");
4017 return (ENXIO);
4018 }
4019
4020 intrstr = pci_intr_string(pc, ih);
4021 sc->sc_intrhand = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
4022 em_link_intr_msix, sc, DEVNAME(sc));
4023 if (sc->sc_intrhand == NULL) {
4024 printf(": couldn't establish interrupt");
4025 if (intrstr != NULL)
4026 printf(" at %s", intrstr);
4027 printf("\n");
4028 return (ENXIO);
4029 }
4030 printf(", %s, %d queue%s", intrstr, vec, (vec > 1) ? "s" : "");
4031
4032 return (0);
4033 }
4034
4035 /*
4036 * Interrupt for a specific queue, (not link interrupts). The EICR bit which
4037 * maps to the EIMS bit expresses both RX and TX, therefore we can't
4038 * distinguish if this is a RX completion of TX completion and must do both.
4039 * The bits in EICR are autocleared and we _cannot_ read EICR.
4040 */
4041 int
em_queue_intr_msix(void * vque)4042 em_queue_intr_msix(void *vque)
4043 {
4044 struct em_queue *que = vque;
4045 struct em_softc *sc = que->sc;
4046 struct ifnet *ifp = &sc->sc_ac.ac_if;
4047
4048 if (ifp->if_flags & IFF_RUNNING) {
4049 em_txeof(que);
4050 if (em_rxeof(que))
4051 em_rxrefill_locked(que);
4052 }
4053
4054 em_enable_queue_intr_msix(que);
4055
4056 return (1);
4057 }
4058
4059 int
em_link_intr_msix(void * arg)4060 em_link_intr_msix(void *arg)
4061 {
4062 struct em_softc *sc = arg;
4063 uint32_t icr;
4064
4065 icr = E1000_READ_REG(&sc->hw, ICR);
4066
4067 /* Link status change */
4068 if (icr & E1000_ICR_LSC) {
4069 KERNEL_LOCK();
4070 sc->hw.get_link_status = 1;
4071 em_check_for_link(&sc->hw);
4072 em_update_link_status(sc);
4073 KERNEL_UNLOCK();
4074 }
4075
4076 /* Re-arm unconditionally */
4077 E1000_WRITE_REG(&sc->hw, IMS, E1000_ICR_LSC);
4078 E1000_WRITE_REG(&sc->hw, EIMS, sc->msix_linkmask);
4079
4080 return (1);
4081 }
4082
4083 /*
4084 * Maps queues into msix interrupt vectors.
4085 */
4086 int
em_setup_queues_msix(struct em_softc * sc)4087 em_setup_queues_msix(struct em_softc *sc)
4088 {
4089 uint32_t ivar, newitr, index;
4090 struct em_queue *que;
4091
4092 KASSERT(sc->msix);
4093
4094 /* First turn on RSS capability */
4095 if (sc->hw.mac_type != em_82575)
4096 E1000_WRITE_REG(&sc->hw, GPIE,
4097 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
4098 E1000_GPIE_PBA | E1000_GPIE_NSICR);
4099
4100 /* Turn on MSIX */
4101 switch (sc->hw.mac_type) {
4102 case em_82580:
4103 case em_i350:
4104 case em_i210:
4105 /* RX entries */
4106 /*
4107 * Note, this maps Queues into MSIX vectors, it works fine.
4108 * The funky calculation of offsets and checking if que->me is
4109 * odd is due to the weird register distribution, the datasheet
4110 * explains it well.
4111 */
4112 FOREACH_QUEUE(sc, que) {
4113 index = que->me >> 1;
4114 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4115 if (que->me & 1) {
4116 ivar &= 0xFF00FFFF;
4117 ivar |= (que->me | E1000_IVAR_VALID) << 16;
4118 } else {
4119 ivar &= 0xFFFFFF00;
4120 ivar |= que->me | E1000_IVAR_VALID;
4121 }
4122 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4123 }
4124
4125 /* TX entries */
4126 FOREACH_QUEUE(sc, que) {
4127 index = que->me >> 1;
4128 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4129 if (que->me & 1) {
4130 ivar &= 0x00FFFFFF;
4131 ivar |= (que->me | E1000_IVAR_VALID) << 24;
4132 } else {
4133 ivar &= 0xFFFF00FF;
4134 ivar |= (que->me | E1000_IVAR_VALID) << 8;
4135 }
4136 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4137 sc->msix_queuesmask |= que->eims;
4138 }
4139
4140 /* And for the link interrupt */
4141 ivar = (sc->msix_linkvec | E1000_IVAR_VALID) << 8;
4142 sc->msix_linkmask = 1 << sc->msix_linkvec;
4143 E1000_WRITE_REG(&sc->hw, IVAR_MISC, ivar);
4144 break;
4145 case em_82576:
4146 /* RX entries */
4147 FOREACH_QUEUE(sc, que) {
4148 index = que->me & 0x7; /* Each IVAR has two entries */
4149 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4150 if (que->me < 8) {
4151 ivar &= 0xFFFFFF00;
4152 ivar |= que->me | E1000_IVAR_VALID;
4153 } else {
4154 ivar &= 0xFF00FFFF;
4155 ivar |= (que->me | E1000_IVAR_VALID) << 16;
4156 }
4157 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4158 sc->msix_queuesmask |= que->eims;
4159 }
4160 /* TX entries */
4161 FOREACH_QUEUE(sc, que) {
4162 index = que->me & 0x7; /* Each IVAR has two entries */
4163 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4164 if (que->me < 8) {
4165 ivar &= 0xFFFF00FF;
4166 ivar |= (que->me | E1000_IVAR_VALID) << 8;
4167 } else {
4168 ivar &= 0x00FFFFFF;
4169 ivar |= (que->me | E1000_IVAR_VALID) << 24;
4170 }
4171 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4172 sc->msix_queuesmask |= que->eims;
4173 }
4174
4175 /* And for the link interrupt */
4176 ivar = (sc->msix_linkvec | E1000_IVAR_VALID) << 8;
4177 sc->msix_linkmask = 1 << sc->msix_linkvec;
4178 E1000_WRITE_REG(&sc->hw, IVAR_MISC, ivar);
4179 break;
4180 default:
4181 panic("unsupported mac");
4182 break;
4183 }
4184
4185 /* Set the starting interrupt rate */
4186 newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC;
4187
4188 if (sc->hw.mac_type == em_82575)
4189 newitr |= newitr << 16;
4190 else
4191 newitr |= E1000_EITR_CNT_IGNR;
4192
4193 FOREACH_QUEUE(sc, que)
4194 E1000_WRITE_REG(&sc->hw, EITR(que->me), newitr);
4195
4196 return (0);
4197 }
4198
4199 void
em_enable_queue_intr_msix(struct em_queue * que)4200 em_enable_queue_intr_msix(struct em_queue *que)
4201 {
4202 E1000_WRITE_REG(&que->sc->hw, EIMS, que->eims);
4203 }
4204 #endif /* !SMALL_KERNEL */
4205
4206 int
em_allocate_desc_rings(struct em_softc * sc)4207 em_allocate_desc_rings(struct em_softc *sc)
4208 {
4209 struct em_queue *que;
4210
4211 FOREACH_QUEUE(sc, que) {
4212 /* Allocate Transmit Descriptor ring */
4213 if (em_dma_malloc(sc, sc->sc_tx_slots * sizeof(struct em_tx_desc),
4214 &que->tx.sc_tx_dma) != 0) {
4215 printf("%s: Unable to allocate tx_desc memory\n",
4216 DEVNAME(sc));
4217 return (ENOMEM);
4218 }
4219 que->tx.sc_tx_desc_ring =
4220 (struct em_tx_desc *)que->tx.sc_tx_dma.dma_vaddr;
4221
4222 /* Allocate Receive Descriptor ring */
4223 if (em_dma_malloc(sc, sc->sc_rx_slots * sizeof(struct em_rx_desc),
4224 &que->rx.sc_rx_dma) != 0) {
4225 printf("%s: Unable to allocate rx_desc memory\n",
4226 DEVNAME(sc));
4227 return (ENOMEM);
4228 }
4229 que->rx.sc_rx_desc_ring =
4230 (struct em_rx_desc *)que->rx.sc_rx_dma.dma_vaddr;
4231 }
4232
4233 return (0);
4234 }
4235
4236 int
em_get_sffpage(struct em_softc * sc,struct if_sffpage * sff)4237 em_get_sffpage(struct em_softc *sc, struct if_sffpage *sff)
4238 {
4239 struct em_hw *hw = &sc->hw;
4240 size_t i;
4241 int off;
4242
4243 if (hw->mac_type != em_82575 && hw->mac_type != em_82580 &&
4244 hw->mac_type != em_82576 &&
4245 hw->mac_type != em_i210 && hw->mac_type != em_i350)
4246 return (ENODEV);
4247
4248 if (sff->sff_addr == IFSFF_ADDR_EEPROM)
4249 off = E1000_I2CCMD_SFP_DATA_ADDR(0);
4250 else if (sff->sff_addr == IFSFF_ADDR_DDM)
4251 off = E1000_I2CCMD_SFP_DIAG_ADDR(0);
4252 else
4253 return (EIO);
4254
4255 for (i = 0; i < sizeof(sff->sff_data); i++) {
4256 if (em_read_sfp_data_byte(hw, off + i,
4257 &sff->sff_data[i]) != E1000_SUCCESS)
4258 return (EIO);
4259 }
4260
4261 return (0);
4262 }
4263