1 /**************************************************************************
2
3 Copyright (c) 2001-2003, Intel Corporation
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10 this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in the
14 documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17 contributors may be used to endorse or promote products derived from
18 this software without specific prior written permission.
19
20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 POSSIBILITY OF SUCH DAMAGE.
31
32 ***************************************************************************/
33
34 /* $OpenBSD: if_em.c,v 1.378 2024/08/31 16:23:09 deraadt Exp $ */
35 /* $FreeBSD: if_em.c,v 1.46 2004/09/29 18:28:28 mlaier Exp $ */
36
37 #include <dev/pci/if_em.h>
38 #include <dev/pci/if_em_soc.h>
39
40 /*********************************************************************
41 * Driver version
42 *********************************************************************/
43
44 #define EM_DRIVER_VERSION "6.2.9"
45
46 /*********************************************************************
47 * PCI Device ID Table
48 *********************************************************************/
49 const struct pci_matchid em_devices[] = {
50 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_CPR_DPT },
51 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_SDS_DPT },
52 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_CPR_SPT },
53 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_80003ES2LAN_SDS_SPT },
54 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EM },
55 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EM_LOM },
56 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EP },
57 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EP_LOM },
58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82540EP_LP },
59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541EI },
60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541EI_MOBILE },
61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541ER },
62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541ER_LOM },
63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541GI },
64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541GI_LF },
65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82541GI_MOBILE },
66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82542 },
67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82543GC_COPPER },
68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82543GC_FIBER },
69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544EI_COPPER },
70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544EI_FIBER },
71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544GC_COPPER },
72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82544GC_LOM },
73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545EM_COPPER },
74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545EM_FIBER },
75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545GM_COPPER },
76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545GM_FIBER },
77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82545GM_SERDES },
78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546EB_COPPER },
79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546EB_FIBER },
80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546EB_QUAD_CPR },
81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_COPPER },
82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_FIBER },
83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_PCIE },
84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_QUAD_CPR },
85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_QUAD_CPR_K },
86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_SERDES },
87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82546GB_2 },
88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82547EI },
89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82547EI_MOBILE },
90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82547GI },
91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_AF },
92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_AT },
93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_COPPER },
94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_FIBER },
95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_QUAD_CPR },
96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_QUAD_CPR_LP },
97 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_QUAD_FBR },
98 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_SERDES },
99 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_SDS_DUAL },
100 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571EB_SDS_QUAD },
101 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82571PT_QUAD_CPR },
102 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI_COPPER },
103 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI_FIBER },
104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI_SERDES },
105 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82572EI },
106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573E },
107 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573E_IAMT },
108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573E_PM },
109 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573L },
110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573L_PL_1 },
111 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573L_PL_2 },
112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82573V_PM },
113 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82574L },
114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82574LA },
115 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575EB_COPPER },
116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575EB_SERDES },
117 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575GB_QUAD_CPR },
118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82575GB_QP_PM },
119 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576 },
120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_FIBER },
121 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_SERDES },
122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_QUAD_COPPER },
123 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_QUAD_CU_ET2 },
124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_NS },
125 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_NS_SERDES },
126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82576_SERDES_QUAD },
127 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82577LC },
128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82577LM },
129 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82578DC },
130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82578DM },
131 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82579LM },
132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82579V },
133 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER },
134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER_OEM1 },
135 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER_IT },
136 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_FIBER },
137 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_SERDES },
138 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_SGMII },
139 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_COPPER_NF },
140 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I210_SERDES_NF },
141 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I211_COPPER },
142 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I217_LM },
143 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I217_V },
144 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_LM },
145 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_LM_2 },
146 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_LM_3 },
147 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_V },
148 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_V_2 },
149 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I218_V_3 },
150 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM },
151 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM2 },
152 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM3 },
153 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM4 },
154 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM5 },
155 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM6 },
156 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM7 },
157 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM8 },
158 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM9 },
159 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM10 },
160 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM11 },
161 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM12 },
162 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM13 },
163 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM14 },
164 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM15 },
165 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM16 },
166 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM17 },
167 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM18 },
168 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM19 },
169 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM20 },
170 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM21 },
171 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM22 },
172 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM23 },
173 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_LM24 },
174 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V },
175 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V2 },
176 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V4 },
177 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V5 },
178 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V6 },
179 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V7 },
180 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V8 },
181 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V9 },
182 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V10 },
183 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V11 },
184 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V12 },
185 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V13 },
186 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V14 },
187 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V15 },
188 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V16 },
189 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V17 },
190 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V18 },
191 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V19 },
192 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V20 },
193 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V21 },
194 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V22 },
195 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V23 },
196 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I219_V24 },
197 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_COPPER },
198 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_FIBER },
199 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_SERDES },
200 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_SGMII },
201 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_COPPER_DUAL },
202 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82580_QUAD_FIBER },
203 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_SGMII },
204 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_SERDES },
205 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_BPLANE },
206 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DH89XXCC_SFP },
207 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82583V },
208 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_COPPER },
209 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_FIBER },
210 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_SERDES },
211 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I350_SGMII },
212 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I354_BP_1GBPS },
213 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I354_BP_2_5GBPS },
214 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I354_SGMII },
215 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_82567V_3 },
216 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IFE },
217 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IFE_G },
218 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IFE_GT },
219 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_AMT },
220 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_C },
221 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_M },
222 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH8_IGP_M_AMT },
223 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_BM },
224 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IFE },
225 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IFE_G },
226 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IFE_GT },
227 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_AMT },
228 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_C },
229 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_M },
230 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_M_AMT },
231 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH9_IGP_M_V },
232 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_D_BM_LF },
233 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_D_BM_LM },
234 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_D_BM_V },
235 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_R_BM_LF },
236 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_R_BM_LM },
237 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_ICH10_R_BM_V },
238 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_1 },
239 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_2 },
240 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_3 },
241 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_4 },
242 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_5 },
243 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EP80579_LAN_6 }
244 };
245
246 /*********************************************************************
247 * Function prototypes
248 *********************************************************************/
249 int em_probe(struct device *, void *, void *);
250 void em_attach(struct device *, struct device *, void *);
251 void em_defer_attach(struct device*);
252 int em_detach(struct device *, int);
253 int em_activate(struct device *, int);
254 int em_intr(void *);
255 int em_allocate_legacy(struct em_softc *);
256 void em_start(struct ifqueue *);
257 int em_ioctl(struct ifnet *, u_long, caddr_t);
258 void em_watchdog(struct ifnet *);
259 void em_init(void *);
260 void em_stop(void *, int);
261 void em_media_status(struct ifnet *, struct ifmediareq *);
262 int em_media_change(struct ifnet *);
263 uint64_t em_flowstatus(struct em_softc *);
264 void em_identify_hardware(struct em_softc *);
265 int em_allocate_pci_resources(struct em_softc *);
266 void em_free_pci_resources(struct em_softc *);
267 void em_local_timer(void *);
268 int em_hardware_init(struct em_softc *);
269 void em_setup_interface(struct em_softc *);
270 int em_setup_transmit_structures(struct em_softc *);
271 void em_initialize_transmit_unit(struct em_softc *);
272 int em_setup_receive_structures(struct em_softc *);
273 void em_initialize_receive_unit(struct em_softc *);
274 void em_enable_intr(struct em_softc *);
275 void em_disable_intr(struct em_softc *);
276 void em_free_transmit_structures(struct em_softc *);
277 void em_free_receive_structures(struct em_softc *);
278 void em_disable_aspm(struct em_softc *);
279 void em_txeof(struct em_queue *);
280 int em_allocate_receive_structures(struct em_softc *);
281 int em_allocate_transmit_structures(struct em_softc *);
282 int em_allocate_desc_rings(struct em_softc *);
283 int em_rxfill(struct em_queue *);
284 void em_rxrefill(void *);
285 void em_rxrefill_locked(struct em_queue *);
286 int em_rxeof(struct em_queue *);
287 void em_receive_checksum(struct em_softc *, struct em_rx_desc *,
288 struct mbuf *);
289 u_int em_transmit_checksum_setup(struct em_queue *, struct mbuf *, u_int,
290 u_int32_t *, u_int32_t *);
291 u_int em_tso_setup(struct em_queue *, struct mbuf *, u_int, u_int32_t *,
292 u_int32_t *);
293 u_int em_tx_ctx_setup(struct em_queue *, struct mbuf *, u_int, u_int32_t *,
294 u_int32_t *);
295 void em_iff(struct em_softc *);
296 void em_update_link_status(struct em_softc *);
297 int em_get_buf(struct em_queue *, int);
298 void em_enable_hw_vlans(struct em_softc *);
299 u_int em_encap(struct em_queue *, struct mbuf *);
300 void em_smartspeed(struct em_softc *);
301 int em_82547_fifo_workaround(struct em_softc *, int);
302 void em_82547_update_fifo_head(struct em_softc *, int);
303 int em_82547_tx_fifo_reset(struct em_softc *);
304 void em_82547_move_tail(void *arg);
305 void em_82547_move_tail_locked(struct em_softc *);
306 int em_dma_malloc(struct em_softc *, bus_size_t, struct em_dma_alloc *);
307 void em_dma_free(struct em_softc *, struct em_dma_alloc *);
308 u_int32_t em_fill_descriptors(u_int64_t address, u_int32_t length,
309 PDESC_ARRAY desc_array);
310 void em_flush_tx_ring(struct em_queue *);
311 void em_flush_rx_ring(struct em_queue *);
312 void em_flush_desc_rings(struct em_softc *);
313 int em_get_sffpage(struct em_softc *, struct if_sffpage *);
314
315 #ifndef SMALL_KERNEL
316 /* MSIX/Multiqueue functions */
317 int em_allocate_msix(struct em_softc *);
318 int em_setup_queues_msix(struct em_softc *);
319 int em_queue_intr_msix(void *);
320 int em_link_intr_msix(void *);
321 void em_enable_queue_intr_msix(struct em_queue *);
322 #else
323 #define em_allocate_msix(_sc) (-1)
324 #endif
325
326 #if NKSTAT > 0
327 void em_kstat_attach(struct em_softc *);
328 int em_kstat_read(struct kstat *);
329 void em_tbi_adjust_stats(struct em_softc *, uint32_t, uint8_t *);
330 #endif
331
332 /*********************************************************************
333 * OpenBSD Device Interface Entry Points
334 *********************************************************************/
335
336 const struct cfattach em_ca = {
337 sizeof(struct em_softc), em_probe, em_attach, em_detach,
338 em_activate
339 };
340
341 struct cfdriver em_cd = {
342 NULL, "em", DV_IFNET
343 };
344
345 static int em_smart_pwr_down = FALSE;
346 int em_enable_msix = 0;
347
348 /*********************************************************************
349 * Device identification routine
350 *
351 * em_probe determines if the driver should be loaded on
352 * adapter based on PCI vendor/device id of the adapter.
353 *
354 * return 0 on no match, positive on match
355 *********************************************************************/
356
357 int
em_probe(struct device * parent,void * match,void * aux)358 em_probe(struct device *parent, void *match, void *aux)
359 {
360 INIT_DEBUGOUT("em_probe: begin");
361
362 return (pci_matchbyid((struct pci_attach_args *)aux, em_devices,
363 nitems(em_devices)));
364 }
365
366 void
em_defer_attach(struct device * self)367 em_defer_attach(struct device *self)
368 {
369 struct em_softc *sc = (struct em_softc *)self;
370 struct pci_attach_args *pa = &sc->osdep.em_pa;
371 pci_chipset_tag_t pc = pa->pa_pc;
372 void *gcu;
373
374 INIT_DEBUGOUT("em_defer_attach: begin");
375
376 if ((gcu = em_lookup_gcu(self)) == 0) {
377 printf("%s: No GCU found, deferred attachment failed\n",
378 DEVNAME(sc));
379
380 if (sc->sc_intrhand)
381 pci_intr_disestablish(pc, sc->sc_intrhand);
382 sc->sc_intrhand = 0;
383
384 em_stop(sc, 1);
385
386 em_free_pci_resources(sc);
387
388 return;
389 }
390
391 sc->hw.gcu = gcu;
392
393 em_attach_miibus(self);
394
395 em_setup_interface(sc);
396
397 em_setup_link(&sc->hw);
398
399 em_update_link_status(sc);
400 }
401
402 /*********************************************************************
403 * Device initialization routine
404 *
405 * The attach entry point is called when the driver is being loaded.
406 * This routine identifies the type of hardware, allocates all resources
407 * and initializes the hardware.
408 *
409 *********************************************************************/
410
411 void
em_attach(struct device * parent,struct device * self,void * aux)412 em_attach(struct device *parent, struct device *self, void *aux)
413 {
414 struct pci_attach_args *pa = aux;
415 struct em_softc *sc;
416 int defer = 0;
417
418 INIT_DEBUGOUT("em_attach: begin");
419
420 sc = (struct em_softc *)self;
421 sc->sc_dmat = pa->pa_dmat;
422 sc->osdep.em_pa = *pa;
423
424 timeout_set(&sc->timer_handle, em_local_timer, sc);
425 timeout_set(&sc->tx_fifo_timer_handle, em_82547_move_tail, sc);
426
427 rw_init(&sc->sfflock, "emsff");
428
429 /* Determine hardware revision */
430 em_identify_hardware(sc);
431
432 /*
433 * Only use MSI on the newer PCIe parts, with the exception
434 * of 82571/82572 due to "Byte Enables 2 and 3 Are Not Set" errata
435 */
436 if (sc->hw.mac_type <= em_82572)
437 sc->osdep.em_pa.pa_flags &= ~PCI_FLAGS_MSI_ENABLED;
438
439 /* Parameters (to be read from user) */
440 if (sc->hw.mac_type >= em_82544) {
441 sc->sc_tx_slots = EM_MAX_TXD;
442 sc->sc_rx_slots = EM_MAX_RXD;
443 } else {
444 sc->sc_tx_slots = EM_MAX_TXD_82543;
445 sc->sc_rx_slots = EM_MAX_RXD_82543;
446 }
447 sc->tx_int_delay = EM_TIDV;
448 sc->tx_abs_int_delay = EM_TADV;
449 sc->rx_int_delay = EM_RDTR;
450 sc->rx_abs_int_delay = EM_RADV;
451 sc->hw.autoneg = DO_AUTO_NEG;
452 sc->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
453 sc->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
454 sc->hw.tbi_compatibility_en = TRUE;
455 sc->sc_rx_buffer_len = EM_RXBUFFER_2048;
456
457 sc->hw.phy_init_script = 1;
458 sc->hw.phy_reset_disable = FALSE;
459
460 #ifndef EM_MASTER_SLAVE
461 sc->hw.master_slave = em_ms_hw_default;
462 #else
463 sc->hw.master_slave = EM_MASTER_SLAVE;
464 #endif
465
466 /*
467 * This controls when hardware reports transmit completion
468 * status.
469 */
470 sc->hw.report_tx_early = 1;
471
472 if (em_allocate_pci_resources(sc))
473 goto err_pci;
474
475 /* Initialize eeprom parameters */
476 em_init_eeprom_params(&sc->hw);
477
478 /*
479 * Set the max frame size assuming standard Ethernet
480 * sized frames.
481 */
482 switch (sc->hw.mac_type) {
483 case em_82573:
484 {
485 uint16_t eeprom_data = 0;
486
487 /*
488 * 82573 only supports Jumbo frames
489 * if ASPM is disabled.
490 */
491 em_read_eeprom(&sc->hw, EEPROM_INIT_3GIO_3,
492 1, &eeprom_data);
493 if (eeprom_data & EEPROM_WORD1A_ASPM_MASK) {
494 sc->hw.max_frame_size = ETHER_MAX_LEN;
495 break;
496 }
497 /* Allow Jumbo frames */
498 /* FALLTHROUGH */
499 }
500 case em_82571:
501 case em_82572:
502 case em_82574:
503 case em_82575:
504 case em_82576:
505 case em_82580:
506 case em_i210:
507 case em_i350:
508 case em_ich9lan:
509 case em_ich10lan:
510 case em_pch2lan:
511 case em_pch_lpt:
512 case em_pch_spt:
513 case em_pch_cnp:
514 case em_pch_tgp:
515 case em_pch_adp:
516 case em_80003es2lan:
517 /* 9K Jumbo Frame size */
518 sc->hw.max_frame_size = 9234;
519 break;
520 case em_pchlan:
521 sc->hw.max_frame_size = 4096;
522 break;
523 case em_82542_rev2_0:
524 case em_82542_rev2_1:
525 case em_ich8lan:
526 /* Adapters that do not support Jumbo frames */
527 sc->hw.max_frame_size = ETHER_MAX_LEN;
528 break;
529 default:
530 sc->hw.max_frame_size =
531 MAX_JUMBO_FRAME_SIZE;
532 }
533
534 sc->hw.min_frame_size =
535 ETHER_MIN_LEN + ETHER_CRC_LEN;
536
537 if (em_allocate_desc_rings(sc) != 0) {
538 printf("%s: Unable to allocate descriptor ring memory\n",
539 DEVNAME(sc));
540 goto err_pci;
541 }
542
543 /* Initialize the hardware */
544 if ((defer = em_hardware_init(sc))) {
545 if (defer == EAGAIN)
546 config_defer(self, em_defer_attach);
547 else {
548 printf("%s: Unable to initialize the hardware\n",
549 DEVNAME(sc));
550 goto err_pci;
551 }
552 }
553
554 if (sc->hw.mac_type == em_80003es2lan || sc->hw.mac_type == em_82575 ||
555 sc->hw.mac_type == em_82576 ||
556 sc->hw.mac_type == em_82580 || sc->hw.mac_type == em_i210 ||
557 sc->hw.mac_type == em_i350) {
558 uint32_t reg = EM_READ_REG(&sc->hw, E1000_STATUS);
559 sc->hw.bus_func = (reg & E1000_STATUS_FUNC_MASK) >>
560 E1000_STATUS_FUNC_SHIFT;
561
562 switch (sc->hw.bus_func) {
563 case 0:
564 sc->hw.swfw = E1000_SWFW_PHY0_SM;
565 break;
566 case 1:
567 sc->hw.swfw = E1000_SWFW_PHY1_SM;
568 break;
569 case 2:
570 sc->hw.swfw = E1000_SWFW_PHY2_SM;
571 break;
572 case 3:
573 sc->hw.swfw = E1000_SWFW_PHY3_SM;
574 break;
575 }
576 } else {
577 sc->hw.bus_func = 0;
578 }
579
580 /* Copy the permanent MAC address out of the EEPROM */
581 if (em_read_mac_addr(&sc->hw) < 0) {
582 printf("%s: EEPROM read error while reading mac address\n",
583 DEVNAME(sc));
584 goto err_pci;
585 }
586
587 bcopy(sc->hw.mac_addr, sc->sc_ac.ac_enaddr, ETHER_ADDR_LEN);
588
589 /* Setup OS specific network interface */
590 if (!defer)
591 em_setup_interface(sc);
592
593 /* Initialize statistics */
594 em_clear_hw_cntrs(&sc->hw);
595 #if NKSTAT > 0
596 em_kstat_attach(sc);
597 #endif
598 sc->hw.get_link_status = 1;
599 if (!defer)
600 em_update_link_status(sc);
601
602 #ifdef EM_DEBUG
603 printf(", mac %#x phy %#x", sc->hw.mac_type, sc->hw.phy_type);
604 #endif
605 printf(", address %s\n", ether_sprintf(sc->sc_ac.ac_enaddr));
606
607 /* Indicate SOL/IDER usage */
608 if (em_check_phy_reset_block(&sc->hw))
609 printf("%s: PHY reset is blocked due to SOL/IDER session.\n",
610 DEVNAME(sc));
611
612 /* Identify 82544 on PCI-X */
613 em_get_bus_info(&sc->hw);
614 if (sc->hw.bus_type == em_bus_type_pcix &&
615 sc->hw.mac_type == em_82544)
616 sc->pcix_82544 = TRUE;
617 else
618 sc->pcix_82544 = FALSE;
619
620 sc->hw.icp_xxxx_is_link_up = FALSE;
621
622 INIT_DEBUGOUT("em_attach: end");
623 return;
624
625 err_pci:
626 em_free_pci_resources(sc);
627 }
628
629 /*********************************************************************
630 * Transmit entry point
631 *
632 * em_start is called by the stack to initiate a transmit.
633 * The driver will remain in this routine as long as there are
634 * packets to transmit and transmit resources are available.
635 * In case resources are not available stack is notified and
636 * the packet is requeued.
637 **********************************************************************/
638
639 void
em_start(struct ifqueue * ifq)640 em_start(struct ifqueue *ifq)
641 {
642 struct ifnet *ifp = ifq->ifq_if;
643 struct em_softc *sc = ifp->if_softc;
644 u_int head, free, used;
645 struct mbuf *m;
646 int post = 0;
647 struct em_queue *que = sc->queues; /* Use only first queue. */
648
649 if (!sc->link_active) {
650 ifq_purge(ifq);
651 return;
652 }
653
654 /* calculate free space */
655 head = que->tx.sc_tx_desc_head;
656 free = que->tx.sc_tx_desc_tail;
657 if (free <= head)
658 free += sc->sc_tx_slots;
659 free -= head;
660
661 if (sc->hw.mac_type != em_82547) {
662 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
663 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
664 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
665 }
666
667 for (;;) {
668 /* use 2 because cksum setup can use an extra slot */
669 if (EM_MAX_SCATTER + 2 > free) {
670 ifq_set_oactive(ifq);
671 break;
672 }
673
674 m = ifq_dequeue(ifq);
675 if (m == NULL)
676 break;
677
678 used = em_encap(que, m);
679 if (used == 0) {
680 m_freem(m);
681 continue;
682 }
683
684 KASSERT(used <= free);
685
686 free -= used;
687
688 #if NBPFILTER > 0
689 /* Send a copy of the frame to the BPF listener */
690 if (ifp->if_bpf)
691 bpf_mtap_ether(ifp->if_bpf, m, BPF_DIRECTION_OUT);
692 #endif
693
694 /* Set timeout in case hardware has problems transmitting */
695 ifp->if_timer = EM_TX_TIMEOUT;
696
697 if (sc->hw.mac_type == em_82547) {
698 int len = m->m_pkthdr.len;
699
700 if (sc->link_duplex == HALF_DUPLEX)
701 em_82547_move_tail_locked(sc);
702 else {
703 E1000_WRITE_REG(&sc->hw, TDT(que->me),
704 que->tx.sc_tx_desc_head);
705 em_82547_update_fifo_head(sc, len);
706 }
707 }
708
709 post = 1;
710 }
711
712 if (sc->hw.mac_type != em_82547) {
713 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
714 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
715 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
716 /*
717 * Advance the Transmit Descriptor Tail (Tdt),
718 * this tells the E1000 that this frame is
719 * available to transmit.
720 */
721 if (post)
722 E1000_WRITE_REG(&sc->hw, TDT(que->me),
723 que->tx.sc_tx_desc_head);
724 }
725 }
726
727 /*********************************************************************
728 * Ioctl entry point
729 *
730 * em_ioctl is called when the user wants to configure the
731 * interface.
732 *
733 * return 0 on success, positive on failure
734 **********************************************************************/
735
736 int
em_ioctl(struct ifnet * ifp,u_long command,caddr_t data)737 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
738 {
739 int error = 0;
740 struct ifreq *ifr = (struct ifreq *) data;
741 struct em_softc *sc = ifp->if_softc;
742 int s;
743
744 s = splnet();
745
746 switch (command) {
747 case SIOCSIFADDR:
748 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFADDR (Set Interface "
749 "Addr)");
750 if (!(ifp->if_flags & IFF_UP)) {
751 ifp->if_flags |= IFF_UP;
752 em_init(sc);
753 }
754 break;
755
756 case SIOCSIFFLAGS:
757 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
758 if (ifp->if_flags & IFF_UP) {
759 if (ifp->if_flags & IFF_RUNNING)
760 error = ENETRESET;
761 else
762 em_init(sc);
763 } else {
764 if (ifp->if_flags & IFF_RUNNING)
765 em_stop(sc, 0);
766 }
767 break;
768
769 case SIOCSIFMEDIA:
770 /* Check SOL/IDER usage */
771 if (em_check_phy_reset_block(&sc->hw)) {
772 printf("%s: Media change is blocked due to SOL/IDER session.\n",
773 DEVNAME(sc));
774 break;
775 }
776 case SIOCGIFMEDIA:
777 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
778 error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
779 break;
780
781 case SIOCGIFRXR:
782 error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
783 NULL, EM_MCLBYTES, &sc->queues->rx.sc_rx_ring);
784 break;
785
786 case SIOCGIFSFFPAGE:
787 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR);
788 if (error != 0)
789 break;
790
791 error = em_get_sffpage(sc, (struct if_sffpage *)data);
792 rw_exit(&sc->sfflock);
793 break;
794
795 default:
796 error = ether_ioctl(ifp, &sc->sc_ac, command, data);
797 }
798
799 if (error == ENETRESET) {
800 if (ifp->if_flags & IFF_RUNNING) {
801 em_disable_intr(sc);
802 em_iff(sc);
803 if (sc->hw.mac_type == em_82542_rev2_0)
804 em_initialize_receive_unit(sc);
805 em_enable_intr(sc);
806 }
807 error = 0;
808 }
809
810 splx(s);
811 return (error);
812 }
813
814 /*********************************************************************
815 * Watchdog entry point
816 *
817 * This routine is called whenever hardware quits transmitting.
818 *
819 **********************************************************************/
820
821 void
em_watchdog(struct ifnet * ifp)822 em_watchdog(struct ifnet *ifp)
823 {
824 struct em_softc *sc = ifp->if_softc;
825 struct em_queue *que = sc->queues; /* Use only first queue. */
826
827
828 /* If we are in this routine because of pause frames, then
829 * don't reset the hardware.
830 */
831 if (E1000_READ_REG(&sc->hw, STATUS) & E1000_STATUS_TXOFF) {
832 ifp->if_timer = EM_TX_TIMEOUT;
833 return;
834 }
835 printf("%s: watchdog: head %u tail %u TDH %u TDT %u\n",
836 DEVNAME(sc),
837 que->tx.sc_tx_desc_head, que->tx.sc_tx_desc_tail,
838 E1000_READ_REG(&sc->hw, TDH(que->me)),
839 E1000_READ_REG(&sc->hw, TDT(que->me)));
840
841 em_init(sc);
842
843 sc->watchdog_events++;
844 }
845
846 /*********************************************************************
847 * Init entry point
848 *
849 * This routine is used in two ways. It is used by the stack as
850 * init entry point in network interface structure. It is also used
851 * by the driver as a hw/sw initialization routine to get to a
852 * consistent state.
853 *
854 **********************************************************************/
855
856 void
em_init(void * arg)857 em_init(void *arg)
858 {
859 struct em_softc *sc = arg;
860 struct ifnet *ifp = &sc->sc_ac.ac_if;
861 uint32_t pba;
862 int s;
863
864 s = splnet();
865
866 INIT_DEBUGOUT("em_init: begin");
867
868 em_stop(sc, 0);
869
870 /*
871 * Packet Buffer Allocation (PBA)
872 * Writing PBA sets the receive portion of the buffer
873 * the remainder is used for the transmit buffer.
874 *
875 * Devices before the 82547 had a Packet Buffer of 64K.
876 * Default allocation: PBA=48K for Rx, leaving 16K for Tx.
877 * After the 82547 the buffer was reduced to 40K.
878 * Default allocation: PBA=30K for Rx, leaving 10K for Tx.
879 * Note: default does not leave enough room for Jumbo Frame >10k.
880 */
881 switch (sc->hw.mac_type) {
882 case em_82547:
883 case em_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
884 if (sc->hw.max_frame_size > EM_RXBUFFER_8192)
885 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
886 else
887 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
888 sc->tx_fifo_head = 0;
889 sc->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
890 sc->tx_fifo_size = (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
891 break;
892 case em_82571:
893 case em_82572: /* Total Packet Buffer on these is 48k */
894 case em_82575:
895 case em_82576:
896 case em_82580:
897 case em_80003es2lan:
898 case em_i350:
899 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
900 break;
901 case em_i210:
902 pba = E1000_PBA_34K;
903 break;
904 case em_82573: /* 82573: Total Packet Buffer is 32K */
905 /* Jumbo frames not supported */
906 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
907 break;
908 case em_82574: /* Total Packet Buffer is 40k */
909 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
910 break;
911 case em_ich8lan:
912 pba = E1000_PBA_8K;
913 break;
914 case em_ich9lan:
915 case em_ich10lan:
916 /* Boost Receive side for jumbo frames */
917 if (sc->hw.max_frame_size > EM_RXBUFFER_4096)
918 pba = E1000_PBA_14K;
919 else
920 pba = E1000_PBA_10K;
921 break;
922 case em_pchlan:
923 case em_pch2lan:
924 case em_pch_lpt:
925 case em_pch_spt:
926 case em_pch_cnp:
927 case em_pch_tgp:
928 case em_pch_adp:
929 pba = E1000_PBA_26K;
930 break;
931 default:
932 /* Devices before 82547 had a Packet Buffer of 64K. */
933 if (sc->hw.max_frame_size > EM_RXBUFFER_8192)
934 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
935 else
936 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
937 }
938 INIT_DEBUGOUT1("em_init: pba=%dK",pba);
939 E1000_WRITE_REG(&sc->hw, PBA, pba);
940
941 /* Get the latest mac address, User can use a LAA */
942 bcopy(sc->sc_ac.ac_enaddr, sc->hw.mac_addr, ETHER_ADDR_LEN);
943
944 /* Initialize the hardware */
945 if (em_hardware_init(sc)) {
946 printf("%s: Unable to initialize the hardware\n",
947 DEVNAME(sc));
948 splx(s);
949 return;
950 }
951 em_update_link_status(sc);
952
953 E1000_WRITE_REG(&sc->hw, VET, ETHERTYPE_VLAN);
954 if (ifp->if_capabilities & IFCAP_VLAN_HWTAGGING)
955 em_enable_hw_vlans(sc);
956
957 /* Prepare transmit descriptors and buffers */
958 if (em_setup_transmit_structures(sc)) {
959 printf("%s: Could not setup transmit structures\n",
960 DEVNAME(sc));
961 em_stop(sc, 0);
962 splx(s);
963 return;
964 }
965 em_initialize_transmit_unit(sc);
966
967 /* Prepare receive descriptors and buffers */
968 if (em_setup_receive_structures(sc)) {
969 printf("%s: Could not setup receive structures\n",
970 DEVNAME(sc));
971 em_stop(sc, 0);
972 splx(s);
973 return;
974 }
975 em_initialize_receive_unit(sc);
976
977 #ifndef SMALL_KERNEL
978 if (sc->msix) {
979 if (em_setup_queues_msix(sc)) {
980 printf("%s: Can't setup msix queues\n", DEVNAME(sc));
981 splx(s);
982 return;
983 }
984 }
985 #endif
986
987 /* Program promiscuous mode and multicast filters. */
988 em_iff(sc);
989
990 ifp->if_flags |= IFF_RUNNING;
991 ifq_clr_oactive(&ifp->if_snd);
992
993 timeout_add_sec(&sc->timer_handle, 1);
994 em_clear_hw_cntrs(&sc->hw);
995 em_enable_intr(sc);
996
997 /* Don't reset the phy next time init gets called */
998 sc->hw.phy_reset_disable = TRUE;
999
1000 splx(s);
1001 }
1002
1003 /*********************************************************************
1004 *
1005 * Interrupt Service routine
1006 *
1007 **********************************************************************/
1008 int
em_intr(void * arg)1009 em_intr(void *arg)
1010 {
1011 struct em_softc *sc = arg;
1012 struct em_queue *que = sc->queues; /* single queue */
1013 struct ifnet *ifp = &sc->sc_ac.ac_if;
1014 u_int32_t reg_icr, test_icr;
1015
1016 test_icr = reg_icr = E1000_READ_REG(&sc->hw, ICR);
1017 if (sc->hw.mac_type >= em_82571)
1018 test_icr = (reg_icr & E1000_ICR_INT_ASSERTED);
1019 if (!test_icr)
1020 return (0);
1021
1022 if (ifp->if_flags & IFF_RUNNING) {
1023 em_txeof(que);
1024 if (em_rxeof(que))
1025 em_rxrefill_locked(que);
1026 }
1027
1028 /* Link status change */
1029 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1030 KERNEL_LOCK();
1031 sc->hw.get_link_status = 1;
1032 em_check_for_link(&sc->hw);
1033 em_update_link_status(sc);
1034 KERNEL_UNLOCK();
1035 }
1036
1037 return (1);
1038 }
1039
1040 /*********************************************************************
1041 *
1042 * Media Ioctl callback
1043 *
1044 * This routine is called whenever the user queries the status of
1045 * the interface using ifconfig.
1046 *
1047 **********************************************************************/
1048 void
em_media_status(struct ifnet * ifp,struct ifmediareq * ifmr)1049 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1050 {
1051 struct em_softc *sc = ifp->if_softc;
1052 uint64_t fiber_type = IFM_1000_SX;
1053 u_int16_t gsr;
1054
1055 INIT_DEBUGOUT("em_media_status: begin");
1056
1057 em_check_for_link(&sc->hw);
1058 em_update_link_status(sc);
1059
1060 ifmr->ifm_status = IFM_AVALID;
1061 ifmr->ifm_active = IFM_ETHER;
1062
1063 if (!sc->link_active) {
1064 ifmr->ifm_active |= IFM_NONE;
1065 return;
1066 }
1067
1068 ifmr->ifm_status |= IFM_ACTIVE;
1069
1070 if (sc->hw.media_type == em_media_type_fiber ||
1071 sc->hw.media_type == em_media_type_internal_serdes) {
1072 if (sc->hw.mac_type == em_82545)
1073 fiber_type = IFM_1000_LX;
1074 ifmr->ifm_active |= fiber_type | IFM_FDX;
1075 } else {
1076 switch (sc->link_speed) {
1077 case 10:
1078 ifmr->ifm_active |= IFM_10_T;
1079 break;
1080 case 100:
1081 ifmr->ifm_active |= IFM_100_TX;
1082 break;
1083 case 1000:
1084 ifmr->ifm_active |= IFM_1000_T;
1085 break;
1086 }
1087
1088 if (sc->link_duplex == FULL_DUPLEX)
1089 ifmr->ifm_active |= em_flowstatus(sc) | IFM_FDX;
1090 else
1091 ifmr->ifm_active |= IFM_HDX;
1092
1093 if (IFM_SUBTYPE(ifmr->ifm_active) == IFM_1000_T) {
1094 em_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &gsr);
1095 if (gsr & SR_1000T_MS_CONFIG_RES)
1096 ifmr->ifm_active |= IFM_ETH_MASTER;
1097 }
1098 }
1099 }
1100
1101 /*********************************************************************
1102 *
1103 * Media Ioctl callback
1104 *
1105 * This routine is called when the user changes speed/duplex using
1106 * media/mediopt option with ifconfig.
1107 *
1108 **********************************************************************/
1109 int
em_media_change(struct ifnet * ifp)1110 em_media_change(struct ifnet *ifp)
1111 {
1112 struct em_softc *sc = ifp->if_softc;
1113 struct ifmedia *ifm = &sc->media;
1114
1115 INIT_DEBUGOUT("em_media_change: begin");
1116
1117 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1118 return (EINVAL);
1119
1120 switch (IFM_SUBTYPE(ifm->ifm_media)) {
1121 case IFM_AUTO:
1122 sc->hw.autoneg = DO_AUTO_NEG;
1123 sc->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1124 break;
1125 case IFM_1000_LX:
1126 case IFM_1000_SX:
1127 case IFM_1000_T:
1128 sc->hw.autoneg = DO_AUTO_NEG;
1129 sc->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1130 break;
1131 case IFM_100_TX:
1132 sc->hw.autoneg = FALSE;
1133 sc->hw.autoneg_advertised = 0;
1134 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1135 sc->hw.forced_speed_duplex = em_100_full;
1136 else
1137 sc->hw.forced_speed_duplex = em_100_half;
1138 break;
1139 case IFM_10_T:
1140 sc->hw.autoneg = FALSE;
1141 sc->hw.autoneg_advertised = 0;
1142 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1143 sc->hw.forced_speed_duplex = em_10_full;
1144 else
1145 sc->hw.forced_speed_duplex = em_10_half;
1146 break;
1147 default:
1148 printf("%s: Unsupported media type\n", DEVNAME(sc));
1149 }
1150
1151 /*
1152 * As the speed/duplex settings may have changed we need to
1153 * reset the PHY.
1154 */
1155 sc->hw.phy_reset_disable = FALSE;
1156
1157 em_init(sc);
1158
1159 return (0);
1160 }
1161
1162 uint64_t
em_flowstatus(struct em_softc * sc)1163 em_flowstatus(struct em_softc *sc)
1164 {
1165 u_int16_t ar, lpar;
1166
1167 if (sc->hw.media_type == em_media_type_fiber ||
1168 sc->hw.media_type == em_media_type_internal_serdes)
1169 return (0);
1170
1171 em_read_phy_reg(&sc->hw, PHY_AUTONEG_ADV, &ar);
1172 em_read_phy_reg(&sc->hw, PHY_LP_ABILITY, &lpar);
1173
1174 if ((ar & NWAY_AR_PAUSE) && (lpar & NWAY_LPAR_PAUSE))
1175 return (IFM_FLOW|IFM_ETH_TXPAUSE|IFM_ETH_RXPAUSE);
1176 else if (!(ar & NWAY_AR_PAUSE) && (ar & NWAY_AR_ASM_DIR) &&
1177 (lpar & NWAY_LPAR_PAUSE) && (lpar & NWAY_LPAR_ASM_DIR))
1178 return (IFM_FLOW|IFM_ETH_TXPAUSE);
1179 else if ((ar & NWAY_AR_PAUSE) && (ar & NWAY_AR_ASM_DIR) &&
1180 !(lpar & NWAY_LPAR_PAUSE) && (lpar & NWAY_LPAR_ASM_DIR))
1181 return (IFM_FLOW|IFM_ETH_RXPAUSE);
1182
1183 return (0);
1184 }
1185
1186 /*********************************************************************
1187 *
1188 * This routine maps the mbufs to tx descriptors.
1189 *
1190 * return 0 on failure, positive on success
1191 **********************************************************************/
1192 u_int
em_encap(struct em_queue * que,struct mbuf * m)1193 em_encap(struct em_queue *que, struct mbuf *m)
1194 {
1195 struct em_softc *sc = que->sc;
1196 struct em_packet *pkt;
1197 struct em_tx_desc *desc;
1198 bus_dmamap_t map;
1199 u_int32_t txd_upper, txd_lower;
1200 u_int head, last, used = 0;
1201 int i, j;
1202
1203 /* For 82544 Workaround */
1204 DESC_ARRAY desc_array;
1205 u_int32_t array_elements;
1206
1207 /* get a dmamap for this packet from the next free slot */
1208 head = que->tx.sc_tx_desc_head;
1209 pkt = &que->tx.sc_tx_pkts_ring[head];
1210 map = pkt->pkt_map;
1211
1212 switch (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, BUS_DMA_NOWAIT)) {
1213 case 0:
1214 break;
1215 case EFBIG:
1216 if (m_defrag(m, M_DONTWAIT) == 0 &&
1217 bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
1218 BUS_DMA_NOWAIT) == 0)
1219 break;
1220
1221 /* FALLTHROUGH */
1222 default:
1223 sc->no_tx_dma_setup++;
1224 return (0);
1225 }
1226
1227 bus_dmamap_sync(sc->sc_dmat, map,
1228 0, map->dm_mapsize,
1229 BUS_DMASYNC_PREWRITE);
1230
1231 if (sc->hw.mac_type == em_82547) {
1232 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
1233 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
1234 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1235 }
1236
1237 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210) {
1238 if (ISSET(m->m_pkthdr.csum_flags, M_TCP_TSO)) {
1239 used += em_tso_setup(que, m, head, &txd_upper,
1240 &txd_lower);
1241 if (!used)
1242 return (used);
1243 } else {
1244 used += em_tx_ctx_setup(que, m, head, &txd_upper,
1245 &txd_lower);
1246 }
1247 } else if (sc->hw.mac_type >= em_82543) {
1248 used += em_transmit_checksum_setup(que, m, head,
1249 &txd_upper, &txd_lower);
1250 } else {
1251 txd_upper = txd_lower = 0;
1252 }
1253
1254 head += used;
1255 if (head >= sc->sc_tx_slots)
1256 head -= sc->sc_tx_slots;
1257
1258 for (i = 0; i < map->dm_nsegs; i++) {
1259 /* If sc is 82544 and on PCI-X bus */
1260 if (sc->pcix_82544) {
1261 /*
1262 * Check the Address and Length combination and
1263 * split the data accordingly
1264 */
1265 array_elements = em_fill_descriptors(
1266 map->dm_segs[i].ds_addr, map->dm_segs[i].ds_len,
1267 &desc_array);
1268 for (j = 0; j < array_elements; j++) {
1269 desc = &que->tx.sc_tx_desc_ring[head];
1270
1271 desc->buffer_addr = htole64(
1272 desc_array.descriptor[j].address);
1273 desc->lower.data = htole32(
1274 (que->tx.sc_txd_cmd | txd_lower |
1275 (u_int16_t)desc_array.descriptor[j].length));
1276 desc->upper.data = htole32(txd_upper);
1277
1278 last = head;
1279 if (++head == sc->sc_tx_slots)
1280 head = 0;
1281
1282 used++;
1283 }
1284 } else {
1285 desc = &que->tx.sc_tx_desc_ring[head];
1286
1287 desc->buffer_addr = htole64(map->dm_segs[i].ds_addr);
1288 desc->lower.data = htole32(que->tx.sc_txd_cmd |
1289 txd_lower | map->dm_segs[i].ds_len);
1290 desc->upper.data = htole32(txd_upper);
1291
1292 last = head;
1293 if (++head == sc->sc_tx_slots)
1294 head = 0;
1295
1296 used++;
1297 }
1298 }
1299
1300 #if NVLAN > 0
1301 /* Find out if we are in VLAN mode */
1302 if (m->m_flags & M_VLANTAG && (sc->hw.mac_type < em_82575 ||
1303 sc->hw.mac_type > em_i210)) {
1304 /* Set the VLAN id */
1305 desc->upper.fields.special = htole16(m->m_pkthdr.ether_vtag);
1306
1307 /* Tell hardware to add tag */
1308 desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1309 }
1310 #endif
1311
1312 /* mark the packet with the mbuf and last desc slot */
1313 pkt->pkt_m = m;
1314 pkt->pkt_eop = last;
1315
1316 que->tx.sc_tx_desc_head = head;
1317
1318 /*
1319 * Last Descriptor of Packet
1320 * needs End Of Packet (EOP)
1321 * and Report Status (RS)
1322 */
1323 desc->lower.data |= htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1324
1325 if (sc->hw.mac_type == em_82547) {
1326 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
1327 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
1328 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1329 }
1330
1331 return (used);
1332 }
1333
1334 /*********************************************************************
1335 *
1336 * 82547 workaround to avoid controller hang in half-duplex environment.
1337 * The workaround is to avoid queuing a large packet that would span
1338 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1339 * in this case. We do that only when FIFO is quiescent.
1340 *
1341 **********************************************************************/
1342 void
em_82547_move_tail_locked(struct em_softc * sc)1343 em_82547_move_tail_locked(struct em_softc *sc)
1344 {
1345 uint16_t hw_tdt;
1346 uint16_t sw_tdt;
1347 struct em_tx_desc *tx_desc;
1348 uint16_t length = 0;
1349 boolean_t eop = 0;
1350 struct em_queue *que = sc->queues; /* single queue chip */
1351
1352 hw_tdt = E1000_READ_REG(&sc->hw, TDT(que->me));
1353 sw_tdt = que->tx.sc_tx_desc_head;
1354
1355 while (hw_tdt != sw_tdt) {
1356 tx_desc = &que->tx.sc_tx_desc_ring[hw_tdt];
1357 length += tx_desc->lower.flags.length;
1358 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1359 if (++hw_tdt == sc->sc_tx_slots)
1360 hw_tdt = 0;
1361
1362 if (eop) {
1363 if (em_82547_fifo_workaround(sc, length)) {
1364 sc->tx_fifo_wrk_cnt++;
1365 timeout_add(&sc->tx_fifo_timer_handle, 1);
1366 break;
1367 }
1368 E1000_WRITE_REG(&sc->hw, TDT(que->me), hw_tdt);
1369 em_82547_update_fifo_head(sc, length);
1370 length = 0;
1371 }
1372 }
1373 }
1374
1375 void
em_82547_move_tail(void * arg)1376 em_82547_move_tail(void *arg)
1377 {
1378 struct em_softc *sc = arg;
1379 int s;
1380
1381 s = splnet();
1382 em_82547_move_tail_locked(sc);
1383 splx(s);
1384 }
1385
1386 int
em_82547_fifo_workaround(struct em_softc * sc,int len)1387 em_82547_fifo_workaround(struct em_softc *sc, int len)
1388 {
1389 int fifo_space, fifo_pkt_len;
1390
1391 fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1392
1393 if (sc->link_duplex == HALF_DUPLEX) {
1394 fifo_space = sc->tx_fifo_size - sc->tx_fifo_head;
1395
1396 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1397 if (em_82547_tx_fifo_reset(sc))
1398 return (0);
1399 else
1400 return (1);
1401 }
1402 }
1403
1404 return (0);
1405 }
1406
1407 void
em_82547_update_fifo_head(struct em_softc * sc,int len)1408 em_82547_update_fifo_head(struct em_softc *sc, int len)
1409 {
1410 int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1411
1412 /* tx_fifo_head is always 16 byte aligned */
1413 sc->tx_fifo_head += fifo_pkt_len;
1414 if (sc->tx_fifo_head >= sc->tx_fifo_size)
1415 sc->tx_fifo_head -= sc->tx_fifo_size;
1416 }
1417
1418 int
em_82547_tx_fifo_reset(struct em_softc * sc)1419 em_82547_tx_fifo_reset(struct em_softc *sc)
1420 {
1421 uint32_t tctl;
1422 struct em_queue *que = sc->queues; /* single queue chip */
1423
1424 if ((E1000_READ_REG(&sc->hw, TDT(que->me)) ==
1425 E1000_READ_REG(&sc->hw, TDH(que->me))) &&
1426 (E1000_READ_REG(&sc->hw, TDFT) ==
1427 E1000_READ_REG(&sc->hw, TDFH)) &&
1428 (E1000_READ_REG(&sc->hw, TDFTS) ==
1429 E1000_READ_REG(&sc->hw, TDFHS)) &&
1430 (E1000_READ_REG(&sc->hw, TDFPC) == 0)) {
1431
1432 /* Disable TX unit */
1433 tctl = E1000_READ_REG(&sc->hw, TCTL);
1434 E1000_WRITE_REG(&sc->hw, TCTL, tctl & ~E1000_TCTL_EN);
1435
1436 /* Reset FIFO pointers */
1437 E1000_WRITE_REG(&sc->hw, TDFT, sc->tx_head_addr);
1438 E1000_WRITE_REG(&sc->hw, TDFH, sc->tx_head_addr);
1439 E1000_WRITE_REG(&sc->hw, TDFTS, sc->tx_head_addr);
1440 E1000_WRITE_REG(&sc->hw, TDFHS, sc->tx_head_addr);
1441
1442 /* Re-enable TX unit */
1443 E1000_WRITE_REG(&sc->hw, TCTL, tctl);
1444 E1000_WRITE_FLUSH(&sc->hw);
1445
1446 sc->tx_fifo_head = 0;
1447 sc->tx_fifo_reset_cnt++;
1448
1449 return (TRUE);
1450 } else
1451 return (FALSE);
1452 }
1453
1454 void
em_iff(struct em_softc * sc)1455 em_iff(struct em_softc *sc)
1456 {
1457 struct ifnet *ifp = &sc->sc_ac.ac_if;
1458 struct arpcom *ac = &sc->sc_ac;
1459 u_int32_t reg_rctl = 0;
1460 u_int8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1461 struct ether_multi *enm;
1462 struct ether_multistep step;
1463 int i = 0;
1464
1465 IOCTL_DEBUGOUT("em_iff: begin");
1466
1467 if (sc->hw.mac_type == em_82542_rev2_0) {
1468 reg_rctl = E1000_READ_REG(&sc->hw, RCTL);
1469 if (sc->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1470 em_pci_clear_mwi(&sc->hw);
1471 reg_rctl |= E1000_RCTL_RST;
1472 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
1473 msec_delay(5);
1474 }
1475
1476 reg_rctl = E1000_READ_REG(&sc->hw, RCTL);
1477 reg_rctl &= ~(E1000_RCTL_MPE | E1000_RCTL_UPE);
1478 ifp->if_flags &= ~IFF_ALLMULTI;
1479
1480 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 ||
1481 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) {
1482 ifp->if_flags |= IFF_ALLMULTI;
1483 reg_rctl |= E1000_RCTL_MPE;
1484 if (ifp->if_flags & IFF_PROMISC)
1485 reg_rctl |= E1000_RCTL_UPE;
1486 } else {
1487 ETHER_FIRST_MULTI(step, ac, enm);
1488 while (enm != NULL) {
1489 bcopy(enm->enm_addrlo, mta + i, ETH_LENGTH_OF_ADDRESS);
1490 i += ETH_LENGTH_OF_ADDRESS;
1491
1492 ETHER_NEXT_MULTI(step, enm);
1493 }
1494
1495 em_mc_addr_list_update(&sc->hw, mta, ac->ac_multicnt, 0);
1496 }
1497
1498 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
1499
1500 if (sc->hw.mac_type == em_82542_rev2_0) {
1501 reg_rctl = E1000_READ_REG(&sc->hw, RCTL);
1502 reg_rctl &= ~E1000_RCTL_RST;
1503 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
1504 msec_delay(5);
1505 if (sc->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1506 em_pci_set_mwi(&sc->hw);
1507 }
1508 }
1509
1510 /*********************************************************************
1511 * Timer routine
1512 *
1513 * This routine checks for link status and updates statistics.
1514 *
1515 **********************************************************************/
1516
1517 void
em_local_timer(void * arg)1518 em_local_timer(void *arg)
1519 {
1520 struct em_softc *sc = arg;
1521 int s;
1522
1523 timeout_add_sec(&sc->timer_handle, 1);
1524
1525 s = splnet();
1526 em_smartspeed(sc);
1527 splx(s);
1528
1529 #if NKSTAT > 0
1530 if (sc->kstat != NULL && mtx_enter_try(&sc->kstat_mtx)) {
1531 em_kstat_read(sc->kstat);
1532 mtx_leave(&sc->kstat_mtx);
1533 }
1534 #endif
1535 }
1536
1537 void
em_update_link_status(struct em_softc * sc)1538 em_update_link_status(struct em_softc *sc)
1539 {
1540 struct ifnet *ifp = &sc->sc_ac.ac_if;
1541 u_char link_state;
1542
1543 if (E1000_READ_REG(&sc->hw, STATUS) & E1000_STATUS_LU) {
1544 if (sc->link_active == 0) {
1545 em_get_speed_and_duplex(&sc->hw,
1546 &sc->link_speed,
1547 &sc->link_duplex);
1548 /* Check if we may set SPEED_MODE bit on PCI-E */
1549 if ((sc->link_speed == SPEED_1000) &&
1550 ((sc->hw.mac_type == em_82571) ||
1551 (sc->hw.mac_type == em_82572) ||
1552 (sc->hw.mac_type == em_82575) ||
1553 (sc->hw.mac_type == em_82576) ||
1554 (sc->hw.mac_type == em_82580))) {
1555 int tarc0;
1556
1557 tarc0 = E1000_READ_REG(&sc->hw, TARC0);
1558 tarc0 |= SPEED_MODE_BIT;
1559 E1000_WRITE_REG(&sc->hw, TARC0, tarc0);
1560 }
1561 sc->link_active = 1;
1562 sc->smartspeed = 0;
1563 ifp->if_baudrate = IF_Mbps(sc->link_speed);
1564 }
1565 link_state = (sc->link_duplex == FULL_DUPLEX) ?
1566 LINK_STATE_FULL_DUPLEX : LINK_STATE_HALF_DUPLEX;
1567 } else {
1568 if (sc->link_active == 1) {
1569 ifp->if_baudrate = sc->link_speed = 0;
1570 sc->link_duplex = 0;
1571 sc->link_active = 0;
1572 }
1573 link_state = LINK_STATE_DOWN;
1574 }
1575 if (ifp->if_link_state != link_state) {
1576 ifp->if_link_state = link_state;
1577 if_link_state_change(ifp);
1578 }
1579
1580 /* Disable TSO for 10/100 speeds to avoid some hardware issues */
1581 switch (sc->link_speed) {
1582 case SPEED_10:
1583 case SPEED_100:
1584 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210) {
1585 ifp->if_capabilities &= ~IFCAP_TSOv4;
1586 ifp->if_capabilities &= ~IFCAP_TSOv6;
1587 }
1588 break;
1589 case SPEED_1000:
1590 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210)
1591 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
1592 break;
1593 }
1594 }
1595
1596 /*********************************************************************
1597 *
1598 * This routine disables all traffic on the adapter by issuing a
1599 * global reset on the MAC and deallocates TX/RX buffers.
1600 *
1601 **********************************************************************/
1602
1603 void
em_stop(void * arg,int softonly)1604 em_stop(void *arg, int softonly)
1605 {
1606 struct em_softc *sc = arg;
1607 struct em_queue *que = sc->queues; /* Use only first queue. */
1608 struct ifnet *ifp = &sc->sc_ac.ac_if;
1609
1610 /* Tell the stack that the interface is no longer active */
1611 ifp->if_flags &= ~IFF_RUNNING;
1612
1613 INIT_DEBUGOUT("em_stop: begin");
1614
1615 timeout_del(&que->rx_refill);
1616 timeout_del(&sc->timer_handle);
1617 timeout_del(&sc->tx_fifo_timer_handle);
1618
1619 if (!softonly)
1620 em_disable_intr(sc);
1621 if (sc->hw.mac_type >= em_pch_spt)
1622 em_flush_desc_rings(sc);
1623 if (!softonly)
1624 em_reset_hw(&sc->hw);
1625
1626 intr_barrier(sc->sc_intrhand);
1627 ifq_barrier(&ifp->if_snd);
1628
1629 KASSERT((ifp->if_flags & IFF_RUNNING) == 0);
1630
1631 ifq_clr_oactive(&ifp->if_snd);
1632 ifp->if_timer = 0;
1633
1634 em_free_transmit_structures(sc);
1635 em_free_receive_structures(sc);
1636 }
1637
1638 /*********************************************************************
1639 *
1640 * Determine hardware revision.
1641 *
1642 **********************************************************************/
1643 void
em_identify_hardware(struct em_softc * sc)1644 em_identify_hardware(struct em_softc *sc)
1645 {
1646 u_int32_t reg;
1647 struct pci_attach_args *pa = &sc->osdep.em_pa;
1648
1649 /* Make sure our PCI config space has the necessary stuff set */
1650 sc->hw.pci_cmd_word = pci_conf_read(pa->pa_pc, pa->pa_tag,
1651 PCI_COMMAND_STATUS_REG);
1652
1653 /* Save off the information about this board */
1654 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id);
1655 sc->hw.device_id = PCI_PRODUCT(pa->pa_id);
1656
1657 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG);
1658 sc->hw.revision_id = PCI_REVISION(reg);
1659
1660 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
1661 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg);
1662 sc->hw.subsystem_id = PCI_PRODUCT(reg);
1663
1664 /* Identify the MAC */
1665 if (em_set_mac_type(&sc->hw))
1666 printf("%s: Unknown MAC Type\n", DEVNAME(sc));
1667
1668 if (sc->hw.mac_type == em_pchlan)
1669 sc->hw.revision_id = PCI_PRODUCT(pa->pa_id) & 0x0f;
1670
1671 if (sc->hw.mac_type == em_82541 ||
1672 sc->hw.mac_type == em_82541_rev_2 ||
1673 sc->hw.mac_type == em_82547 ||
1674 sc->hw.mac_type == em_82547_rev_2)
1675 sc->hw.phy_init_script = TRUE;
1676 }
1677
1678 void
em_legacy_irq_quirk_spt(struct em_softc * sc)1679 em_legacy_irq_quirk_spt(struct em_softc *sc)
1680 {
1681 uint32_t reg;
1682
1683 /* Legacy interrupt: SPT needs a quirk. */
1684 if (sc->hw.mac_type != em_pch_spt && sc->hw.mac_type != em_pch_cnp &&
1685 sc->hw.mac_type != em_pch_tgp && sc->hw.mac_type != em_pch_adp)
1686 return;
1687 if (sc->legacy_irq == 0)
1688 return;
1689
1690 reg = EM_READ_REG(&sc->hw, E1000_FEXTNVM7);
1691 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE;
1692 EM_WRITE_REG(&sc->hw, E1000_FEXTNVM7, reg);
1693
1694 reg = EM_READ_REG(&sc->hw, E1000_FEXTNVM9);
1695 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS |
1696 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS;
1697 EM_WRITE_REG(&sc->hw, E1000_FEXTNVM9, reg);
1698 }
1699
1700 int
em_allocate_pci_resources(struct em_softc * sc)1701 em_allocate_pci_resources(struct em_softc *sc)
1702 {
1703 int val, rid;
1704 struct pci_attach_args *pa = &sc->osdep.em_pa;
1705 struct em_queue *que = NULL;
1706
1707 val = pci_conf_read(pa->pa_pc, pa->pa_tag, EM_MMBA);
1708 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) {
1709 printf(": mmba is not mem space\n");
1710 return (ENXIO);
1711 }
1712 if (pci_mapreg_map(pa, EM_MMBA, PCI_MAPREG_MEM_TYPE(val), 0,
1713 &sc->osdep.mem_bus_space_tag, &sc->osdep.mem_bus_space_handle,
1714 &sc->osdep.em_membase, &sc->osdep.em_memsize, 0)) {
1715 printf(": cannot find mem space\n");
1716 return (ENXIO);
1717 }
1718
1719 switch (sc->hw.mac_type) {
1720 case em_82544:
1721 case em_82540:
1722 case em_82545:
1723 case em_82546:
1724 case em_82541:
1725 case em_82541_rev_2:
1726 /* Figure out where our I/O BAR is ? */
1727 for (rid = PCI_MAPREG_START; rid < PCI_MAPREG_END;) {
1728 val = pci_conf_read(pa->pa_pc, pa->pa_tag, rid);
1729 if (PCI_MAPREG_TYPE(val) == PCI_MAPREG_TYPE_IO) {
1730 sc->io_rid = rid;
1731 break;
1732 }
1733 rid += 4;
1734 if (PCI_MAPREG_MEM_TYPE(val) ==
1735 PCI_MAPREG_MEM_TYPE_64BIT)
1736 rid += 4; /* skip high bits, too */
1737 }
1738
1739 if (pci_mapreg_map(pa, rid, PCI_MAPREG_TYPE_IO, 0,
1740 &sc->osdep.io_bus_space_tag, &sc->osdep.io_bus_space_handle,
1741 &sc->osdep.em_iobase, &sc->osdep.em_iosize, 0)) {
1742 printf(": cannot find i/o space\n");
1743 return (ENXIO);
1744 }
1745
1746 sc->hw.io_base = 0;
1747 break;
1748 default:
1749 break;
1750 }
1751
1752 sc->osdep.em_flashoffset = 0;
1753 /* for ICH8 and family we need to find the flash memory */
1754 if (sc->hw.mac_type >= em_pch_spt) {
1755 sc->osdep.flash_bus_space_tag = sc->osdep.mem_bus_space_tag;
1756 sc->osdep.flash_bus_space_handle = sc->osdep.mem_bus_space_handle;
1757 sc->osdep.em_flashbase = 0;
1758 sc->osdep.em_flashsize = 0;
1759 sc->osdep.em_flashoffset = 0xe000;
1760 } else if (IS_ICH8(sc->hw.mac_type)) {
1761 val = pci_conf_read(pa->pa_pc, pa->pa_tag, EM_FLASH);
1762 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) {
1763 printf(": flash is not mem space\n");
1764 return (ENXIO);
1765 }
1766
1767 if (pci_mapreg_map(pa, EM_FLASH, PCI_MAPREG_MEM_TYPE(val), 0,
1768 &sc->osdep.flash_bus_space_tag, &sc->osdep.flash_bus_space_handle,
1769 &sc->osdep.em_flashbase, &sc->osdep.em_flashsize, 0)) {
1770 printf(": cannot find mem space\n");
1771 return (ENXIO);
1772 }
1773 }
1774
1775 sc->osdep.dev = (struct device *)sc;
1776 sc->hw.back = &sc->osdep;
1777
1778 /* Only one queue for the moment. */
1779 que = malloc(sizeof(struct em_queue), M_DEVBUF, M_NOWAIT | M_ZERO);
1780 if (que == NULL) {
1781 printf(": unable to allocate queue memory\n");
1782 return (ENOMEM);
1783 }
1784 que->me = 0;
1785 que->sc = sc;
1786 timeout_set(&que->rx_refill, em_rxrefill, que);
1787
1788 sc->queues = que;
1789 sc->num_queues = 1;
1790 sc->msix = 0;
1791 sc->legacy_irq = 0;
1792 if (em_allocate_msix(sc) && em_allocate_legacy(sc))
1793 return (ENXIO);
1794
1795 /*
1796 * the ICP_xxxx device has multiple, duplicate register sets for
1797 * use when it is being used as a network processor. Disable those
1798 * registers here, as they are not necessary in this context and
1799 * can confuse the system
1800 */
1801 if(sc->hw.mac_type == em_icp_xxxx) {
1802 int offset;
1803 pcireg_t val;
1804
1805 if (!pci_get_capability(sc->osdep.em_pa.pa_pc,
1806 sc->osdep.em_pa.pa_tag, PCI_CAP_ID_ST, &offset, &val)) {
1807 return (0);
1808 }
1809 offset += PCI_ST_SMIA_OFFSET;
1810 pci_conf_write(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
1811 offset, 0x06);
1812 E1000_WRITE_REG(&sc->hw, IMC1, ~0x0);
1813 E1000_WRITE_REG(&sc->hw, IMC2, ~0x0);
1814 }
1815 return (0);
1816 }
1817
1818 void
em_free_pci_resources(struct em_softc * sc)1819 em_free_pci_resources(struct em_softc *sc)
1820 {
1821 struct pci_attach_args *pa = &sc->osdep.em_pa;
1822 pci_chipset_tag_t pc = pa->pa_pc;
1823 struct em_queue *que = NULL;
1824 if (sc->sc_intrhand)
1825 pci_intr_disestablish(pc, sc->sc_intrhand);
1826 sc->sc_intrhand = 0;
1827
1828 if (sc->osdep.em_flashbase)
1829 bus_space_unmap(sc->osdep.flash_bus_space_tag, sc->osdep.flash_bus_space_handle,
1830 sc->osdep.em_flashsize);
1831 sc->osdep.em_flashbase = 0;
1832
1833 if (sc->osdep.em_iobase)
1834 bus_space_unmap(sc->osdep.io_bus_space_tag, sc->osdep.io_bus_space_handle,
1835 sc->osdep.em_iosize);
1836 sc->osdep.em_iobase = 0;
1837
1838 if (sc->osdep.em_membase)
1839 bus_space_unmap(sc->osdep.mem_bus_space_tag, sc->osdep.mem_bus_space_handle,
1840 sc->osdep.em_memsize);
1841 sc->osdep.em_membase = 0;
1842
1843 FOREACH_QUEUE(sc, que) {
1844 if (que->rx.sc_rx_desc_ring != NULL) {
1845 que->rx.sc_rx_desc_ring = NULL;
1846 em_dma_free(sc, &que->rx.sc_rx_dma);
1847 }
1848 if (que->tx.sc_tx_desc_ring != NULL) {
1849 que->tx.sc_tx_desc_ring = NULL;
1850 em_dma_free(sc, &que->tx.sc_tx_dma);
1851 }
1852 if (que->tag)
1853 pci_intr_disestablish(pc, que->tag);
1854 que->tag = NULL;
1855 que->eims = 0;
1856 que->me = 0;
1857 que->sc = NULL;
1858 }
1859 sc->legacy_irq = 0;
1860 sc->msix_linkvec = 0;
1861 sc->msix_queuesmask = 0;
1862 if (sc->queues)
1863 free(sc->queues, M_DEVBUF,
1864 sc->num_queues * sizeof(struct em_queue));
1865 sc->num_queues = 0;
1866 sc->queues = NULL;
1867 }
1868
1869 /*********************************************************************
1870 *
1871 * Initialize the hardware to a configuration as specified by the
1872 * em_softc structure. The controller is reset, the EEPROM is
1873 * verified, the MAC address is set, then the shared initialization
1874 * routines are called.
1875 *
1876 **********************************************************************/
1877 int
em_hardware_init(struct em_softc * sc)1878 em_hardware_init(struct em_softc *sc)
1879 {
1880 uint32_t ret_val;
1881 u_int16_t rx_buffer_size;
1882
1883 INIT_DEBUGOUT("em_hardware_init: begin");
1884 if (sc->hw.mac_type >= em_pch_spt)
1885 em_flush_desc_rings(sc);
1886 /* Issue a global reset */
1887 em_reset_hw(&sc->hw);
1888
1889 /* When hardware is reset, fifo_head is also reset */
1890 sc->tx_fifo_head = 0;
1891
1892 /* Make sure we have a good EEPROM before we read from it */
1893 if (em_get_flash_presence_i210(&sc->hw) &&
1894 em_validate_eeprom_checksum(&sc->hw) < 0) {
1895 /*
1896 * Some PCIe parts fail the first check due to
1897 * the link being in sleep state, call it again,
1898 * if it fails a second time its a real issue.
1899 */
1900 if (em_validate_eeprom_checksum(&sc->hw) < 0) {
1901 printf("%s: The EEPROM Checksum Is Not Valid\n",
1902 DEVNAME(sc));
1903 return (EIO);
1904 }
1905 }
1906
1907 if (em_get_flash_presence_i210(&sc->hw) &&
1908 em_read_part_num(&sc->hw, &(sc->part_num)) < 0) {
1909 printf("%s: EEPROM read error while reading part number\n",
1910 DEVNAME(sc));
1911 return (EIO);
1912 }
1913
1914 /* Set up smart power down as default off on newer adapters */
1915 if (!em_smart_pwr_down &&
1916 (sc->hw.mac_type == em_82571 ||
1917 sc->hw.mac_type == em_82572 ||
1918 sc->hw.mac_type == em_82575 ||
1919 sc->hw.mac_type == em_82576 ||
1920 sc->hw.mac_type == em_82580 ||
1921 sc->hw.mac_type == em_i210 ||
1922 sc->hw.mac_type == em_i350 )) {
1923 uint16_t phy_tmp = 0;
1924
1925 /* Speed up time to link by disabling smart power down */
1926 em_read_phy_reg(&sc->hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
1927 phy_tmp &= ~IGP02E1000_PM_SPD;
1928 em_write_phy_reg(&sc->hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
1929 }
1930
1931 em_legacy_irq_quirk_spt(sc);
1932
1933 /*
1934 * These parameters control the automatic generation (Tx) and
1935 * response (Rx) to Ethernet PAUSE frames.
1936 * - High water mark should allow for at least two frames to be
1937 * received after sending an XOFF.
1938 * - Low water mark works best when it is very near the high water mark.
1939 * This allows the receiver to restart by sending XON when it has
1940 * drained a bit. Here we use an arbitrary value of 1500 which will
1941 * restart after one full frame is pulled from the buffer. There
1942 * could be several smaller frames in the buffer and if so they will
1943 * not trigger the XON until their total number reduces the buffer
1944 * by 1500.
1945 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1946 */
1947 rx_buffer_size = ((E1000_READ_REG(&sc->hw, PBA) & 0xffff) << 10 );
1948
1949 sc->hw.fc_high_water = rx_buffer_size -
1950 EM_ROUNDUP(sc->hw.max_frame_size, 1024);
1951 sc->hw.fc_low_water = sc->hw.fc_high_water - 1500;
1952 if (sc->hw.mac_type == em_80003es2lan)
1953 sc->hw.fc_pause_time = 0xFFFF;
1954 else
1955 sc->hw.fc_pause_time = 1000;
1956 sc->hw.fc_send_xon = TRUE;
1957 sc->hw.fc = E1000_FC_FULL;
1958
1959 em_disable_aspm(sc);
1960
1961 if ((ret_val = em_init_hw(sc)) != 0) {
1962 if (ret_val == E1000_DEFER_INIT) {
1963 INIT_DEBUGOUT("\nHardware Initialization Deferred ");
1964 return (EAGAIN);
1965 }
1966 printf("\n%s: Hardware Initialization Failed: %d\n",
1967 DEVNAME(sc), ret_val);
1968 return (EIO);
1969 }
1970
1971 em_check_for_link(&sc->hw);
1972
1973 return (0);
1974 }
1975
1976 /*********************************************************************
1977 *
1978 * Setup networking device structure and register an interface.
1979 *
1980 **********************************************************************/
1981 void
em_setup_interface(struct em_softc * sc)1982 em_setup_interface(struct em_softc *sc)
1983 {
1984 struct ifnet *ifp;
1985 uint64_t fiber_type = IFM_1000_SX;
1986
1987 INIT_DEBUGOUT("em_setup_interface: begin");
1988
1989 ifp = &sc->sc_ac.ac_if;
1990 strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
1991 ifp->if_softc = sc;
1992 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1993 ifp->if_xflags = IFXF_MPSAFE;
1994 ifp->if_ioctl = em_ioctl;
1995 ifp->if_qstart = em_start;
1996 ifp->if_watchdog = em_watchdog;
1997 ifp->if_hardmtu =
1998 sc->hw.max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN;
1999 ifq_init_maxlen(&ifp->if_snd, sc->sc_tx_slots - 1);
2000
2001 ifp->if_capabilities = IFCAP_VLAN_MTU;
2002
2003 #if NVLAN > 0
2004 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
2005 #endif
2006
2007 if (sc->hw.mac_type >= em_82543) {
2008 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4;
2009 }
2010 if (sc->hw.mac_type >= em_82575 && sc->hw.mac_type <= em_i210) {
2011 ifp->if_capabilities |= IFCAP_CSUM_IPv4;
2012 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6;
2013 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
2014 }
2015
2016 /*
2017 * Specify the media types supported by this adapter and register
2018 * callbacks to update media and link information
2019 */
2020 ifmedia_init(&sc->media, IFM_IMASK, em_media_change,
2021 em_media_status);
2022 if (sc->hw.media_type == em_media_type_fiber ||
2023 sc->hw.media_type == em_media_type_internal_serdes) {
2024 if (sc->hw.mac_type == em_82545)
2025 fiber_type = IFM_1000_LX;
2026 ifmedia_add(&sc->media, IFM_ETHER | fiber_type | IFM_FDX,
2027 0, NULL);
2028 ifmedia_add(&sc->media, IFM_ETHER | fiber_type,
2029 0, NULL);
2030 } else {
2031 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL);
2032 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2033 0, NULL);
2034 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX,
2035 0, NULL);
2036 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2037 0, NULL);
2038 if (sc->hw.phy_type != em_phy_ife) {
2039 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
2040 0, NULL);
2041 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2042 }
2043 }
2044 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2045 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2046
2047 if_attach(ifp);
2048 ether_ifattach(ifp);
2049 em_enable_intr(sc);
2050 }
2051
2052 int
em_detach(struct device * self,int flags)2053 em_detach(struct device *self, int flags)
2054 {
2055 struct em_softc *sc = (struct em_softc *)self;
2056 struct ifnet *ifp = &sc->sc_ac.ac_if;
2057 struct pci_attach_args *pa = &sc->osdep.em_pa;
2058 pci_chipset_tag_t pc = pa->pa_pc;
2059
2060 if (sc->sc_intrhand)
2061 pci_intr_disestablish(pc, sc->sc_intrhand);
2062 sc->sc_intrhand = 0;
2063
2064 em_stop(sc, 1);
2065
2066 em_free_pci_resources(sc);
2067
2068 ether_ifdetach(ifp);
2069 if_detach(ifp);
2070
2071 return (0);
2072 }
2073
2074 int
em_activate(struct device * self,int act)2075 em_activate(struct device *self, int act)
2076 {
2077 struct em_softc *sc = (struct em_softc *)self;
2078 struct ifnet *ifp = &sc->sc_ac.ac_if;
2079
2080 switch (act) {
2081 case DVACT_SUSPEND:
2082 if (ifp->if_flags & IFF_RUNNING)
2083 em_stop(sc, 0);
2084 break;
2085 case DVACT_RESUME:
2086 if (ifp->if_flags & IFF_UP)
2087 em_init(sc);
2088 break;
2089 }
2090 return (0);
2091 }
2092
2093 /*********************************************************************
2094 *
2095 * Workaround for SmartSpeed on 82541 and 82547 controllers
2096 *
2097 **********************************************************************/
2098 void
em_smartspeed(struct em_softc * sc)2099 em_smartspeed(struct em_softc *sc)
2100 {
2101 uint16_t phy_tmp;
2102
2103 if (sc->link_active || (sc->hw.phy_type != em_phy_igp) ||
2104 !sc->hw.autoneg || !(sc->hw.autoneg_advertised & ADVERTISE_1000_FULL))
2105 return;
2106
2107 if (sc->smartspeed == 0) {
2108 /* If Master/Slave config fault is asserted twice,
2109 * we assume back-to-back */
2110 em_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp);
2111 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2112 return;
2113 em_read_phy_reg(&sc->hw, PHY_1000T_STATUS, &phy_tmp);
2114 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2115 em_read_phy_reg(&sc->hw, PHY_1000T_CTRL,
2116 &phy_tmp);
2117 if (phy_tmp & CR_1000T_MS_ENABLE) {
2118 phy_tmp &= ~CR_1000T_MS_ENABLE;
2119 em_write_phy_reg(&sc->hw,
2120 PHY_1000T_CTRL, phy_tmp);
2121 sc->smartspeed++;
2122 if (sc->hw.autoneg &&
2123 !em_phy_setup_autoneg(&sc->hw) &&
2124 !em_read_phy_reg(&sc->hw, PHY_CTRL,
2125 &phy_tmp)) {
2126 phy_tmp |= (MII_CR_AUTO_NEG_EN |
2127 MII_CR_RESTART_AUTO_NEG);
2128 em_write_phy_reg(&sc->hw,
2129 PHY_CTRL, phy_tmp);
2130 }
2131 }
2132 }
2133 return;
2134 } else if (sc->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2135 /* If still no link, perhaps using 2/3 pair cable */
2136 em_read_phy_reg(&sc->hw, PHY_1000T_CTRL, &phy_tmp);
2137 phy_tmp |= CR_1000T_MS_ENABLE;
2138 em_write_phy_reg(&sc->hw, PHY_1000T_CTRL, phy_tmp);
2139 if (sc->hw.autoneg &&
2140 !em_phy_setup_autoneg(&sc->hw) &&
2141 !em_read_phy_reg(&sc->hw, PHY_CTRL, &phy_tmp)) {
2142 phy_tmp |= (MII_CR_AUTO_NEG_EN |
2143 MII_CR_RESTART_AUTO_NEG);
2144 em_write_phy_reg(&sc->hw, PHY_CTRL, phy_tmp);
2145 }
2146 }
2147 /* Restart process after EM_SMARTSPEED_MAX iterations */
2148 if (sc->smartspeed++ == EM_SMARTSPEED_MAX)
2149 sc->smartspeed = 0;
2150 }
2151
2152 /*
2153 * Manage DMA'able memory.
2154 */
2155 int
em_dma_malloc(struct em_softc * sc,bus_size_t size,struct em_dma_alloc * dma)2156 em_dma_malloc(struct em_softc *sc, bus_size_t size, struct em_dma_alloc *dma)
2157 {
2158 int r;
2159
2160 r = bus_dmamap_create(sc->sc_dmat, size, 1,
2161 size, 0, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map);
2162 if (r != 0)
2163 return (r);
2164
2165 r = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, &dma->dma_seg,
2166 1, &dma->dma_nseg, BUS_DMA_WAITOK | BUS_DMA_ZERO);
2167 if (r != 0)
2168 goto destroy;
2169
2170 r = bus_dmamem_map(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg, size,
2171 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT);
2172 if (r != 0)
2173 goto free;
2174
2175 r = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr, size,
2176 NULL, BUS_DMA_WAITOK);
2177 if (r != 0)
2178 goto unmap;
2179
2180 dma->dma_size = size;
2181 return (0);
2182
2183 unmap:
2184 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, size);
2185 free:
2186 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2187 destroy:
2188 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2189
2190 return (r);
2191 }
2192
2193 void
em_dma_free(struct em_softc * sc,struct em_dma_alloc * dma)2194 em_dma_free(struct em_softc *sc, struct em_dma_alloc *dma)
2195 {
2196 bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
2197 bus_dmamem_unmap(sc->sc_dmat, dma->dma_vaddr, dma->dma_size);
2198 bus_dmamem_free(sc->sc_dmat, &dma->dma_seg, dma->dma_nseg);
2199 bus_dmamap_destroy(sc->sc_dmat, dma->dma_map);
2200 }
2201
2202 /*********************************************************************
2203 *
2204 * Allocate memory for tx_buffer structures. The tx_buffer stores all
2205 * the information needed to transmit a packet on the wire.
2206 *
2207 **********************************************************************/
2208 int
em_allocate_transmit_structures(struct em_softc * sc)2209 em_allocate_transmit_structures(struct em_softc *sc)
2210 {
2211 struct em_queue *que;
2212
2213 FOREACH_QUEUE(sc, que) {
2214 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2215 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2216 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2217
2218 que->tx.sc_tx_pkts_ring = mallocarray(sc->sc_tx_slots,
2219 sizeof(*que->tx.sc_tx_pkts_ring), M_DEVBUF, M_NOWAIT | M_ZERO);
2220 if (que->tx.sc_tx_pkts_ring == NULL) {
2221 printf("%s: Unable to allocate tx_buffer memory\n",
2222 DEVNAME(sc));
2223 return (ENOMEM);
2224 }
2225 }
2226
2227 return (0);
2228 }
2229
2230 /*********************************************************************
2231 *
2232 * Allocate and initialize transmit structures.
2233 *
2234 **********************************************************************/
2235 int
em_setup_transmit_structures(struct em_softc * sc)2236 em_setup_transmit_structures(struct em_softc *sc)
2237 {
2238 struct em_queue *que;
2239 struct em_packet *pkt;
2240 int error, i;
2241
2242 if ((error = em_allocate_transmit_structures(sc)) != 0)
2243 goto fail;
2244
2245 FOREACH_QUEUE(sc, que) {
2246 bzero((void *) que->tx.sc_tx_desc_ring,
2247 (sizeof(struct em_tx_desc)) * sc->sc_tx_slots);
2248
2249 for (i = 0; i < sc->sc_tx_slots; i++) {
2250 pkt = &que->tx.sc_tx_pkts_ring[i];
2251 error = bus_dmamap_create(sc->sc_dmat, EM_TSO_SIZE,
2252 EM_MAX_SCATTER / (sc->pcix_82544 ? 2 : 1),
2253 EM_TSO_SEG_SIZE, 0, BUS_DMA_NOWAIT, &pkt->pkt_map);
2254 if (error != 0) {
2255 printf("%s: Unable to create TX DMA map\n",
2256 DEVNAME(sc));
2257 goto fail;
2258 }
2259 }
2260
2261 que->tx.sc_tx_desc_head = 0;
2262 que->tx.sc_tx_desc_tail = 0;
2263
2264 /* Set checksum context */
2265 que->tx.active_checksum_context = OFFLOAD_NONE;
2266 }
2267
2268 return (0);
2269
2270 fail:
2271 em_free_transmit_structures(sc);
2272 return (error);
2273 }
2274
2275 /*********************************************************************
2276 *
2277 * Enable transmit unit.
2278 *
2279 **********************************************************************/
2280 void
em_initialize_transmit_unit(struct em_softc * sc)2281 em_initialize_transmit_unit(struct em_softc *sc)
2282 {
2283 u_int32_t reg_tctl, reg_tipg = 0;
2284 u_int64_t bus_addr;
2285 struct em_queue *que;
2286
2287 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
2288
2289 FOREACH_QUEUE(sc, que) {
2290 /* Setup the Base and Length of the Tx Descriptor Ring */
2291 bus_addr = que->tx.sc_tx_dma.dma_map->dm_segs[0].ds_addr;
2292 E1000_WRITE_REG(&sc->hw, TDLEN(que->me),
2293 sc->sc_tx_slots *
2294 sizeof(struct em_tx_desc));
2295 E1000_WRITE_REG(&sc->hw, TDBAH(que->me), (u_int32_t)(bus_addr >> 32));
2296 E1000_WRITE_REG(&sc->hw, TDBAL(que->me), (u_int32_t)bus_addr);
2297
2298 /* Setup the HW Tx Head and Tail descriptor pointers */
2299 E1000_WRITE_REG(&sc->hw, TDT(que->me), 0);
2300 E1000_WRITE_REG(&sc->hw, TDH(que->me), 0);
2301
2302 HW_DEBUGOUT2("Base = %x, Length = %x\n",
2303 E1000_READ_REG(&sc->hw, TDBAL(que->me)),
2304 E1000_READ_REG(&sc->hw, TDLEN(que->me)));
2305
2306 /* Set the default values for the Tx Inter Packet Gap timer */
2307 switch (sc->hw.mac_type) {
2308 case em_82542_rev2_0:
2309 case em_82542_rev2_1:
2310 reg_tipg = DEFAULT_82542_TIPG_IPGT;
2311 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2312 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2313 break;
2314 case em_80003es2lan:
2315 reg_tipg = DEFAULT_82543_TIPG_IPGR1;
2316 reg_tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2317 break;
2318 default:
2319 if (sc->hw.media_type == em_media_type_fiber ||
2320 sc->hw.media_type == em_media_type_internal_serdes)
2321 reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2322 else
2323 reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2324 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2325 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2326 }
2327
2328
2329 E1000_WRITE_REG(&sc->hw, TIPG, reg_tipg);
2330 E1000_WRITE_REG(&sc->hw, TIDV, sc->tx_int_delay);
2331 if (sc->hw.mac_type >= em_82540)
2332 E1000_WRITE_REG(&sc->hw, TADV, sc->tx_abs_int_delay);
2333
2334 /* Setup Transmit Descriptor Base Settings */
2335 que->tx.sc_txd_cmd = E1000_TXD_CMD_IFCS;
2336
2337 if (sc->hw.mac_type == em_82575 || sc->hw.mac_type == em_82580 ||
2338 sc->hw.mac_type == em_82576 ||
2339 sc->hw.mac_type == em_i210 || sc->hw.mac_type == em_i350) {
2340 /* 82575/6 need to enable the TX queue and lack the IDE bit */
2341 reg_tctl = E1000_READ_REG(&sc->hw, TXDCTL(que->me));
2342 reg_tctl |= E1000_TXDCTL_QUEUE_ENABLE;
2343 E1000_WRITE_REG(&sc->hw, TXDCTL(que->me), reg_tctl);
2344 } else if (sc->tx_int_delay > 0)
2345 que->tx.sc_txd_cmd |= E1000_TXD_CMD_IDE;
2346 }
2347
2348 /* Program the Transmit Control Register */
2349 reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
2350 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2351 if (sc->hw.mac_type >= em_82571)
2352 reg_tctl |= E1000_TCTL_MULR;
2353 if (sc->link_duplex == FULL_DUPLEX)
2354 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2355 else
2356 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2357 /* This write will effectively turn on the transmit unit */
2358 E1000_WRITE_REG(&sc->hw, TCTL, reg_tctl);
2359
2360 /* SPT Si errata workaround to avoid data corruption */
2361
2362 if (sc->hw.mac_type == em_pch_spt) {
2363 uint32_t reg_val;
2364
2365 reg_val = EM_READ_REG(&sc->hw, E1000_IOSFPC);
2366 reg_val |= E1000_RCTL_RDMTS_HEX;
2367 EM_WRITE_REG(&sc->hw, E1000_IOSFPC, reg_val);
2368
2369 reg_val = E1000_READ_REG(&sc->hw, TARC0);
2370 /* i218-i219 Specification Update 1.5.4.5 */
2371 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ;
2372 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ;
2373 E1000_WRITE_REG(&sc->hw, TARC0, reg_val);
2374 }
2375 }
2376
2377 /*********************************************************************
2378 *
2379 * Free all transmit related data structures.
2380 *
2381 **********************************************************************/
2382 void
em_free_transmit_structures(struct em_softc * sc)2383 em_free_transmit_structures(struct em_softc *sc)
2384 {
2385 struct em_queue *que;
2386 struct em_packet *pkt;
2387 int i;
2388
2389 INIT_DEBUGOUT("free_transmit_structures: begin");
2390
2391 FOREACH_QUEUE(sc, que) {
2392 if (que->tx.sc_tx_pkts_ring != NULL) {
2393 for (i = 0; i < sc->sc_tx_slots; i++) {
2394 pkt = &que->tx.sc_tx_pkts_ring[i];
2395
2396 if (pkt->pkt_m != NULL) {
2397 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
2398 0, pkt->pkt_map->dm_mapsize,
2399 BUS_DMASYNC_POSTWRITE);
2400 bus_dmamap_unload(sc->sc_dmat,
2401 pkt->pkt_map);
2402
2403 m_freem(pkt->pkt_m);
2404 pkt->pkt_m = NULL;
2405 }
2406
2407 if (pkt->pkt_map != NULL) {
2408 bus_dmamap_destroy(sc->sc_dmat,
2409 pkt->pkt_map);
2410 pkt->pkt_map = NULL;
2411 }
2412 }
2413
2414 free(que->tx.sc_tx_pkts_ring, M_DEVBUF,
2415 sc->sc_tx_slots * sizeof(*que->tx.sc_tx_pkts_ring));
2416 que->tx.sc_tx_pkts_ring = NULL;
2417 }
2418
2419 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2420 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2421 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2422 }
2423 }
2424
2425 u_int
em_tso_setup(struct em_queue * que,struct mbuf * mp,u_int head,u_int32_t * olinfo_status,u_int32_t * cmd_type_len)2426 em_tso_setup(struct em_queue *que, struct mbuf *mp, u_int head,
2427 u_int32_t *olinfo_status, u_int32_t *cmd_type_len)
2428 {
2429 struct ether_extracted ext;
2430 struct e1000_adv_tx_context_desc *TD;
2431 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
2432
2433 *olinfo_status = 0;
2434 *cmd_type_len = 0;
2435 TD = (struct e1000_adv_tx_context_desc *)&que->tx.sc_tx_desc_ring[head];
2436
2437 #if NVLAN > 0
2438 if (ISSET(mp->m_flags, M_VLANTAG)) {
2439 uint32_t vtag = mp->m_pkthdr.ether_vtag;
2440 vlan_macip_lens |= vtag << E1000_ADVTXD_VLAN_SHIFT;
2441 *cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2442 }
2443 #endif
2444
2445 ether_extract_headers(mp, &ext);
2446 if (ext.tcp == NULL || mp->m_pkthdr.ph_mss == 0)
2447 goto out;
2448
2449 vlan_macip_lens |= (sizeof(*ext.eh) << E1000_ADVTXD_MACLEN_SHIFT);
2450
2451 if (ext.ip4) {
2452 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
2453 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2454 #ifdef INET6
2455 } else if (ext.ip6) {
2456 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
2457 #endif
2458 } else {
2459 goto out;
2460 }
2461
2462 *cmd_type_len |= E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS;
2463 *cmd_type_len |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DCMD_TSE;
2464 *olinfo_status |= ext.paylen << E1000_ADVTXD_PAYLEN_SHIFT;
2465 vlan_macip_lens |= ext.iphlen;
2466 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
2467
2468 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
2469 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2470
2471 mss_l4len_idx |= mp->m_pkthdr.ph_mss << E1000_ADVTXD_MSS_SHIFT;
2472 mss_l4len_idx |= ext.tcphlen << E1000_ADVTXD_L4LEN_SHIFT;
2473 /* 82575 needs the queue index added */
2474 if (que->sc->hw.mac_type == em_82575)
2475 mss_l4len_idx |= (que->me & 0xff) << 4;
2476
2477 htolem32(&TD->vlan_macip_lens, vlan_macip_lens);
2478 htolem32(&TD->type_tucmd_mlhl, type_tucmd_mlhl);
2479 htolem32(&TD->u.seqnum_seed, 0);
2480 htolem32(&TD->mss_l4len_idx, mss_l4len_idx);
2481
2482 tcpstat_add(tcps_outpkttso, (ext.paylen + mp->m_pkthdr.ph_mss - 1) /
2483 mp->m_pkthdr.ph_mss);
2484
2485 return 1;
2486
2487 out:
2488 tcpstat_inc(tcps_outbadtso);
2489 return 0;
2490 }
2491
2492 u_int
em_tx_ctx_setup(struct em_queue * que,struct mbuf * mp,u_int head,u_int32_t * olinfo_status,u_int32_t * cmd_type_len)2493 em_tx_ctx_setup(struct em_queue *que, struct mbuf *mp, u_int head,
2494 u_int32_t *olinfo_status, u_int32_t *cmd_type_len)
2495 {
2496 struct ether_extracted ext;
2497 struct e1000_adv_tx_context_desc *TD;
2498 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0, mss_l4len_idx = 0;
2499 int off = 0;
2500
2501 *olinfo_status = 0;
2502 *cmd_type_len = 0;
2503 TD = (struct e1000_adv_tx_context_desc *)&que->tx.sc_tx_desc_ring[head];
2504
2505 #if NVLAN > 0
2506 if (ISSET(mp->m_flags, M_VLANTAG)) {
2507 uint32_t vtag = mp->m_pkthdr.ether_vtag;
2508 vlan_macip_lens |= vtag << E1000_ADVTXD_VLAN_SHIFT;
2509 *cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2510 off = 1;
2511 }
2512 #endif
2513
2514 ether_extract_headers(mp, &ext);
2515
2516 vlan_macip_lens |= (sizeof(*ext.eh) << E1000_ADVTXD_MACLEN_SHIFT);
2517
2518 if (ext.ip4) {
2519 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
2520 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) {
2521 *olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2522 off = 1;
2523 }
2524 #ifdef INET6
2525 } else if (ext.ip6) {
2526 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
2527 #endif
2528 }
2529
2530 *cmd_type_len |= E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS;
2531 *cmd_type_len |= E1000_ADVTXD_DCMD_DEXT;
2532 *olinfo_status |= mp->m_pkthdr.len << E1000_ADVTXD_PAYLEN_SHIFT;
2533 vlan_macip_lens |= ext.iphlen;
2534 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
2535
2536 if (ext.tcp) {
2537 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
2538 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) {
2539 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2540 off = 1;
2541 }
2542 } else if (ext.udp) {
2543 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
2544 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) {
2545 *olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2546 off = 1;
2547 }
2548 }
2549
2550 if (!off)
2551 return (0);
2552
2553 /* 82575 needs the queue index added */
2554 if (que->sc->hw.mac_type == em_82575)
2555 mss_l4len_idx |= (que->me & 0xff) << 4;
2556
2557 htolem32(&TD->vlan_macip_lens, vlan_macip_lens);
2558 htolem32(&TD->type_tucmd_mlhl, type_tucmd_mlhl);
2559 htolem32(&TD->u.seqnum_seed, 0);
2560 htolem32(&TD->mss_l4len_idx, mss_l4len_idx);
2561
2562 return (1);
2563 }
2564
2565 /*********************************************************************
2566 *
2567 * The offload context needs to be set when we transfer the first
2568 * packet of a particular protocol (TCP/UDP). We change the
2569 * context only if the protocol type changes.
2570 *
2571 **********************************************************************/
2572 u_int
em_transmit_checksum_setup(struct em_queue * que,struct mbuf * mp,u_int head,u_int32_t * txd_upper,u_int32_t * txd_lower)2573 em_transmit_checksum_setup(struct em_queue *que, struct mbuf *mp, u_int head,
2574 u_int32_t *txd_upper, u_int32_t *txd_lower)
2575 {
2576 struct em_context_desc *TXD;
2577
2578 if (mp->m_pkthdr.csum_flags & M_TCP_CSUM_OUT) {
2579 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2580 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2581 if (que->tx.active_checksum_context == OFFLOAD_TCP_IP)
2582 return (0);
2583 else
2584 que->tx.active_checksum_context = OFFLOAD_TCP_IP;
2585 } else if (mp->m_pkthdr.csum_flags & M_UDP_CSUM_OUT) {
2586 *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2587 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2588 if (que->tx.active_checksum_context == OFFLOAD_UDP_IP)
2589 return (0);
2590 else
2591 que->tx.active_checksum_context = OFFLOAD_UDP_IP;
2592 } else {
2593 *txd_upper = 0;
2594 *txd_lower = 0;
2595 return (0);
2596 }
2597
2598 /* If we reach this point, the checksum offload context
2599 * needs to be reset.
2600 */
2601 TXD = (struct em_context_desc *)&que->tx.sc_tx_desc_ring[head];
2602
2603 TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2604 TXD->lower_setup.ip_fields.ipcso =
2605 ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2606 TXD->lower_setup.ip_fields.ipcse =
2607 htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2608
2609 TXD->upper_setup.tcp_fields.tucss =
2610 ETHER_HDR_LEN + sizeof(struct ip);
2611 TXD->upper_setup.tcp_fields.tucse = htole16(0);
2612
2613 if (que->tx.active_checksum_context == OFFLOAD_TCP_IP) {
2614 TXD->upper_setup.tcp_fields.tucso =
2615 ETHER_HDR_LEN + sizeof(struct ip) +
2616 offsetof(struct tcphdr, th_sum);
2617 } else if (que->tx.active_checksum_context == OFFLOAD_UDP_IP) {
2618 TXD->upper_setup.tcp_fields.tucso =
2619 ETHER_HDR_LEN + sizeof(struct ip) +
2620 offsetof(struct udphdr, uh_sum);
2621 }
2622
2623 TXD->tcp_seg_setup.data = htole32(0);
2624 TXD->cmd_and_length = htole32(que->tx.sc_txd_cmd | E1000_TXD_CMD_DEXT);
2625
2626 return (1);
2627 }
2628
2629 /**********************************************************************
2630 *
2631 * Examine each tx_buffer in the used queue. If the hardware is done
2632 * processing the packet then free associated resources. The
2633 * tx_buffer is put back on the free queue.
2634 *
2635 **********************************************************************/
2636 void
em_txeof(struct em_queue * que)2637 em_txeof(struct em_queue *que)
2638 {
2639 struct em_softc *sc = que->sc;
2640 struct ifnet *ifp = &sc->sc_ac.ac_if;
2641 struct em_packet *pkt;
2642 struct em_tx_desc *desc;
2643 u_int head, tail;
2644 u_int free = 0;
2645
2646 head = que->tx.sc_tx_desc_head;
2647 tail = que->tx.sc_tx_desc_tail;
2648
2649 if (head == tail)
2650 return;
2651
2652 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2653 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2654 BUS_DMASYNC_POSTREAD);
2655
2656 do {
2657 pkt = &que->tx.sc_tx_pkts_ring[tail];
2658 desc = &que->tx.sc_tx_desc_ring[pkt->pkt_eop];
2659
2660 if (!ISSET(desc->upper.fields.status, E1000_TXD_STAT_DD))
2661 break;
2662
2663 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
2664 0, pkt->pkt_map->dm_mapsize,
2665 BUS_DMASYNC_POSTWRITE);
2666 bus_dmamap_unload(sc->sc_dmat, pkt->pkt_map);
2667
2668 KASSERT(pkt->pkt_m != NULL);
2669
2670 m_freem(pkt->pkt_m);
2671 pkt->pkt_m = NULL;
2672
2673 tail = pkt->pkt_eop;
2674
2675 if (++tail == sc->sc_tx_slots)
2676 tail = 0;
2677
2678 free++;
2679 } while (tail != head);
2680
2681 bus_dmamap_sync(sc->sc_dmat, que->tx.sc_tx_dma.dma_map,
2682 0, que->tx.sc_tx_dma.dma_map->dm_mapsize,
2683 BUS_DMASYNC_PREREAD);
2684
2685 if (free == 0)
2686 return;
2687
2688 que->tx.sc_tx_desc_tail = tail;
2689
2690 if (ifq_is_oactive(&ifp->if_snd))
2691 ifq_restart(&ifp->if_snd);
2692 else if (tail == head)
2693 ifp->if_timer = 0;
2694 }
2695
2696 /*********************************************************************
2697 *
2698 * Get a buffer from system mbuf buffer pool.
2699 *
2700 **********************************************************************/
2701 int
em_get_buf(struct em_queue * que,int i)2702 em_get_buf(struct em_queue *que, int i)
2703 {
2704 struct em_softc *sc = que->sc;
2705 struct mbuf *m;
2706 struct em_packet *pkt;
2707 struct em_rx_desc *desc;
2708 int error;
2709
2710 pkt = &que->rx.sc_rx_pkts_ring[i];
2711 desc = &que->rx.sc_rx_desc_ring[i];
2712
2713 KASSERT(pkt->pkt_m == NULL);
2714
2715 m = MCLGETL(NULL, M_DONTWAIT, EM_MCLBYTES);
2716 if (m == NULL) {
2717 sc->mbuf_cluster_failed++;
2718 return (ENOBUFS);
2719 }
2720 m->m_len = m->m_pkthdr.len = EM_MCLBYTES;
2721 m_adj(m, ETHER_ALIGN);
2722
2723 error = bus_dmamap_load_mbuf(sc->sc_dmat, pkt->pkt_map,
2724 m, BUS_DMA_NOWAIT);
2725 if (error) {
2726 m_freem(m);
2727 return (error);
2728 }
2729
2730 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
2731 0, pkt->pkt_map->dm_mapsize,
2732 BUS_DMASYNC_PREREAD);
2733 pkt->pkt_m = m;
2734
2735 memset(desc, 0, sizeof(*desc));
2736 htolem64(&desc->buffer_addr, pkt->pkt_map->dm_segs[0].ds_addr);
2737
2738 return (0);
2739 }
2740
2741 /*********************************************************************
2742 *
2743 * Allocate memory for rx_buffer structures. Since we use one
2744 * rx_buffer per received packet, the maximum number of rx_buffer's
2745 * that we'll need is equal to the number of receive descriptors
2746 * that we've allocated.
2747 *
2748 **********************************************************************/
2749 int
em_allocate_receive_structures(struct em_softc * sc)2750 em_allocate_receive_structures(struct em_softc *sc)
2751 {
2752 struct em_queue *que;
2753 struct em_packet *pkt;
2754 int i;
2755 int error;
2756
2757 FOREACH_QUEUE(sc, que) {
2758 que->rx.sc_rx_pkts_ring = mallocarray(sc->sc_rx_slots,
2759 sizeof(*que->rx.sc_rx_pkts_ring),
2760 M_DEVBUF, M_NOWAIT | M_ZERO);
2761 if (que->rx.sc_rx_pkts_ring == NULL) {
2762 printf("%s: Unable to allocate rx_buffer memory\n",
2763 DEVNAME(sc));
2764 return (ENOMEM);
2765 }
2766
2767 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
2768 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
2769 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2770
2771 for (i = 0; i < sc->sc_rx_slots; i++) {
2772 pkt = &que->rx.sc_rx_pkts_ring[i];
2773
2774 error = bus_dmamap_create(sc->sc_dmat, EM_MCLBYTES, 1,
2775 EM_MCLBYTES, 0, BUS_DMA_NOWAIT, &pkt->pkt_map);
2776 if (error != 0) {
2777 printf("%s: em_allocate_receive_structures: "
2778 "bus_dmamap_create failed; error %u\n",
2779 DEVNAME(sc), error);
2780 goto fail;
2781 }
2782
2783 pkt->pkt_m = NULL;
2784 }
2785 }
2786
2787 return (0);
2788
2789 fail:
2790 em_free_receive_structures(sc);
2791 return (error);
2792 }
2793
2794 /*********************************************************************
2795 *
2796 * Allocate and initialize receive structures.
2797 *
2798 **********************************************************************/
2799 int
em_setup_receive_structures(struct em_softc * sc)2800 em_setup_receive_structures(struct em_softc *sc)
2801 {
2802 struct ifnet *ifp = &sc->sc_ac.ac_if;
2803 struct em_queue *que;
2804 u_int lwm;
2805
2806 if (em_allocate_receive_structures(sc))
2807 return (ENOMEM);
2808
2809 FOREACH_QUEUE(sc, que) {
2810 memset(que->rx.sc_rx_desc_ring, 0,
2811 sc->sc_rx_slots * sizeof(*que->rx.sc_rx_desc_ring));
2812
2813 /* Setup our descriptor pointers */
2814 que->rx.sc_rx_desc_tail = 0;
2815 que->rx.sc_rx_desc_head = sc->sc_rx_slots - 1;
2816
2817 lwm = max(4, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1));
2818 if_rxr_init(&que->rx.sc_rx_ring, lwm, sc->sc_rx_slots);
2819
2820 if (em_rxfill(que) == 0) {
2821 printf("%s: unable to fill any rx descriptors\n",
2822 DEVNAME(sc));
2823 return (ENOMEM);
2824 }
2825 }
2826
2827 return (0);
2828 }
2829
2830 /*********************************************************************
2831 *
2832 * Enable receive unit.
2833 *
2834 **********************************************************************/
2835 void
em_initialize_receive_unit(struct em_softc * sc)2836 em_initialize_receive_unit(struct em_softc *sc)
2837 {
2838 struct em_queue *que;
2839 u_int32_t reg_rctl;
2840 u_int32_t reg_rxcsum;
2841 u_int32_t reg_srrctl;
2842 u_int64_t bus_addr;
2843
2844 INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2845
2846 /* Make sure receives are disabled while setting up the descriptor ring */
2847 E1000_WRITE_REG(&sc->hw, RCTL, 0);
2848
2849 /* Set the Receive Delay Timer Register */
2850 E1000_WRITE_REG(&sc->hw, RDTR,
2851 sc->rx_int_delay | E1000_RDT_FPDB);
2852
2853 if (sc->hw.mac_type >= em_82540) {
2854 if (sc->rx_int_delay)
2855 E1000_WRITE_REG(&sc->hw, RADV, sc->rx_abs_int_delay);
2856
2857 /* Set the interrupt throttling rate. Value is calculated
2858 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) */
2859 E1000_WRITE_REG(&sc->hw, ITR, DEFAULT_ITR);
2860 }
2861
2862 /* Setup the Receive Control Register */
2863 reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2864 E1000_RCTL_RDMTS_HALF |
2865 (sc->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2866
2867 if (sc->hw.tbi_compatibility_on == TRUE)
2868 reg_rctl |= E1000_RCTL_SBP;
2869
2870 /*
2871 * The i350 has a bug where it always strips the CRC whether
2872 * asked to or not. So ask for stripped CRC here and
2873 * cope in rxeof
2874 */
2875 if (sc->hw.mac_type == em_i210 || sc->hw.mac_type == em_i350)
2876 reg_rctl |= E1000_RCTL_SECRC;
2877
2878 switch (sc->sc_rx_buffer_len) {
2879 default:
2880 case EM_RXBUFFER_2048:
2881 reg_rctl |= E1000_RCTL_SZ_2048;
2882 break;
2883 case EM_RXBUFFER_4096:
2884 reg_rctl |= E1000_RCTL_SZ_4096|E1000_RCTL_BSEX|E1000_RCTL_LPE;
2885 break;
2886 case EM_RXBUFFER_8192:
2887 reg_rctl |= E1000_RCTL_SZ_8192|E1000_RCTL_BSEX|E1000_RCTL_LPE;
2888 break;
2889 case EM_RXBUFFER_16384:
2890 reg_rctl |= E1000_RCTL_SZ_16384|E1000_RCTL_BSEX|E1000_RCTL_LPE;
2891 break;
2892 }
2893
2894 if (sc->hw.max_frame_size != ETHER_MAX_LEN)
2895 reg_rctl |= E1000_RCTL_LPE;
2896
2897 /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2898 if (sc->hw.mac_type >= em_82543) {
2899 reg_rxcsum = E1000_READ_REG(&sc->hw, RXCSUM);
2900 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2901 E1000_WRITE_REG(&sc->hw, RXCSUM, reg_rxcsum);
2902 }
2903
2904 /*
2905 * XXX TEMPORARY WORKAROUND: on some systems with 82573
2906 * long latencies are observed, like Lenovo X60.
2907 */
2908 if (sc->hw.mac_type == em_82573)
2909 E1000_WRITE_REG(&sc->hw, RDTR, 0x20);
2910
2911 FOREACH_QUEUE(sc, que) {
2912 if (sc->num_queues > 1) {
2913 /*
2914 * Disable Drop Enable for every queue, default has
2915 * it enabled for queues > 0
2916 */
2917 reg_srrctl = E1000_READ_REG(&sc->hw, SRRCTL(que->me));
2918 reg_srrctl &= ~E1000_SRRCTL_DROP_EN;
2919 E1000_WRITE_REG(&sc->hw, SRRCTL(que->me), reg_srrctl);
2920 }
2921
2922 /* Setup the Base and Length of the Rx Descriptor Ring */
2923 bus_addr = que->rx.sc_rx_dma.dma_map->dm_segs[0].ds_addr;
2924 E1000_WRITE_REG(&sc->hw, RDLEN(que->me),
2925 sc->sc_rx_slots * sizeof(*que->rx.sc_rx_desc_ring));
2926 E1000_WRITE_REG(&sc->hw, RDBAH(que->me), (u_int32_t)(bus_addr >> 32));
2927 E1000_WRITE_REG(&sc->hw, RDBAL(que->me), (u_int32_t)bus_addr);
2928
2929 if (sc->hw.mac_type == em_82575 || sc->hw.mac_type == em_82580 ||
2930 sc->hw.mac_type == em_82576 ||
2931 sc->hw.mac_type == em_i210 || sc->hw.mac_type == em_i350) {
2932 /* 82575/6 need to enable the RX queue */
2933 uint32_t reg;
2934 reg = E1000_READ_REG(&sc->hw, RXDCTL(que->me));
2935 reg |= E1000_RXDCTL_QUEUE_ENABLE;
2936 E1000_WRITE_REG(&sc->hw, RXDCTL(que->me), reg);
2937 }
2938 }
2939
2940 /* Enable Receives */
2941 E1000_WRITE_REG(&sc->hw, RCTL, reg_rctl);
2942
2943 /* Setup the HW Rx Head and Tail Descriptor Pointers */
2944 FOREACH_QUEUE(sc, que) {
2945 E1000_WRITE_REG(&sc->hw, RDH(que->me), 0);
2946 E1000_WRITE_REG(&sc->hw, RDT(que->me), que->rx.sc_rx_desc_head);
2947 }
2948 }
2949
2950 /*********************************************************************
2951 *
2952 * Free receive related data structures.
2953 *
2954 **********************************************************************/
2955 void
em_free_receive_structures(struct em_softc * sc)2956 em_free_receive_structures(struct em_softc *sc)
2957 {
2958 struct em_queue *que;
2959 struct em_packet *pkt;
2960 int i;
2961
2962 INIT_DEBUGOUT("free_receive_structures: begin");
2963
2964 FOREACH_QUEUE(sc, que) {
2965 if_rxr_init(&que->rx.sc_rx_ring, 0, 0);
2966
2967 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
2968 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
2969 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2970
2971 if (que->rx.sc_rx_pkts_ring != NULL) {
2972 for (i = 0; i < sc->sc_rx_slots; i++) {
2973 pkt = &que->rx.sc_rx_pkts_ring[i];
2974 if (pkt->pkt_m != NULL) {
2975 bus_dmamap_sync(sc->sc_dmat,
2976 pkt->pkt_map,
2977 0, pkt->pkt_map->dm_mapsize,
2978 BUS_DMASYNC_POSTREAD);
2979 bus_dmamap_unload(sc->sc_dmat,
2980 pkt->pkt_map);
2981 m_freem(pkt->pkt_m);
2982 pkt->pkt_m = NULL;
2983 }
2984 bus_dmamap_destroy(sc->sc_dmat, pkt->pkt_map);
2985 }
2986
2987 free(que->rx.sc_rx_pkts_ring, M_DEVBUF,
2988 sc->sc_rx_slots * sizeof(*que->rx.sc_rx_pkts_ring));
2989 que->rx.sc_rx_pkts_ring = NULL;
2990 }
2991
2992 if (que->rx.fmp != NULL) {
2993 m_freem(que->rx.fmp);
2994 que->rx.fmp = NULL;
2995 que->rx.lmp = NULL;
2996 }
2997 }
2998 }
2999
3000 int
em_rxfill(struct em_queue * que)3001 em_rxfill(struct em_queue *que)
3002 {
3003 struct em_softc *sc = que->sc;
3004 u_int slots;
3005 int post = 0;
3006 int i;
3007
3008 i = que->rx.sc_rx_desc_head;
3009
3010 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3011 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3012 BUS_DMASYNC_POSTWRITE);
3013
3014 for (slots = if_rxr_get(&que->rx.sc_rx_ring, sc->sc_rx_slots);
3015 slots > 0; slots--) {
3016 if (++i == sc->sc_rx_slots)
3017 i = 0;
3018
3019 if (em_get_buf(que, i) != 0)
3020 break;
3021
3022 que->rx.sc_rx_desc_head = i;
3023 post = 1;
3024 }
3025
3026 if_rxr_put(&que->rx.sc_rx_ring, slots);
3027
3028 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3029 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3030 BUS_DMASYNC_PREWRITE);
3031
3032 return (post);
3033 }
3034
3035 void
em_rxrefill(void * arg)3036 em_rxrefill(void *arg)
3037 {
3038 struct em_queue *que = arg;
3039 int s;
3040
3041 s = splnet();
3042 em_rxrefill_locked(que);
3043 splx(s);
3044 }
3045
3046 void
em_rxrefill_locked(struct em_queue * que)3047 em_rxrefill_locked(struct em_queue *que)
3048 {
3049 struct em_softc *sc = que->sc;
3050
3051 if (em_rxfill(que))
3052 E1000_WRITE_REG(&sc->hw, RDT(que->me), que->rx.sc_rx_desc_head);
3053 else if (if_rxr_needrefill(&que->rx.sc_rx_ring))
3054 timeout_add(&que->rx_refill, 1);
3055 }
3056
3057 /*********************************************************************
3058 *
3059 * This routine executes in interrupt context. It replenishes
3060 * the mbufs in the descriptor and sends data which has been
3061 * dma'ed into host memory to upper layer.
3062 *
3063 *********************************************************************/
3064 int
em_rxeof(struct em_queue * que)3065 em_rxeof(struct em_queue *que)
3066 {
3067 struct em_softc *sc = que->sc;
3068 struct ifnet *ifp = &sc->sc_ac.ac_if;
3069 struct mbuf_list ml = MBUF_LIST_INITIALIZER();
3070 struct mbuf *m;
3071 u_int8_t accept_frame = 0;
3072 u_int8_t eop = 0;
3073 u_int16_t len, desc_len, prev_len_adj;
3074 int i, rv = 0;
3075
3076 /* Pointer to the receive descriptor being examined. */
3077 struct em_rx_desc *desc;
3078 struct em_packet *pkt;
3079 u_int8_t status;
3080
3081 if (if_rxr_inuse(&que->rx.sc_rx_ring) == 0)
3082 return (0);
3083
3084 i = que->rx.sc_rx_desc_tail;
3085
3086 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3087 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3088 BUS_DMASYNC_POSTREAD);
3089
3090 do {
3091 m = NULL;
3092
3093 pkt = &que->rx.sc_rx_pkts_ring[i];
3094 desc = &que->rx.sc_rx_desc_ring[i];
3095
3096 status = desc->status;
3097 if (!ISSET(status, E1000_RXD_STAT_DD))
3098 break;
3099
3100 /* pull the mbuf off the ring */
3101 bus_dmamap_sync(sc->sc_dmat, pkt->pkt_map,
3102 0, pkt->pkt_map->dm_mapsize,
3103 BUS_DMASYNC_POSTREAD);
3104 bus_dmamap_unload(sc->sc_dmat, pkt->pkt_map);
3105 m = pkt->pkt_m;
3106 pkt->pkt_m = NULL;
3107
3108 KASSERT(m != NULL);
3109
3110 if_rxr_put(&que->rx.sc_rx_ring, 1);
3111 rv = 1;
3112
3113 accept_frame = 1;
3114 prev_len_adj = 0;
3115 desc_len = letoh16(desc->length);
3116
3117 if (status & E1000_RXD_STAT_EOP) {
3118 eop = 1;
3119 if (desc_len < ETHER_CRC_LEN) {
3120 len = 0;
3121 prev_len_adj = ETHER_CRC_LEN - desc_len;
3122 } else if (sc->hw.mac_type == em_i210 ||
3123 sc->hw.mac_type == em_i350)
3124 len = desc_len;
3125 else
3126 len = desc_len - ETHER_CRC_LEN;
3127 } else {
3128 eop = 0;
3129 len = desc_len;
3130 }
3131
3132 if (desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3133 u_int8_t last_byte;
3134 u_int32_t pkt_len = desc_len;
3135
3136 if (que->rx.fmp != NULL)
3137 pkt_len += que->rx.fmp->m_pkthdr.len;
3138
3139 last_byte = *(mtod(m, caddr_t) + desc_len - 1);
3140 if (TBI_ACCEPT(&sc->hw, status, desc->errors,
3141 pkt_len, last_byte)) {
3142 #if NKSTAT > 0
3143 em_tbi_adjust_stats(sc,
3144 pkt_len, sc->hw.mac_addr);
3145 #endif
3146 if (len > 0)
3147 len--;
3148 } else
3149 accept_frame = 0;
3150 }
3151
3152 if (accept_frame) {
3153 /* Assign correct length to the current fragment */
3154 m->m_len = len;
3155
3156 if (que->rx.fmp == NULL) {
3157 m->m_pkthdr.len = m->m_len;
3158 que->rx.fmp = m; /* Store the first mbuf */
3159 que->rx.lmp = m;
3160 } else {
3161 /* Chain mbuf's together */
3162 m->m_flags &= ~M_PKTHDR;
3163 /*
3164 * Adjust length of previous mbuf in chain if
3165 * we received less than 4 bytes in the last
3166 * descriptor.
3167 */
3168 if (prev_len_adj > 0) {
3169 que->rx.lmp->m_len -= prev_len_adj;
3170 que->rx.fmp->m_pkthdr.len -= prev_len_adj;
3171 }
3172 que->rx.lmp->m_next = m;
3173 que->rx.lmp = m;
3174 que->rx.fmp->m_pkthdr.len += m->m_len;
3175 }
3176
3177 if (eop) {
3178 m = que->rx.fmp;
3179
3180 em_receive_checksum(sc, desc, m);
3181 #if NVLAN > 0
3182 if (desc->status & E1000_RXD_STAT_VP) {
3183 m->m_pkthdr.ether_vtag =
3184 letoh16(desc->special);
3185 m->m_flags |= M_VLANTAG;
3186 }
3187 #endif
3188 ml_enqueue(&ml, m);
3189
3190 que->rx.fmp = NULL;
3191 que->rx.lmp = NULL;
3192 }
3193 } else {
3194 que->rx.dropped_pkts++;
3195
3196 if (que->rx.fmp != NULL) {
3197 m_freem(que->rx.fmp);
3198 que->rx.fmp = NULL;
3199 que->rx.lmp = NULL;
3200 }
3201
3202 m_freem(m);
3203 }
3204
3205 /* Advance our pointers to the next descriptor. */
3206 if (++i == sc->sc_rx_slots)
3207 i = 0;
3208 } while (if_rxr_inuse(&que->rx.sc_rx_ring) > 0);
3209
3210 bus_dmamap_sync(sc->sc_dmat, que->rx.sc_rx_dma.dma_map,
3211 0, que->rx.sc_rx_dma.dma_map->dm_mapsize,
3212 BUS_DMASYNC_PREREAD);
3213
3214 que->rx.sc_rx_desc_tail = i;
3215
3216 if (ifiq_input(&ifp->if_rcv, &ml))
3217 if_rxr_livelocked(&que->rx.sc_rx_ring);
3218
3219 return (rv);
3220 }
3221
3222 /*********************************************************************
3223 *
3224 * Verify that the hardware indicated that the checksum is valid.
3225 * Inform the stack about the status of checksum so that stack
3226 * doesn't spend time verifying the checksum.
3227 *
3228 *********************************************************************/
3229 void
em_receive_checksum(struct em_softc * sc,struct em_rx_desc * rx_desc,struct mbuf * mp)3230 em_receive_checksum(struct em_softc *sc, struct em_rx_desc *rx_desc,
3231 struct mbuf *mp)
3232 {
3233 /* 82543 or newer only */
3234 if ((sc->hw.mac_type < em_82543) ||
3235 /* Ignore Checksum bit is set */
3236 (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3237 mp->m_pkthdr.csum_flags = 0;
3238 return;
3239 }
3240
3241 if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3242 /* Did it pass? */
3243 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3244 /* IP Checksum Good */
3245 mp->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK;
3246
3247 } else
3248 mp->m_pkthdr.csum_flags = 0;
3249 }
3250
3251 if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3252 /* Did it pass? */
3253 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE))
3254 mp->m_pkthdr.csum_flags |=
3255 M_TCP_CSUM_IN_OK | M_UDP_CSUM_IN_OK;
3256 }
3257 }
3258
3259 /*
3260 * This turns on the hardware offload of the VLAN
3261 * tag insertion and strip
3262 */
3263 void
em_enable_hw_vlans(struct em_softc * sc)3264 em_enable_hw_vlans(struct em_softc *sc)
3265 {
3266 uint32_t ctrl;
3267
3268 ctrl = E1000_READ_REG(&sc->hw, CTRL);
3269 ctrl |= E1000_CTRL_VME;
3270 E1000_WRITE_REG(&sc->hw, CTRL, ctrl);
3271 }
3272
3273 void
em_enable_intr(struct em_softc * sc)3274 em_enable_intr(struct em_softc *sc)
3275 {
3276 uint32_t mask;
3277
3278 if (sc->msix) {
3279 mask = sc->msix_queuesmask | sc->msix_linkmask;
3280 E1000_WRITE_REG(&sc->hw, EIAC, mask);
3281 E1000_WRITE_REG(&sc->hw, EIAM, mask);
3282 E1000_WRITE_REG(&sc->hw, EIMS, mask);
3283 E1000_WRITE_REG(&sc->hw, IMS, E1000_IMS_LSC);
3284 } else
3285 E1000_WRITE_REG(&sc->hw, IMS, (IMS_ENABLE_MASK));
3286 }
3287
3288 void
em_disable_intr(struct em_softc * sc)3289 em_disable_intr(struct em_softc *sc)
3290 {
3291 /*
3292 * The first version of 82542 had an errata where when link
3293 * was forced it would stay up even if the cable was disconnected
3294 * Sequence errors were used to detect the disconnect and then
3295 * the driver would unforce the link. This code is in the ISR.
3296 * For this to work correctly the Sequence error interrupt had
3297 * to be enabled all the time.
3298 */
3299 if (sc->msix) {
3300 E1000_WRITE_REG(&sc->hw, EIMC, ~0);
3301 E1000_WRITE_REG(&sc->hw, EIAC, 0);
3302 } else if (sc->hw.mac_type == em_82542_rev2_0)
3303 E1000_WRITE_REG(&sc->hw, IMC, (0xffffffff & ~E1000_IMC_RXSEQ));
3304 else
3305 E1000_WRITE_REG(&sc->hw, IMC, 0xffffffff);
3306 }
3307
3308 void
em_write_pci_cfg(struct em_hw * hw,uint32_t reg,uint16_t * value)3309 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3310 {
3311 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3312 pcireg_t val;
3313
3314 val = pci_conf_read(pa->pa_pc, pa->pa_tag, reg & ~0x3);
3315 if (reg & 0x2) {
3316 val &= 0x0000ffff;
3317 val |= (*value << 16);
3318 } else {
3319 val &= 0xffff0000;
3320 val |= *value;
3321 }
3322 pci_conf_write(pa->pa_pc, pa->pa_tag, reg & ~0x3, val);
3323 }
3324
3325 void
em_read_pci_cfg(struct em_hw * hw,uint32_t reg,uint16_t * value)3326 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3327 {
3328 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3329 pcireg_t val;
3330
3331 val = pci_conf_read(pa->pa_pc, pa->pa_tag, reg & ~0x3);
3332 if (reg & 0x2)
3333 *value = (val >> 16) & 0xffff;
3334 else
3335 *value = val & 0xffff;
3336 }
3337
3338 void
em_pci_set_mwi(struct em_hw * hw)3339 em_pci_set_mwi(struct em_hw *hw)
3340 {
3341 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3342
3343 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
3344 (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE));
3345 }
3346
3347 void
em_pci_clear_mwi(struct em_hw * hw)3348 em_pci_clear_mwi(struct em_hw *hw)
3349 {
3350 struct pci_attach_args *pa = &((struct em_osdep *)hw->back)->em_pa;
3351
3352 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
3353 (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE));
3354 }
3355
3356 /*
3357 * We may eventually really do this, but its unnecessary
3358 * for now so we just return unsupported.
3359 */
3360 int32_t
em_read_pcie_cap_reg(struct em_hw * hw,uint32_t reg,uint16_t * value)3361 em_read_pcie_cap_reg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3362 {
3363 return -E1000_NOT_IMPLEMENTED;
3364 }
3365
3366 /*********************************************************************
3367 * 82544 Coexistence issue workaround.
3368 * There are 2 issues.
3369 * 1. Transmit Hang issue.
3370 * To detect this issue, following equation can be used...
3371 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3372 * If SUM[3:0] is in between 1 to 4, we will have this issue.
3373 *
3374 * 2. DAC issue.
3375 * To detect this issue, following equation can be used...
3376 * SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3377 * If SUM[3:0] is in between 9 to c, we will have this issue.
3378 *
3379 *
3380 * WORKAROUND:
3381 * Make sure we do not have ending address as 1,2,3,4(Hang) or 9,a,b,c (DAC)
3382 *
3383 *** *********************************************************************/
3384 u_int32_t
em_fill_descriptors(u_int64_t address,u_int32_t length,PDESC_ARRAY desc_array)3385 em_fill_descriptors(u_int64_t address, u_int32_t length,
3386 PDESC_ARRAY desc_array)
3387 {
3388 /* Since issue is sensitive to length and address.*/
3389 /* Let us first check the address...*/
3390 u_int32_t safe_terminator;
3391 if (length <= 4) {
3392 desc_array->descriptor[0].address = address;
3393 desc_array->descriptor[0].length = length;
3394 desc_array->elements = 1;
3395 return desc_array->elements;
3396 }
3397 safe_terminator = (u_int32_t)((((u_int32_t)address & 0x7) + (length & 0xF)) & 0xF);
3398 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3399 if (safe_terminator == 0 ||
3400 (safe_terminator > 4 &&
3401 safe_terminator < 9) ||
3402 (safe_terminator > 0xC &&
3403 safe_terminator <= 0xF)) {
3404 desc_array->descriptor[0].address = address;
3405 desc_array->descriptor[0].length = length;
3406 desc_array->elements = 1;
3407 return desc_array->elements;
3408 }
3409
3410 desc_array->descriptor[0].address = address;
3411 desc_array->descriptor[0].length = length - 4;
3412 desc_array->descriptor[1].address = address + (length - 4);
3413 desc_array->descriptor[1].length = 4;
3414 desc_array->elements = 2;
3415 return desc_array->elements;
3416 }
3417
3418 /*
3419 * Disable the L0S and L1 LINK states.
3420 */
3421 void
em_disable_aspm(struct em_softc * sc)3422 em_disable_aspm(struct em_softc *sc)
3423 {
3424 int offset;
3425 pcireg_t val;
3426
3427 switch (sc->hw.mac_type) {
3428 case em_82571:
3429 case em_82572:
3430 case em_82573:
3431 case em_82574:
3432 break;
3433 default:
3434 return;
3435 }
3436
3437 if (!pci_get_capability(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
3438 PCI_CAP_PCIEXPRESS, &offset, NULL))
3439 return;
3440
3441 /* Disable PCIe Active State Power Management (ASPM). */
3442 val = pci_conf_read(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
3443 offset + PCI_PCIE_LCSR);
3444
3445 switch (sc->hw.mac_type) {
3446 case em_82571:
3447 case em_82572:
3448 val &= ~PCI_PCIE_LCSR_ASPM_L1;
3449 break;
3450 case em_82573:
3451 case em_82574:
3452 val &= ~(PCI_PCIE_LCSR_ASPM_L0S |
3453 PCI_PCIE_LCSR_ASPM_L1);
3454 break;
3455 default:
3456 break;
3457 }
3458
3459 pci_conf_write(sc->osdep.em_pa.pa_pc, sc->osdep.em_pa.pa_tag,
3460 offset + PCI_PCIE_LCSR, val);
3461 }
3462
3463 /*
3464 * em_flush_tx_ring - remove all descriptors from the tx_ring
3465 *
3466 * We want to clear all pending descriptors from the TX ring.
3467 * zeroing happens when the HW reads the regs. We assign the ring itself as
3468 * the data of the next descriptor. We don't care about the data we are about
3469 * to reset the HW.
3470 */
3471 void
em_flush_tx_ring(struct em_queue * que)3472 em_flush_tx_ring(struct em_queue *que)
3473 {
3474 struct em_softc *sc = que->sc;
3475 uint32_t tctl, txd_lower = E1000_TXD_CMD_IFCS;
3476 uint16_t size = 512;
3477 struct em_tx_desc *txd;
3478
3479 KASSERT(que->tx.sc_tx_desc_ring != NULL);
3480
3481 tctl = EM_READ_REG(&sc->hw, E1000_TCTL);
3482 EM_WRITE_REG(&sc->hw, E1000_TCTL, tctl | E1000_TCTL_EN);
3483
3484 KASSERT(EM_READ_REG(&sc->hw, E1000_TDT(que->me)) == que->tx.sc_tx_desc_head);
3485
3486 txd = &que->tx.sc_tx_desc_ring[que->tx.sc_tx_desc_head];
3487 txd->buffer_addr = que->tx.sc_tx_dma.dma_map->dm_segs[0].ds_addr;
3488 txd->lower.data = htole32(txd_lower | size);
3489 txd->upper.data = 0;
3490
3491 /* flush descriptors to memory before notifying the HW */
3492 bus_space_barrier(sc->osdep.mem_bus_space_tag,
3493 sc->osdep.mem_bus_space_handle, 0, 0, BUS_SPACE_BARRIER_WRITE);
3494
3495 if (++que->tx.sc_tx_desc_head == sc->sc_tx_slots)
3496 que->tx.sc_tx_desc_head = 0;
3497
3498 EM_WRITE_REG(&sc->hw, E1000_TDT(que->me), que->tx.sc_tx_desc_head);
3499 bus_space_barrier(sc->osdep.mem_bus_space_tag, sc->osdep.mem_bus_space_handle,
3500 0, 0, BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE);
3501 usec_delay(250);
3502 }
3503
3504 /*
3505 * em_flush_rx_ring - remove all descriptors from the rx_ring
3506 *
3507 * Mark all descriptors in the RX ring as consumed and disable the rx ring
3508 */
3509 void
em_flush_rx_ring(struct em_queue * que)3510 em_flush_rx_ring(struct em_queue *que)
3511 {
3512 uint32_t rctl, rxdctl;
3513 struct em_softc *sc = que->sc;
3514
3515 rctl = EM_READ_REG(&sc->hw, E1000_RCTL);
3516 EM_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3517 E1000_WRITE_FLUSH(&sc->hw);
3518 usec_delay(150);
3519
3520 rxdctl = EM_READ_REG(&sc->hw, E1000_RXDCTL(que->me));
3521 /* zero the lower 14 bits (prefetch and host thresholds) */
3522 rxdctl &= 0xffffc000;
3523 /*
3524 * update thresholds: prefetch threshold to 31, host threshold to 1
3525 * and make sure the granularity is "descriptors" and not "cache lines"
3526 */
3527 rxdctl |= (0x1F | (1 << 8) | E1000_RXDCTL_THRESH_UNIT_DESC);
3528 EM_WRITE_REG(&sc->hw, E1000_RXDCTL(que->me), rxdctl);
3529
3530 /* momentarily enable the RX ring for the changes to take effect */
3531 EM_WRITE_REG(&sc->hw, E1000_RCTL, rctl | E1000_RCTL_EN);
3532 E1000_WRITE_FLUSH(&sc->hw);
3533 usec_delay(150);
3534 EM_WRITE_REG(&sc->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3535 }
3536
3537 /*
3538 * em_flush_desc_rings - remove all descriptors from the descriptor rings
3539 *
3540 * In i219, the descriptor rings must be emptied before resetting the HW
3541 * or before changing the device state to D3 during runtime (runtime PM).
3542 *
3543 * Failure to do this will cause the HW to enter a unit hang state which can
3544 * only be released by PCI reset on the device
3545 *
3546 */
3547 void
em_flush_desc_rings(struct em_softc * sc)3548 em_flush_desc_rings(struct em_softc *sc)
3549 {
3550 struct em_queue *que = sc->queues; /* Use only first queue. */
3551 struct pci_attach_args *pa = &sc->osdep.em_pa;
3552 uint32_t fextnvm11, tdlen;
3553 uint16_t hang_state;
3554
3555 /* First, disable MULR fix in FEXTNVM11 */
3556 fextnvm11 = EM_READ_REG(&sc->hw, E1000_FEXTNVM11);
3557 fextnvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;
3558 EM_WRITE_REG(&sc->hw, E1000_FEXTNVM11, fextnvm11);
3559
3560 /* do nothing if we're not in faulty state, or if the queue is empty */
3561 tdlen = EM_READ_REG(&sc->hw, E1000_TDLEN(que->me));
3562 hang_state = pci_conf_read(pa->pa_pc, pa->pa_tag, PCICFG_DESC_RING_STATUS);
3563 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen)
3564 return;
3565 em_flush_tx_ring(que);
3566
3567 /* recheck, maybe the fault is caused by the rx ring */
3568 hang_state = pci_conf_read(pa->pa_pc, pa->pa_tag, PCICFG_DESC_RING_STATUS);
3569 if (hang_state & FLUSH_DESC_REQUIRED)
3570 em_flush_rx_ring(que);
3571 }
3572
3573 int
em_allocate_legacy(struct em_softc * sc)3574 em_allocate_legacy(struct em_softc *sc)
3575 {
3576 pci_intr_handle_t ih;
3577 const char *intrstr = NULL;
3578 struct pci_attach_args *pa = &sc->osdep.em_pa;
3579 pci_chipset_tag_t pc = pa->pa_pc;
3580
3581 if (pci_intr_map_msi(pa, &ih)) {
3582 if (pci_intr_map(pa, &ih)) {
3583 printf(": couldn't map interrupt\n");
3584 return (ENXIO);
3585 }
3586 sc->legacy_irq = 1;
3587 }
3588
3589 intrstr = pci_intr_string(pc, ih);
3590 sc->sc_intrhand = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
3591 em_intr, sc, DEVNAME(sc));
3592 if (sc->sc_intrhand == NULL) {
3593 printf(": couldn't establish interrupt");
3594 if (intrstr != NULL)
3595 printf(" at %s", intrstr);
3596 printf("\n");
3597 return (ENXIO);
3598 }
3599 printf(": %s", intrstr);
3600
3601 return (0);
3602 }
3603
3604 #if NKSTAT > 0
3605 /* this is used to look up the array of kstats quickly */
3606 enum em_stat {
3607 em_stat_crcerrs,
3608 em_stat_algnerrc,
3609 em_stat_symerrs,
3610 em_stat_rxerrc,
3611 em_stat_mpc,
3612 em_stat_scc,
3613 em_stat_ecol,
3614 em_stat_mcc,
3615 em_stat_latecol,
3616 em_stat_colc,
3617 em_stat_dc,
3618 em_stat_tncrs,
3619 em_stat_sec,
3620 em_stat_cexterr,
3621 em_stat_rlec,
3622 em_stat_xonrxc,
3623 em_stat_xontxc,
3624 em_stat_xoffrxc,
3625 em_stat_xofftxc,
3626 em_stat_fcruc,
3627 em_stat_prc64,
3628 em_stat_prc127,
3629 em_stat_prc255,
3630 em_stat_prc511,
3631 em_stat_prc1023,
3632 em_stat_prc1522,
3633 em_stat_gprc,
3634 em_stat_bprc,
3635 em_stat_mprc,
3636 em_stat_gptc,
3637 em_stat_gorc,
3638 em_stat_gotc,
3639 em_stat_rnbc,
3640 em_stat_ruc,
3641 em_stat_rfc,
3642 em_stat_roc,
3643 em_stat_rjc,
3644 em_stat_mgtprc,
3645 em_stat_mgtpdc,
3646 em_stat_mgtptc,
3647 em_stat_tor,
3648 em_stat_tot,
3649 em_stat_tpr,
3650 em_stat_tpt,
3651 em_stat_ptc64,
3652 em_stat_ptc127,
3653 em_stat_ptc255,
3654 em_stat_ptc511,
3655 em_stat_ptc1023,
3656 em_stat_ptc1522,
3657 em_stat_mptc,
3658 em_stat_bptc,
3659 #if 0
3660 em_stat_tsctc,
3661 em_stat_tsctf,
3662 #endif
3663
3664 em_stat_count,
3665 };
3666
3667 struct em_counter {
3668 const char *name;
3669 enum kstat_kv_unit unit;
3670 uint32_t reg;
3671 };
3672
3673 static const struct em_counter em_counters[em_stat_count] = {
3674 [em_stat_crcerrs] =
3675 { "rx crc errs", KSTAT_KV_U_PACKETS, E1000_CRCERRS },
3676 [em_stat_algnerrc] = /* >= em_82543 */
3677 { "rx align errs", KSTAT_KV_U_PACKETS, 0 },
3678 [em_stat_symerrs] = /* >= em_82543 */
3679 { "rx align errs", KSTAT_KV_U_PACKETS, 0 },
3680 [em_stat_rxerrc] =
3681 { "rx errs", KSTAT_KV_U_PACKETS, E1000_RXERRC },
3682 [em_stat_mpc] =
3683 { "rx missed", KSTAT_KV_U_PACKETS, E1000_MPC },
3684 [em_stat_scc] =
3685 { "tx single coll", KSTAT_KV_U_PACKETS, E1000_SCC },
3686 [em_stat_ecol] =
3687 { "tx excess coll", KSTAT_KV_U_PACKETS, E1000_ECOL },
3688 [em_stat_mcc] =
3689 { "tx multi coll", KSTAT_KV_U_PACKETS, E1000_MCC },
3690 [em_stat_latecol] =
3691 { "tx late coll", KSTAT_KV_U_PACKETS, E1000_LATECOL },
3692 [em_stat_colc] =
3693 { "tx coll", KSTAT_KV_U_NONE, E1000_COLC },
3694 [em_stat_dc] =
3695 { "tx defers", KSTAT_KV_U_NONE, E1000_DC },
3696 [em_stat_tncrs] = /* >= em_82543 */
3697 { "tx no CRS", KSTAT_KV_U_PACKETS, 0 },
3698 [em_stat_sec] =
3699 { "seq errs", KSTAT_KV_U_NONE, E1000_SEC },
3700 [em_stat_cexterr] = /* >= em_82543 */
3701 { "carr ext errs", KSTAT_KV_U_PACKETS, 0 },
3702 [em_stat_rlec] =
3703 { "rx len errs", KSTAT_KV_U_PACKETS, E1000_RLEC },
3704 [em_stat_xonrxc] =
3705 { "rx xon", KSTAT_KV_U_PACKETS, E1000_XONRXC },
3706 [em_stat_xontxc] =
3707 { "tx xon", KSTAT_KV_U_PACKETS, E1000_XONTXC },
3708 [em_stat_xoffrxc] =
3709 { "rx xoff", KSTAT_KV_U_PACKETS, E1000_XOFFRXC },
3710 [em_stat_xofftxc] =
3711 { "tx xoff", KSTAT_KV_U_PACKETS, E1000_XOFFTXC },
3712 [em_stat_fcruc] =
3713 { "FC unsupported", KSTAT_KV_U_PACKETS, E1000_FCRUC },
3714 [em_stat_prc64] =
3715 { "rx 64B", KSTAT_KV_U_PACKETS, E1000_PRC64 },
3716 [em_stat_prc127] =
3717 { "rx 65-127B", KSTAT_KV_U_PACKETS, E1000_PRC127 },
3718 [em_stat_prc255] =
3719 { "rx 128-255B", KSTAT_KV_U_PACKETS, E1000_PRC255 },
3720 [em_stat_prc511] =
3721 { "rx 256-511B", KSTAT_KV_U_PACKETS, E1000_PRC511 },
3722 [em_stat_prc1023] =
3723 { "rx 512-1023B", KSTAT_KV_U_PACKETS, E1000_PRC1023 },
3724 [em_stat_prc1522] =
3725 { "rx 1024-maxB", KSTAT_KV_U_PACKETS, E1000_PRC1522 },
3726 [em_stat_gprc] =
3727 { "rx good", KSTAT_KV_U_PACKETS, E1000_GPRC },
3728 [em_stat_bprc] =
3729 { "rx bcast", KSTAT_KV_U_PACKETS, E1000_BPRC },
3730 [em_stat_mprc] =
3731 { "rx mcast", KSTAT_KV_U_PACKETS, E1000_MPRC },
3732 [em_stat_gptc] =
3733 { "tx good", KSTAT_KV_U_PACKETS, E1000_GPTC },
3734 [em_stat_gorc] = /* 64bit */
3735 { "rx good", KSTAT_KV_U_BYTES, 0 },
3736 [em_stat_gotc] = /* 64bit */
3737 { "tx good", KSTAT_KV_U_BYTES, 0 },
3738 [em_stat_rnbc] =
3739 { "rx no buffers", KSTAT_KV_U_PACKETS, E1000_RNBC },
3740 [em_stat_ruc] =
3741 { "rx undersize", KSTAT_KV_U_PACKETS, E1000_RUC },
3742 [em_stat_rfc] =
3743 { "rx fragments", KSTAT_KV_U_PACKETS, E1000_RFC },
3744 [em_stat_roc] =
3745 { "rx oversize", KSTAT_KV_U_PACKETS, E1000_ROC },
3746 [em_stat_rjc] =
3747 { "rx jabbers", KSTAT_KV_U_PACKETS, E1000_RJC },
3748 [em_stat_mgtprc] =
3749 { "rx mgmt", KSTAT_KV_U_PACKETS, E1000_MGTPRC },
3750 [em_stat_mgtpdc] =
3751 { "rx mgmt drops", KSTAT_KV_U_PACKETS, E1000_MGTPDC },
3752 [em_stat_mgtptc] =
3753 { "tx mgmt", KSTAT_KV_U_PACKETS, E1000_MGTPTC },
3754 [em_stat_tor] = /* 64bit */
3755 { "rx total", KSTAT_KV_U_BYTES, 0 },
3756 [em_stat_tot] = /* 64bit */
3757 { "tx total", KSTAT_KV_U_BYTES, 0 },
3758 [em_stat_tpr] =
3759 { "rx total", KSTAT_KV_U_PACKETS, E1000_TPR },
3760 [em_stat_tpt] =
3761 { "tx total", KSTAT_KV_U_PACKETS, E1000_TPT },
3762 [em_stat_ptc64] =
3763 { "tx 64B", KSTAT_KV_U_PACKETS, E1000_PTC64 },
3764 [em_stat_ptc127] =
3765 { "tx 65-127B", KSTAT_KV_U_PACKETS, E1000_PTC127 },
3766 [em_stat_ptc255] =
3767 { "tx 128-255B", KSTAT_KV_U_PACKETS, E1000_PTC255 },
3768 [em_stat_ptc511] =
3769 { "tx 256-511B", KSTAT_KV_U_PACKETS, E1000_PTC511 },
3770 [em_stat_ptc1023] =
3771 { "tx 512-1023B", KSTAT_KV_U_PACKETS, E1000_PTC1023 },
3772 [em_stat_ptc1522] =
3773 { "tx 1024-maxB", KSTAT_KV_U_PACKETS, E1000_PTC1522 },
3774 [em_stat_mptc] =
3775 { "tx mcast", KSTAT_KV_U_PACKETS, E1000_MPTC },
3776 [em_stat_bptc] =
3777 { "tx bcast", KSTAT_KV_U_PACKETS, E1000_BPTC },
3778 };
3779
3780 /**********************************************************************
3781 *
3782 * Update the board statistics counters.
3783 *
3784 **********************************************************************/
3785 int
em_kstat_read(struct kstat * ks)3786 em_kstat_read(struct kstat *ks)
3787 {
3788 struct em_softc *sc = ks->ks_softc;
3789 struct em_hw *hw = &sc->hw;
3790 struct kstat_kv *kvs = ks->ks_data;
3791 uint32_t lo, hi;
3792 unsigned int i;
3793
3794 for (i = 0; i < nitems(em_counters); i++) {
3795 const struct em_counter *c = &em_counters[i];
3796 if (c->reg == 0)
3797 continue;
3798
3799 kstat_kv_u64(&kvs[i]) += EM_READ_REG(hw,
3800 E1000_REG_TR(hw, c->reg)); /* wtf */
3801 }
3802
3803 /* Handle the exceptions. */
3804
3805 if (sc->hw.mac_type >= em_82543) {
3806 kstat_kv_u64(&kvs[em_stat_algnerrc]) +=
3807 E1000_READ_REG(hw, ALGNERRC);
3808 kstat_kv_u64(&kvs[em_stat_rxerrc]) +=
3809 E1000_READ_REG(hw, RXERRC);
3810 kstat_kv_u64(&kvs[em_stat_cexterr]) +=
3811 E1000_READ_REG(hw, CEXTERR);
3812 kstat_kv_u64(&kvs[em_stat_tncrs]) +=
3813 E1000_READ_REG(hw, TNCRS);
3814 #if 0
3815 sc->stats.tsctc +=
3816 E1000_READ_REG(hw, TSCTC);
3817 sc->stats.tsctfc +=
3818 E1000_READ_REG(hw, TSCTFC);
3819 #endif
3820 }
3821
3822 /* For the 64-bit byte counters the low dword must be read first. */
3823 /* Both registers clear on the read of the high dword */
3824
3825 lo = E1000_READ_REG(hw, GORCL);
3826 hi = E1000_READ_REG(hw, GORCH);
3827 kstat_kv_u64(&kvs[em_stat_gorc]) +=
3828 ((uint64_t)hi << 32) | (uint64_t)lo;
3829
3830 lo = E1000_READ_REG(hw, GOTCL);
3831 hi = E1000_READ_REG(hw, GOTCH);
3832 kstat_kv_u64(&kvs[em_stat_gotc]) +=
3833 ((uint64_t)hi << 32) | (uint64_t)lo;
3834
3835 lo = E1000_READ_REG(hw, TORL);
3836 hi = E1000_READ_REG(hw, TORH);
3837 kstat_kv_u64(&kvs[em_stat_tor]) +=
3838 ((uint64_t)hi << 32) | (uint64_t)lo;
3839
3840 lo = E1000_READ_REG(hw, TOTL);
3841 hi = E1000_READ_REG(hw, TOTH);
3842 kstat_kv_u64(&kvs[em_stat_tot]) +=
3843 ((uint64_t)hi << 32) | (uint64_t)lo;
3844
3845 getnanouptime(&ks->ks_updated);
3846
3847 return (0);
3848 }
3849
3850 void
em_kstat_attach(struct em_softc * sc)3851 em_kstat_attach(struct em_softc *sc)
3852 {
3853 struct kstat *ks;
3854 struct kstat_kv *kvs;
3855 unsigned int i;
3856
3857 mtx_init(&sc->kstat_mtx, IPL_SOFTCLOCK);
3858
3859 ks = kstat_create(DEVNAME(sc), 0, "em-stats", 0,
3860 KSTAT_T_KV, 0);
3861 if (ks == NULL)
3862 return;
3863
3864 kvs = mallocarray(nitems(em_counters), sizeof(*kvs),
3865 M_DEVBUF, M_WAITOK|M_ZERO);
3866 for (i = 0; i < nitems(em_counters); i++) {
3867 const struct em_counter *c = &em_counters[i];
3868 kstat_kv_unit_init(&kvs[i], c->name,
3869 KSTAT_KV_T_COUNTER64, c->unit);
3870 }
3871
3872 ks->ks_softc = sc;
3873 ks->ks_data = kvs;
3874 ks->ks_datalen = nitems(em_counters) * sizeof(*kvs);
3875 ks->ks_read = em_kstat_read;
3876 kstat_set_mutex(ks, &sc->kstat_mtx);
3877
3878 kstat_install(ks);
3879 }
3880
3881 /******************************************************************************
3882 * Adjusts the statistic counters when a frame is accepted by TBI_ACCEPT
3883 *****************************************************************************/
3884 void
em_tbi_adjust_stats(struct em_softc * sc,uint32_t frame_len,uint8_t * mac_addr)3885 em_tbi_adjust_stats(struct em_softc *sc, uint32_t frame_len, uint8_t *mac_addr)
3886 {
3887 struct em_hw *hw = &sc->hw;
3888 struct kstat *ks = sc->kstat;
3889 struct kstat_kv *kvs;
3890
3891 if (ks == NULL)
3892 return;
3893
3894 /* First adjust the frame length. */
3895 frame_len--;
3896
3897 mtx_enter(&sc->kstat_mtx);
3898 kvs = ks->ks_data;
3899
3900 /*
3901 * We need to adjust the statistics counters, since the hardware
3902 * counters overcount this packet as a CRC error and undercount the
3903 * packet as a good packet
3904 */
3905
3906 /* This packet should not be counted as a CRC error. */
3907 kstat_kv_u64(&kvs[em_stat_crcerrs])--;
3908 /* This packet does count as a Good Packet Received. */
3909 kstat_kv_u64(&kvs[em_stat_gprc])++;
3910
3911 /* Adjust the Good Octets received counters */
3912 kstat_kv_u64(&kvs[em_stat_gorc]) += frame_len;
3913
3914 /*
3915 * Is this a broadcast or multicast? Check broadcast first, since
3916 * the test for a multicast frame will test positive on a broadcast
3917 * frame.
3918 */
3919 if (ETHER_IS_BROADCAST(mac_addr)) {
3920 /* Broadcast packet */
3921 kstat_kv_u64(&kvs[em_stat_bprc])++;
3922 } else if (ETHER_IS_MULTICAST(mac_addr)) {
3923 /* Multicast packet */
3924 kstat_kv_u64(&kvs[em_stat_mprc])++;
3925 }
3926
3927 if (frame_len == hw->max_frame_size) {
3928 /*
3929 * In this case, the hardware has overcounted the number of
3930 * oversize frames.
3931 */
3932 kstat_kv_u64(&kvs[em_stat_roc])--;
3933 }
3934
3935 /*
3936 * Adjust the bin counters when the extra byte put the frame in the
3937 * wrong bin. Remember that the frame_len was adjusted above.
3938 */
3939 if (frame_len == 64) {
3940 kstat_kv_u64(&kvs[em_stat_prc64])++;
3941 kstat_kv_u64(&kvs[em_stat_prc127])--;
3942 } else if (frame_len == 127) {
3943 kstat_kv_u64(&kvs[em_stat_prc127])++;
3944 kstat_kv_u64(&kvs[em_stat_prc255])--;
3945 } else if (frame_len == 255) {
3946 kstat_kv_u64(&kvs[em_stat_prc255])++;
3947 kstat_kv_u64(&kvs[em_stat_prc511])--;
3948 } else if (frame_len == 511) {
3949 kstat_kv_u64(&kvs[em_stat_prc511])++;
3950 kstat_kv_u64(&kvs[em_stat_prc1023])--;
3951 } else if (frame_len == 1023) {
3952 kstat_kv_u64(&kvs[em_stat_prc1023])++;
3953 kstat_kv_u64(&kvs[em_stat_prc1522])--;
3954 } else if (frame_len == 1522) {
3955 kstat_kv_u64(&kvs[em_stat_prc1522])++;
3956 }
3957
3958 mtx_leave(&sc->kstat_mtx);
3959 }
3960 #endif /* NKSTAT > 0 */
3961
3962 #ifndef SMALL_KERNEL
3963 int
em_allocate_msix(struct em_softc * sc)3964 em_allocate_msix(struct em_softc *sc)
3965 {
3966 pci_intr_handle_t ih;
3967 const char *intrstr = NULL;
3968 struct pci_attach_args *pa = &sc->osdep.em_pa;
3969 pci_chipset_tag_t pc = pa->pa_pc;
3970 struct em_queue *que = sc->queues; /* Use only first queue. */
3971 int vec;
3972
3973 if (!em_enable_msix)
3974 return (ENODEV);
3975
3976 switch (sc->hw.mac_type) {
3977 case em_82576:
3978 case em_82580:
3979 case em_i350:
3980 case em_i210:
3981 break;
3982 default:
3983 return (ENODEV);
3984 }
3985
3986 vec = 0;
3987 if (pci_intr_map_msix(pa, vec, &ih))
3988 return (ENODEV);
3989 sc->msix = 1;
3990
3991 que->me = vec;
3992 que->eims = 1 << vec;
3993 snprintf(que->name, sizeof(que->name), "%s:%d", DEVNAME(sc), vec);
3994
3995 intrstr = pci_intr_string(pc, ih);
3996 que->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
3997 em_queue_intr_msix, que, que->name);
3998 if (que->tag == NULL) {
3999 printf(": couldn't establish interrupt");
4000 if (intrstr != NULL)
4001 printf(" at %s", intrstr);
4002 printf("\n");
4003 return (ENXIO);
4004 }
4005
4006 /* Setup linkvector, use last queue vector + 1 */
4007 vec++;
4008 sc->msix_linkvec = vec;
4009 if (pci_intr_map_msix(pa, sc->msix_linkvec, &ih)) {
4010 printf(": couldn't map link vector\n");
4011 return (ENXIO);
4012 }
4013
4014 intrstr = pci_intr_string(pc, ih);
4015 sc->sc_intrhand = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
4016 em_link_intr_msix, sc, DEVNAME(sc));
4017 if (sc->sc_intrhand == NULL) {
4018 printf(": couldn't establish interrupt");
4019 if (intrstr != NULL)
4020 printf(" at %s", intrstr);
4021 printf("\n");
4022 return (ENXIO);
4023 }
4024 printf(", %s, %d queue%s", intrstr, vec, (vec > 1) ? "s" : "");
4025
4026 return (0);
4027 }
4028
4029 /*
4030 * Interrupt for a specific queue, (not link interrupts). The EICR bit which
4031 * maps to the EIMS bit expresses both RX and TX, therefore we can't
4032 * distinguish if this is a RX completion of TX completion and must do both.
4033 * The bits in EICR are autocleared and we _cannot_ read EICR.
4034 */
4035 int
em_queue_intr_msix(void * vque)4036 em_queue_intr_msix(void *vque)
4037 {
4038 struct em_queue *que = vque;
4039 struct em_softc *sc = que->sc;
4040 struct ifnet *ifp = &sc->sc_ac.ac_if;
4041
4042 if (ifp->if_flags & IFF_RUNNING) {
4043 em_txeof(que);
4044 if (em_rxeof(que))
4045 em_rxrefill_locked(que);
4046 }
4047
4048 em_enable_queue_intr_msix(que);
4049
4050 return (1);
4051 }
4052
4053 int
em_link_intr_msix(void * arg)4054 em_link_intr_msix(void *arg)
4055 {
4056 struct em_softc *sc = arg;
4057 uint32_t icr;
4058
4059 icr = E1000_READ_REG(&sc->hw, ICR);
4060
4061 /* Link status change */
4062 if (icr & E1000_ICR_LSC) {
4063 KERNEL_LOCK();
4064 sc->hw.get_link_status = 1;
4065 em_check_for_link(&sc->hw);
4066 em_update_link_status(sc);
4067 KERNEL_UNLOCK();
4068 }
4069
4070 /* Re-arm unconditionally */
4071 E1000_WRITE_REG(&sc->hw, IMS, E1000_ICR_LSC);
4072 E1000_WRITE_REG(&sc->hw, EIMS, sc->msix_linkmask);
4073
4074 return (1);
4075 }
4076
4077 /*
4078 * Maps queues into msix interrupt vectors.
4079 */
4080 int
em_setup_queues_msix(struct em_softc * sc)4081 em_setup_queues_msix(struct em_softc *sc)
4082 {
4083 uint32_t ivar, newitr, index;
4084 struct em_queue *que;
4085
4086 KASSERT(sc->msix);
4087
4088 /* First turn on RSS capability */
4089 if (sc->hw.mac_type != em_82575)
4090 E1000_WRITE_REG(&sc->hw, GPIE,
4091 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
4092 E1000_GPIE_PBA | E1000_GPIE_NSICR);
4093
4094 /* Turn on MSIX */
4095 switch (sc->hw.mac_type) {
4096 case em_82580:
4097 case em_i350:
4098 case em_i210:
4099 /* RX entries */
4100 /*
4101 * Note, this maps Queues into MSIX vectors, it works fine.
4102 * The funky calculation of offsets and checking if que->me is
4103 * odd is due to the weird register distribution, the datasheet
4104 * explains it well.
4105 */
4106 FOREACH_QUEUE(sc, que) {
4107 index = que->me >> 1;
4108 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4109 if (que->me & 1) {
4110 ivar &= 0xFF00FFFF;
4111 ivar |= (que->me | E1000_IVAR_VALID) << 16;
4112 } else {
4113 ivar &= 0xFFFFFF00;
4114 ivar |= que->me | E1000_IVAR_VALID;
4115 }
4116 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4117 }
4118
4119 /* TX entries */
4120 FOREACH_QUEUE(sc, que) {
4121 index = que->me >> 1;
4122 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4123 if (que->me & 1) {
4124 ivar &= 0x00FFFFFF;
4125 ivar |= (que->me | E1000_IVAR_VALID) << 24;
4126 } else {
4127 ivar &= 0xFFFF00FF;
4128 ivar |= (que->me | E1000_IVAR_VALID) << 8;
4129 }
4130 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4131 sc->msix_queuesmask |= que->eims;
4132 }
4133
4134 /* And for the link interrupt */
4135 ivar = (sc->msix_linkvec | E1000_IVAR_VALID) << 8;
4136 sc->msix_linkmask = 1 << sc->msix_linkvec;
4137 E1000_WRITE_REG(&sc->hw, IVAR_MISC, ivar);
4138 break;
4139 case em_82576:
4140 /* RX entries */
4141 FOREACH_QUEUE(sc, que) {
4142 index = que->me & 0x7; /* Each IVAR has two entries */
4143 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4144 if (que->me < 8) {
4145 ivar &= 0xFFFFFF00;
4146 ivar |= que->me | E1000_IVAR_VALID;
4147 } else {
4148 ivar &= 0xFF00FFFF;
4149 ivar |= (que->me | E1000_IVAR_VALID) << 16;
4150 }
4151 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4152 sc->msix_queuesmask |= que->eims;
4153 }
4154 /* TX entries */
4155 FOREACH_QUEUE(sc, que) {
4156 index = que->me & 0x7; /* Each IVAR has two entries */
4157 ivar = E1000_READ_REG_ARRAY(&sc->hw, IVAR0, index);
4158 if (que->me < 8) {
4159 ivar &= 0xFFFF00FF;
4160 ivar |= (que->me | E1000_IVAR_VALID) << 8;
4161 } else {
4162 ivar &= 0x00FFFFFF;
4163 ivar |= (que->me | E1000_IVAR_VALID) << 24;
4164 }
4165 E1000_WRITE_REG_ARRAY(&sc->hw, IVAR0, index, ivar);
4166 sc->msix_queuesmask |= que->eims;
4167 }
4168
4169 /* And for the link interrupt */
4170 ivar = (sc->msix_linkvec | E1000_IVAR_VALID) << 8;
4171 sc->msix_linkmask = 1 << sc->msix_linkvec;
4172 E1000_WRITE_REG(&sc->hw, IVAR_MISC, ivar);
4173 break;
4174 default:
4175 panic("unsupported mac");
4176 break;
4177 }
4178
4179 /* Set the starting interrupt rate */
4180 newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC;
4181
4182 if (sc->hw.mac_type == em_82575)
4183 newitr |= newitr << 16;
4184 else
4185 newitr |= E1000_EITR_CNT_IGNR;
4186
4187 FOREACH_QUEUE(sc, que)
4188 E1000_WRITE_REG(&sc->hw, EITR(que->me), newitr);
4189
4190 return (0);
4191 }
4192
4193 void
em_enable_queue_intr_msix(struct em_queue * que)4194 em_enable_queue_intr_msix(struct em_queue *que)
4195 {
4196 E1000_WRITE_REG(&que->sc->hw, EIMS, que->eims);
4197 }
4198 #endif /* !SMALL_KERNEL */
4199
4200 int
em_allocate_desc_rings(struct em_softc * sc)4201 em_allocate_desc_rings(struct em_softc *sc)
4202 {
4203 struct em_queue *que;
4204
4205 FOREACH_QUEUE(sc, que) {
4206 /* Allocate Transmit Descriptor ring */
4207 if (em_dma_malloc(sc, sc->sc_tx_slots * sizeof(struct em_tx_desc),
4208 &que->tx.sc_tx_dma) != 0) {
4209 printf("%s: Unable to allocate tx_desc memory\n",
4210 DEVNAME(sc));
4211 return (ENOMEM);
4212 }
4213 que->tx.sc_tx_desc_ring =
4214 (struct em_tx_desc *)que->tx.sc_tx_dma.dma_vaddr;
4215
4216 /* Allocate Receive Descriptor ring */
4217 if (em_dma_malloc(sc, sc->sc_rx_slots * sizeof(struct em_rx_desc),
4218 &que->rx.sc_rx_dma) != 0) {
4219 printf("%s: Unable to allocate rx_desc memory\n",
4220 DEVNAME(sc));
4221 return (ENOMEM);
4222 }
4223 que->rx.sc_rx_desc_ring =
4224 (struct em_rx_desc *)que->rx.sc_rx_dma.dma_vaddr;
4225 }
4226
4227 return (0);
4228 }
4229
4230 int
em_get_sffpage(struct em_softc * sc,struct if_sffpage * sff)4231 em_get_sffpage(struct em_softc *sc, struct if_sffpage *sff)
4232 {
4233 struct em_hw *hw = &sc->hw;
4234 size_t i;
4235 int off;
4236
4237 if (hw->mac_type != em_82575 && hw->mac_type != em_82580 &&
4238 hw->mac_type != em_82576 &&
4239 hw->mac_type != em_i210 && hw->mac_type != em_i350)
4240 return (ENODEV);
4241
4242 if (sff->sff_addr == IFSFF_ADDR_EEPROM)
4243 off = E1000_I2CCMD_SFP_DATA_ADDR(0);
4244 else if (sff->sff_addr == IFSFF_ADDR_DDM)
4245 off = E1000_I2CCMD_SFP_DIAG_ADDR(0);
4246 else
4247 return (EIO);
4248
4249 for (i = 0; i < sizeof(sff->sff_data); i++) {
4250 if (em_read_sfp_data_byte(hw, off + i,
4251 &sff->sff_data[i]) != E1000_SUCCESS)
4252 return (EIO);
4253 }
4254
4255 return (0);
4256 }
4257