1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2023 Google LLC
5 *
6 * Redistribution and use in source and binary forms, with or without modification,
7 * are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * 3. Neither the name of the copyright holder nor the names of its contributors
17 * may be used to endorse or promote products derived from this software without
18 * specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31 #include "gve.h"
32 #include "gve_adminq.h"
33
34 #define GVE_DRIVER_VERSION "GVE-FBSD-1.0.1\n"
35 #define GVE_VERSION_MAJOR 1
36 #define GVE_VERSION_MINOR 0
37 #define GVE_VERSION_SUB 1
38
39 #define GVE_DEFAULT_RX_COPYBREAK 256
40
41 /* Devices supported by this driver. */
42 static struct gve_dev {
43 uint16_t vendor_id;
44 uint16_t device_id;
45 const char *name;
46 } gve_devs[] = {
47 { PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC, "gVNIC" }
48 };
49
50 struct sx gve_global_lock;
51
52 static int
gve_verify_driver_compatibility(struct gve_priv * priv)53 gve_verify_driver_compatibility(struct gve_priv *priv)
54 {
55 int err;
56 struct gve_driver_info *driver_info;
57 struct gve_dma_handle driver_info_mem;
58
59 err = gve_dma_alloc_coherent(priv, sizeof(struct gve_driver_info),
60 PAGE_SIZE, &driver_info_mem);
61
62 if (err != 0)
63 return (ENOMEM);
64
65 driver_info = driver_info_mem.cpu_addr;
66
67 *driver_info = (struct gve_driver_info) {
68 .os_type = 3, /* Freebsd */
69 .driver_major = GVE_VERSION_MAJOR,
70 .driver_minor = GVE_VERSION_MINOR,
71 .driver_sub = GVE_VERSION_SUB,
72 .os_version_major = htobe32(FBSD_VERSION_MAJOR),
73 .os_version_minor = htobe32(FBSD_VERSION_MINOR),
74 .os_version_sub = htobe32(FBSD_VERSION_PATCH),
75 .driver_capability_flags = {
76 htobe64(GVE_DRIVER_CAPABILITY_FLAGS1),
77 htobe64(GVE_DRIVER_CAPABILITY_FLAGS2),
78 htobe64(GVE_DRIVER_CAPABILITY_FLAGS3),
79 htobe64(GVE_DRIVER_CAPABILITY_FLAGS4),
80 },
81 };
82
83 snprintf(driver_info->os_version_str1, sizeof(driver_info->os_version_str1),
84 "FreeBSD %u", __FreeBSD_version);
85
86 bus_dmamap_sync(driver_info_mem.tag, driver_info_mem.map,
87 BUS_DMASYNC_PREREAD);
88
89 err = gve_adminq_verify_driver_compatibility(priv,
90 sizeof(struct gve_driver_info), driver_info_mem.bus_addr);
91
92 /* It's ok if the device doesn't support this */
93 if (err == EOPNOTSUPP)
94 err = 0;
95
96 gve_dma_free_coherent(&driver_info_mem);
97
98 return (err);
99 }
100
101 static int
gve_up(struct gve_priv * priv)102 gve_up(struct gve_priv *priv)
103 {
104 if_t ifp = priv->ifp;
105 int err;
106
107 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
108
109 if (device_is_attached(priv->dev) == 0) {
110 device_printf(priv->dev, "Cannot bring the iface up when detached\n");
111 return (ENXIO);
112 }
113
114 if (gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
115 return (0);
116
117 if_clearhwassist(ifp);
118 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
119 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
120 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
121 if_sethwassistbits(ifp, CSUM_IP6_TCP | CSUM_IP6_UDP, 0);
122 if (if_getcapenable(ifp) & IFCAP_TSO4)
123 if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
124 if (if_getcapenable(ifp) & IFCAP_TSO6)
125 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
126
127 err = gve_register_qpls(priv);
128 if (err != 0)
129 goto reset;
130
131 err = gve_create_rx_rings(priv);
132 if (err != 0)
133 goto reset;
134
135 err = gve_create_tx_rings(priv);
136 if (err != 0)
137 goto reset;
138
139 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
140
141 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
142 if_link_state_change(ifp, LINK_STATE_UP);
143 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
144 }
145
146 gve_unmask_all_queue_irqs(priv);
147 gve_set_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
148 priv->interface_up_cnt++;
149 return (0);
150
151 reset:
152 gve_schedule_reset(priv);
153 return (err);
154 }
155
156 static void
gve_down(struct gve_priv * priv)157 gve_down(struct gve_priv *priv)
158 {
159 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
160
161 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
162 return;
163
164 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
165 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
166 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
167 }
168
169 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
170
171 if (gve_destroy_rx_rings(priv) != 0)
172 goto reset;
173
174 if (gve_destroy_tx_rings(priv) != 0)
175 goto reset;
176
177 if (gve_unregister_qpls(priv) != 0)
178 goto reset;
179
180 gve_mask_all_queue_irqs(priv);
181 gve_clear_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
182 priv->interface_down_cnt++;
183 return;
184
185 reset:
186 gve_schedule_reset(priv);
187 }
188
189 static int
gve_set_mtu(if_t ifp,uint32_t new_mtu)190 gve_set_mtu(if_t ifp, uint32_t new_mtu)
191 {
192 struct gve_priv *priv = if_getsoftc(ifp);
193 int err;
194
195 if ((new_mtu > priv->max_mtu) || (new_mtu < ETHERMIN)) {
196 device_printf(priv->dev, "Invalid new MTU setting. new mtu: %d max mtu: %d min mtu: %d\n",
197 new_mtu, priv->max_mtu, ETHERMIN);
198 return (EINVAL);
199 }
200
201 err = gve_adminq_set_mtu(priv, new_mtu);
202 if (err == 0) {
203 if (bootverbose)
204 device_printf(priv->dev, "MTU set to %d\n", new_mtu);
205 if_setmtu(ifp, new_mtu);
206 } else {
207 device_printf(priv->dev, "Failed to set MTU to %d\n", new_mtu);
208 }
209
210 return (err);
211 }
212
213 static void
gve_init(void * arg)214 gve_init(void *arg)
215 {
216 struct gve_priv *priv = (struct gve_priv *)arg;
217
218 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) {
219 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
220 gve_up(priv);
221 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
222 }
223 }
224
225 static int
gve_ioctl(if_t ifp,u_long command,caddr_t data)226 gve_ioctl(if_t ifp, u_long command, caddr_t data)
227 {
228 struct gve_priv *priv;
229 struct ifreq *ifr;
230 int rc = 0;
231
232 priv = if_getsoftc(ifp);
233 ifr = (struct ifreq *)data;
234
235 switch (command) {
236 case SIOCSIFMTU:
237 if (if_getmtu(ifp) == ifr->ifr_mtu)
238 break;
239 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
240 gve_down(priv);
241 gve_set_mtu(ifp, ifr->ifr_mtu);
242 rc = gve_up(priv);
243 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
244 break;
245
246 case SIOCSIFFLAGS:
247 if ((if_getflags(ifp) & IFF_UP) != 0) {
248 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
249 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
250 rc = gve_up(priv);
251 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
252 }
253 } else {
254 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
255 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
256 gve_down(priv);
257 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
258 }
259 }
260 break;
261
262 case SIOCSIFCAP:
263 if (ifr->ifr_reqcap == if_getcapenable(ifp))
264 break;
265 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
266 gve_down(priv);
267 if_setcapenable(ifp, ifr->ifr_reqcap);
268 rc = gve_up(priv);
269 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
270 break;
271
272 case SIOCSIFMEDIA:
273 /* FALLTHROUGH */
274 case SIOCGIFMEDIA:
275 rc = ifmedia_ioctl(ifp, ifr, &priv->media, command);
276 break;
277
278 default:
279 rc = ether_ioctl(ifp, command, data);
280 break;
281 }
282
283 return (rc);
284 }
285
286 static int
gve_media_change(if_t ifp)287 gve_media_change(if_t ifp)
288 {
289 struct gve_priv *priv = if_getsoftc(ifp);
290
291 device_printf(priv->dev, "Media change not supported\n");
292 return (0);
293 }
294
295 static void
gve_media_status(if_t ifp,struct ifmediareq * ifmr)296 gve_media_status(if_t ifp, struct ifmediareq *ifmr)
297 {
298 struct gve_priv *priv = if_getsoftc(ifp);
299
300 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
301
302 ifmr->ifm_status = IFM_AVALID;
303 ifmr->ifm_active = IFM_ETHER;
304
305 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
306 ifmr->ifm_status |= IFM_ACTIVE;
307 ifmr->ifm_active |= IFM_AUTO;
308 } else {
309 ifmr->ifm_active |= IFM_NONE;
310 }
311
312 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
313 }
314
315 static uint64_t
gve_get_counter(if_t ifp,ift_counter cnt)316 gve_get_counter(if_t ifp, ift_counter cnt)
317 {
318 struct gve_priv *priv;
319 uint64_t rpackets = 0;
320 uint64_t tpackets = 0;
321 uint64_t rbytes = 0;
322 uint64_t tbytes = 0;
323 uint64_t rx_dropped_pkt = 0;
324 uint64_t tx_dropped_pkt = 0;
325
326 priv = if_getsoftc(ifp);
327
328 gve_accum_stats(priv, &rpackets, &rbytes, &rx_dropped_pkt, &tpackets,
329 &tbytes, &tx_dropped_pkt);
330
331 switch (cnt) {
332 case IFCOUNTER_IPACKETS:
333 return (rpackets);
334
335 case IFCOUNTER_OPACKETS:
336 return (tpackets);
337
338 case IFCOUNTER_IBYTES:
339 return (rbytes);
340
341 case IFCOUNTER_OBYTES:
342 return (tbytes);
343
344 case IFCOUNTER_IQDROPS:
345 return (rx_dropped_pkt);
346
347 case IFCOUNTER_OQDROPS:
348 return (tx_dropped_pkt);
349
350 default:
351 return (if_get_counter_default(ifp, cnt));
352 }
353 }
354
355 static int
gve_setup_ifnet(device_t dev,struct gve_priv * priv)356 gve_setup_ifnet(device_t dev, struct gve_priv *priv)
357 {
358 int caps = 0;
359 if_t ifp;
360
361 ifp = priv->ifp = if_alloc(IFT_ETHER);
362 if (ifp == NULL) {
363 device_printf(priv->dev, "Failed to allocate ifnet struct\n");
364 return (ENXIO);
365 }
366
367 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
368 if_setsoftc(ifp, priv);
369 if_setdev(ifp, dev);
370 if_setinitfn(ifp, gve_init);
371 if_setioctlfn(ifp, gve_ioctl);
372 if_settransmitfn(ifp, gve_xmit_ifp);
373 if_setqflushfn(ifp, gve_qflush);
374
375 #if __FreeBSD_version >= 1400086
376 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
377 #else
378 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_KNOWSEPOCH);
379 #endif
380
381 ifmedia_init(&priv->media, IFM_IMASK, gve_media_change, gve_media_status);
382 if_setgetcounterfn(ifp, gve_get_counter);
383
384 caps = IFCAP_RXCSUM |
385 IFCAP_TXCSUM |
386 IFCAP_TXCSUM_IPV6 |
387 IFCAP_TSO |
388 IFCAP_LRO;
389
390 if ((priv->supported_features & GVE_SUP_JUMBO_FRAMES_MASK) != 0)
391 caps |= IFCAP_JUMBO_MTU;
392
393 if_setcapabilities(ifp, caps);
394 if_setcapenable(ifp, caps);
395
396 if (bootverbose)
397 device_printf(priv->dev, "Setting initial MTU to %d\n", priv->max_mtu);
398 if_setmtu(ifp, priv->max_mtu);
399
400 ether_ifattach(ifp, priv->mac);
401
402 ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
403 ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO);
404
405 return (0);
406 }
407
408 static int
gve_alloc_counter_array(struct gve_priv * priv)409 gve_alloc_counter_array(struct gve_priv *priv)
410 {
411 int err;
412
413 err = gve_dma_alloc_coherent(priv, sizeof(uint32_t) * priv->num_event_counters,
414 PAGE_SIZE, &priv->counter_array_mem);
415 if (err != 0)
416 return (err);
417
418 priv->counters = priv->counter_array_mem.cpu_addr;
419 return (0);
420 }
421
422 static void
gve_free_counter_array(struct gve_priv * priv)423 gve_free_counter_array(struct gve_priv *priv)
424 {
425 if (priv->counters != NULL)
426 gve_dma_free_coherent(&priv->counter_array_mem);
427 priv->counter_array_mem = (struct gve_dma_handle){};
428 }
429
430 static int
gve_alloc_irq_db_array(struct gve_priv * priv)431 gve_alloc_irq_db_array(struct gve_priv *priv)
432 {
433 int err;
434
435 err = gve_dma_alloc_coherent(priv,
436 sizeof(struct gve_irq_db) * (priv->num_queues), PAGE_SIZE,
437 &priv->irqs_db_mem);
438 if (err != 0)
439 return (err);
440
441 priv->irq_db_indices = priv->irqs_db_mem.cpu_addr;
442 return (0);
443 }
444
445 static void
gve_free_irq_db_array(struct gve_priv * priv)446 gve_free_irq_db_array(struct gve_priv *priv)
447 {
448 if (priv->irq_db_indices != NULL)
449 gve_dma_free_coherent(&priv->irqs_db_mem);
450 priv->irqs_db_mem = (struct gve_dma_handle){};
451 }
452
453 static void
gve_free_rings(struct gve_priv * priv)454 gve_free_rings(struct gve_priv *priv)
455 {
456 gve_free_irqs(priv);
457 gve_free_tx_rings(priv);
458 gve_free_rx_rings(priv);
459 gve_free_qpls(priv);
460 }
461
462 static int
gve_alloc_rings(struct gve_priv * priv)463 gve_alloc_rings(struct gve_priv *priv)
464 {
465 int err;
466
467 err = gve_alloc_qpls(priv);
468 if (err != 0)
469 goto abort;
470
471 err = gve_alloc_rx_rings(priv);
472 if (err != 0)
473 goto abort;
474
475 err = gve_alloc_tx_rings(priv);
476 if (err != 0)
477 goto abort;
478
479 err = gve_alloc_irqs(priv);
480 if (err != 0)
481 goto abort;
482
483 return (0);
484
485 abort:
486 gve_free_rings(priv);
487 return (err);
488 }
489
490 static void
gve_deconfigure_resources(struct gve_priv * priv)491 gve_deconfigure_resources(struct gve_priv *priv)
492 {
493 int err;
494
495 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK)) {
496 err = gve_adminq_deconfigure_device_resources(priv);
497 if (err != 0) {
498 device_printf(priv->dev, "Failed to deconfigure device resources: err=%d\n",
499 err);
500 return;
501 }
502 if (bootverbose)
503 device_printf(priv->dev, "Deconfigured device resources\n");
504 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
505 }
506
507 gve_free_irq_db_array(priv);
508 gve_free_counter_array(priv);
509 }
510
511 static int
gve_configure_resources(struct gve_priv * priv)512 gve_configure_resources(struct gve_priv *priv)
513 {
514 int err;
515
516 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK))
517 return (0);
518
519 err = gve_alloc_counter_array(priv);
520 if (err != 0)
521 return (err);
522
523 err = gve_alloc_irq_db_array(priv);
524 if (err != 0)
525 goto abort;
526
527 err = gve_adminq_configure_device_resources(priv);
528 if (err != 0) {
529 device_printf(priv->dev, "Failed to configure device resources: err=%d\n",
530 err);
531 err = (ENXIO);
532 goto abort;
533 }
534
535 gve_set_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
536 if (bootverbose)
537 device_printf(priv->dev, "Configured device resources\n");
538 return (0);
539
540 abort:
541 gve_deconfigure_resources(priv);
542 return (err);
543 }
544
545 static void
gve_set_queue_cnts(struct gve_priv * priv)546 gve_set_queue_cnts(struct gve_priv *priv)
547 {
548 priv->tx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_TX_QUEUES);
549 priv->rx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_RX_QUEUES);
550 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
551 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
552
553 if (priv->default_num_queues > 0) {
554 priv->tx_cfg.num_queues = MIN(priv->default_num_queues,
555 priv->tx_cfg.num_queues);
556 priv->rx_cfg.num_queues = MIN(priv->default_num_queues,
557 priv->rx_cfg.num_queues);
558 }
559
560 priv->num_queues = priv->tx_cfg.num_queues + priv->rx_cfg.num_queues;
561 priv->mgmt_msix_idx = priv->num_queues;
562 }
563
564 static int
gve_alloc_adminq_and_describe_device(struct gve_priv * priv)565 gve_alloc_adminq_and_describe_device(struct gve_priv *priv)
566 {
567 int err;
568
569 if ((err = gve_adminq_alloc(priv)) != 0)
570 return (err);
571
572 if ((err = gve_verify_driver_compatibility(priv)) != 0) {
573 device_printf(priv->dev,
574 "Failed to verify driver compatibility: err=%d\n", err);
575 goto abort;
576 }
577
578 if ((err = gve_adminq_describe_device(priv)) != 0)
579 goto abort;
580
581 gve_set_queue_cnts(priv);
582
583 priv->num_registered_pages = 0;
584 return (0);
585
586 abort:
587 gve_release_adminq(priv);
588 return (err);
589 }
590
591 void
gve_schedule_reset(struct gve_priv * priv)592 gve_schedule_reset(struct gve_priv *priv)
593 {
594 if (gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET))
595 return;
596
597 device_printf(priv->dev, "Scheduling reset task!\n");
598 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
599 taskqueue_enqueue(priv->service_tq, &priv->service_task);
600 }
601
602 static void
gve_destroy(struct gve_priv * priv)603 gve_destroy(struct gve_priv *priv)
604 {
605 gve_down(priv);
606 gve_deconfigure_resources(priv);
607 gve_release_adminq(priv);
608 }
609
610 static void
gve_restore(struct gve_priv * priv)611 gve_restore(struct gve_priv *priv)
612 {
613 int err;
614
615 err = gve_adminq_alloc(priv);
616 if (err != 0)
617 goto abort;
618
619 err = gve_configure_resources(priv);
620 if (err != 0)
621 goto abort;
622
623 err = gve_up(priv);
624 if (err != 0)
625 goto abort;
626
627 return;
628
629 abort:
630 device_printf(priv->dev, "Restore failed!\n");
631 return;
632 }
633
634 static void
gve_handle_reset(struct gve_priv * priv)635 gve_handle_reset(struct gve_priv *priv)
636 {
637 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_DO_RESET))
638 return;
639
640 gve_clear_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
641 gve_set_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
642
643 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
644
645 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
646 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
647 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
648
649 /*
650 * Releasing the adminq causes the NIC to destroy all resources
651 * registered with it, so by clearing the flags beneath we cause
652 * the subsequent gve_down call below to not attempt to tell the
653 * NIC to destroy these resources again.
654 *
655 * The call to gve_down is needed in the first place to refresh
656 * the state and the DMA-able memory within each driver ring.
657 */
658 gve_release_adminq(priv);
659 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
660 gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK);
661 gve_clear_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);
662 gve_clear_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK);
663
664 gve_down(priv);
665 gve_restore(priv);
666
667 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
668
669 priv->reset_cnt++;
670 gve_clear_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
671 }
672
673 static void
gve_handle_link_status(struct gve_priv * priv)674 gve_handle_link_status(struct gve_priv *priv)
675 {
676 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
677 bool link_up = status & GVE_DEVICE_STATUS_LINK_STATUS;
678
679 if (link_up == gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP))
680 return;
681
682 if (link_up) {
683 if (bootverbose)
684 device_printf(priv->dev, "Device link is up.\n");
685 if_link_state_change(priv->ifp, LINK_STATE_UP);
686 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
687 } else {
688 device_printf(priv->dev, "Device link is down.\n");
689 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
690 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
691 }
692 }
693
694 static void
gve_service_task(void * arg,int pending)695 gve_service_task(void *arg, int pending)
696 {
697 struct gve_priv *priv = (struct gve_priv *)arg;
698 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
699
700 if (((GVE_DEVICE_STATUS_RESET_MASK & status) != 0) &&
701 !gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET)) {
702 device_printf(priv->dev, "Device requested reset\n");
703 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
704 }
705
706 gve_handle_reset(priv);
707 gve_handle_link_status(priv);
708 }
709
710 static int
gve_probe(device_t dev)711 gve_probe(device_t dev)
712 {
713 uint16_t deviceid, vendorid;
714 int i;
715
716 vendorid = pci_get_vendor(dev);
717 deviceid = pci_get_device(dev);
718
719 for (i = 0; i < nitems(gve_devs); i++) {
720 if (vendorid == gve_devs[i].vendor_id &&
721 deviceid == gve_devs[i].device_id) {
722 device_set_desc(dev, gve_devs[i].name);
723 return (BUS_PROBE_DEFAULT);
724 }
725 }
726 return (ENXIO);
727 }
728
729 static void
gve_free_sys_res_mem(struct gve_priv * priv)730 gve_free_sys_res_mem(struct gve_priv *priv)
731 {
732 if (priv->msix_table != NULL)
733 bus_release_resource(priv->dev, SYS_RES_MEMORY,
734 rman_get_rid(priv->msix_table), priv->msix_table);
735
736 if (priv->db_bar != NULL)
737 bus_release_resource(priv->dev, SYS_RES_MEMORY,
738 rman_get_rid(priv->db_bar), priv->db_bar);
739
740 if (priv->reg_bar != NULL)
741 bus_release_resource(priv->dev, SYS_RES_MEMORY,
742 rman_get_rid(priv->reg_bar), priv->reg_bar);
743 }
744
745 static int
gve_attach(device_t dev)746 gve_attach(device_t dev)
747 {
748 struct gve_priv *priv;
749 int rid;
750 int err;
751
752 priv = device_get_softc(dev);
753 priv->dev = dev;
754 GVE_IFACE_LOCK_INIT(priv->gve_iface_lock);
755
756 pci_enable_busmaster(dev);
757
758 rid = PCIR_BAR(GVE_REGISTER_BAR);
759 priv->reg_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
760 &rid, RF_ACTIVE);
761 if (priv->reg_bar == NULL) {
762 device_printf(dev, "Failed to allocate BAR0\n");
763 err = ENXIO;
764 goto abort;
765 }
766
767 rid = PCIR_BAR(GVE_DOORBELL_BAR);
768 priv->db_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
769 &rid, RF_ACTIVE);
770 if (priv->db_bar == NULL) {
771 device_printf(dev, "Failed to allocate BAR2\n");
772 err = ENXIO;
773 goto abort;
774 }
775
776 rid = pci_msix_table_bar(priv->dev);
777 priv->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
778 &rid, RF_ACTIVE);
779 if (priv->msix_table == NULL) {
780 device_printf(dev, "Failed to allocate msix table\n");
781 err = ENXIO;
782 goto abort;
783 }
784
785 err = gve_alloc_adminq_and_describe_device(priv);
786 if (err != 0)
787 goto abort;
788
789 err = gve_configure_resources(priv);
790 if (err != 0)
791 goto abort;
792
793 err = gve_alloc_rings(priv);
794 if (err != 0)
795 goto abort;
796
797 err = gve_setup_ifnet(dev, priv);
798 if (err != 0)
799 goto abort;
800
801 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
802
803 bus_write_multi_1(priv->reg_bar, DRIVER_VERSION, GVE_DRIVER_VERSION,
804 sizeof(GVE_DRIVER_VERSION) - 1);
805
806 TASK_INIT(&priv->service_task, 0, gve_service_task, priv);
807 priv->service_tq = taskqueue_create("gve service", M_WAITOK | M_ZERO,
808 taskqueue_thread_enqueue, &priv->service_tq);
809 taskqueue_start_threads(&priv->service_tq, 1, PI_NET, "%s service tq",
810 device_get_nameunit(priv->dev));
811
812 gve_setup_sysctl(priv);
813
814 if (bootverbose)
815 device_printf(priv->dev, "Successfully attached %s", GVE_DRIVER_VERSION);
816 return (0);
817
818 abort:
819 gve_free_rings(priv);
820 gve_deconfigure_resources(priv);
821 gve_release_adminq(priv);
822 gve_free_sys_res_mem(priv);
823 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
824 return (err);
825 }
826
827 static int
gve_detach(device_t dev)828 gve_detach(device_t dev)
829 {
830 struct gve_priv *priv = device_get_softc(dev);
831 if_t ifp = priv->ifp;
832
833 ether_ifdetach(ifp);
834
835 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
836 gve_destroy(priv);
837 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
838
839 gve_free_rings(priv);
840 gve_free_sys_res_mem(priv);
841 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
842
843 while (taskqueue_cancel(priv->service_tq, &priv->service_task, NULL))
844 taskqueue_drain(priv->service_tq, &priv->service_task);
845 taskqueue_free(priv->service_tq);
846
847 if_free(ifp);
848 return (bus_generic_detach(dev));
849 }
850
851 static device_method_t gve_methods[] = {
852 DEVMETHOD(device_probe, gve_probe),
853 DEVMETHOD(device_attach, gve_attach),
854 DEVMETHOD(device_detach, gve_detach),
855 DEVMETHOD_END
856 };
857
858 static driver_t gve_driver = {
859 "gve",
860 gve_methods,
861 sizeof(struct gve_priv)
862 };
863
864 #if __FreeBSD_version < 1301503
865 static devclass_t gve_devclass;
866
867 DRIVER_MODULE(gve, pci, gve_driver, gve_devclass, 0, 0);
868 #else
869 DRIVER_MODULE(gve, pci, gve_driver, 0, 0);
870 #endif
871 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, gve, gve_devs,
872 nitems(gve_devs));
873